Merge branch 'master' into bratseth/autoscaling-completion

54 files changed, 452 insertions, 526 deletions

diff --git a/config-model/src/main/java/com/yahoo/schema/expressiontransforms/InputRecorder.java b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/InputRecorder.java
index 515745cb6bc..7124628be0c 100644
--- a/config-model/src/main/java/com/yahoo/schema/expressiontransforms/InputRecorder.java
+++ b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/InputRecorder.java
@@ -13,6 +13,7 @@ import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode;
 import com.yahoo.searchlib.rankingexpression.transform.ExpressionTransformer;
 
 import java.io.StringReader;
+import java.util.HashSet;
 import java.util.Set;
 
 /**
@@ -23,6 +24,7 @@ import java.util.Set;
 public class InputRecorder extends ExpressionTransformer<RankProfileTransformContext> {
 
     private final Set<String> neededInputs;
+    private final Set<String> handled = new HashSet<>();
 
     public InputRecorder(Set<String> target) {
         this.neededInputs = target;
@@ -52,9 +54,13 @@ public class InputRecorder extends ExpressionTransformer<RankProfileTransformCon
             simpleFunctionOrIdentifier = true;
         }
         if (simpleFunctionOrIdentifier) {
+            if (handled.contains(name)) {
+                return;
+            }
             var f = context.rankProfile().getFunctions().get(name);
             if (f != null && f.function().arguments().size() == 0) {
                 transform(f.function().getBody().getRoot(), context);
+                handled.add(name);
                 return;
             }
             neededInputs.add(feature.toString());
diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/LoggingRequestHandler.java b/container-core/src/main/java/com/yahoo/container/jdisc/LoggingRequestHandler.java
index d608977a7ff..bacceac9d76 100644
--- a/container-core/src/main/java/com/yahoo/container/jdisc/LoggingRequestHandler.java
+++ b/container-core/src/main/java/com/yahoo/container/jdisc/LoggingRequestHandler.java
@@ -3,7 +3,6 @@ package com.yahoo.container.jdisc;
 
 import com.yahoo.component.annotation.Inject;
 import com.yahoo.container.handler.Timing;
-import com.yahoo.container.logging.AccessLog;
 import com.yahoo.container.logging.AccessLogEntry;
 import com.yahoo.jdisc.Metric;
 import com.yahoo.jdisc.Response;
@@ -102,54 +101,54 @@ public abstract class LoggingRequestHandler extends ThreadedHttpRequestHandler {
 
     private void logTimes(long startTime, String sourceIP,
                           long renderStartTime, long commitStartTime, long endTime,
-                          String req, String normalizedQuery, Timing t) {
+                          String req, ExtendedResponse response) {
 
         // note: intentionally only taking time since request was received
         long totalTime = endTime - startTime;
 
-        long timeoutInterval = Long.MAX_VALUE;
-        long requestOverhead = 0;
-        long summaryStartTime = 0;
+        long timeoutInterval;
+        long requestOverhead;
+        long summaryStartTime;
+        Timing t = response.getTiming();
         if (t != null) {
             timeoutInterval = t.getTimeout();
             long queryStartTime = t.getQueryStartTime();
-            if (queryStartTime > 0) {
-                requestOverhead = queryStartTime - startTime;
-            }
+            requestOverhead = (queryStartTime > 0) ? queryStartTime - startTime : 0;
             summaryStartTime = t.getSummaryStartTime();
-        }
-
-        if (totalTime <= timeoutInterval) {
-            return;
-        }
-
-        StringBuilder b = new StringBuilder();
-        b.append(normalizedQuery);
-        b.append(" from ").append(sourceIP).append(". ");
-
-        if (requestOverhead > 0) {
-            b.append("Time from HTTP connection open to request reception ");
-            b.append(requestOverhead).append(" ms. ");
-        }
-        if (summaryStartTime != 0) {
-            b.append("Request time: ");
-            b.append(summaryStartTime - startTime).append(" ms. ");
-            b.append("Summary fetch time: ");
-            b.append(renderStartTime - summaryStartTime).append(" ms. ");
         } else {
-            long spentSearching = renderStartTime - startTime;
-            b.append("Processing time: ").append(spentSearching).append(" ms. ");
+            requestOverhead = 0;
+            summaryStartTime = 0;
+            timeoutInterval = Long.MAX_VALUE;
         }
 
-        b.append("Result rendering/transfer: ");
-        b.append(commitStartTime - renderStartTime).append(" ms. ");
-        b.append("End transaction: ");
-        b.append(endTime - commitStartTime).append(" ms. ");
-        b.append("Total: ").append(totalTime).append(" ms. ");
-        b.append("Timeout: ").append(timeoutInterval).append(" ms. ");
-        b.append("Request string: ").append(req);
+        if (totalTime <= timeoutInterval) return;
 
-        log.log(Level.WARNING, "Slow execution. " + b);
+        log.log(Level.FINE, () -> {
+            StringBuilder b = new StringBuilder();
+            b.append(response.getParsedQuery());
+            b.append(" from ").append(sourceIP).append(". ");
+            if (requestOverhead > 0) {
+                b.append("Time from HTTP connection open to request reception ");
+                b.append(requestOverhead).append(" ms. ");
+            }
+            if (summaryStartTime != 0) {
+                b.append("Request time: ");
+                b.append(summaryStartTime - startTime).append(" ms. ");
+                b.append("Summary fetch time: ");
+                b.append(renderStartTime - summaryStartTime).append(" ms. ");
+            } else {
+                long spentSearching = renderStartTime - startTime;
+                b.append("Processing time: ").append(spentSearching).append(" ms. ");
+            }
+            b.append("Result rendering/transfer: ");
+            b.append(commitStartTime - renderStartTime).append(" ms. ");
+            b.append("End transaction: ");
+            b.append(endTime - commitStartTime).append(" ms. ");
+            b.append("Total: ").append(totalTime).append(" ms. ");
+            b.append("Timeout: ").append(timeoutInterval).append(" ms. ");
+            b.append("Request string: ").append(req);
+            return b.toString();
+        });
     }
 
     private static class NullResponse extends ExtendedResponse {
@@ -224,8 +223,7 @@ public abstract class LoggingRequestHandler extends ThreadedHttpRequestHandler {
                      commitStartTime,
                      endTime,
                      getUri(jdiscRequest),
-                     extendedResponse.getParsedQuery(),
-                     extendedResponse.getTiming());
+                     extendedResponse);
 
             Optional<AccessLogEntry> jdiscRequestAccessLogEntry =
                     AccessLoggingRequestHandler.getAccessLogEntry(jdiscRequest);
diff --git a/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/TensorConverter.java b/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/TensorConverter.java
index 9c79961eddf..eef75a32c0a 100644
--- a/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/TensorConverter.java
+++ b/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/TensorConverter.java
@@ -180,12 +180,25 @@ class TensorConverter {
     }
 
     static private TensorType.Value toVespaValueType(TensorInfo.OnnxTensorType onnxType) {
+        // NOTE:
+        // should match best_cell_type in onnx_wrapper.cpp
         switch (onnxType) {
-            case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8: return TensorType.Value.INT8;
-            case ONNX_TENSOR_ELEMENT_DATA_TYPE_BFLOAT16: return TensorType.Value.BFLOAT16;
-            case ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT: return TensorType.Value.FLOAT;
-            case ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE: return TensorType.Value.DOUBLE;
-        }
+            case ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL:
+            case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8:
+                return TensorType.Value.INT8;
+
+            case ONNX_TENSOR_ELEMENT_DATA_TYPE_BFLOAT16:
+                return TensorType.Value.BFLOAT16;
+
+            case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8:
+            case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16:
+            case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16:
+            case ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT:
+                return TensorType.Value.FLOAT;
+
+            case ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE:
+                return TensorType.Value.DOUBLE;
+            }
         return TensorType.Value.DOUBLE;
     }
 
diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/onnx/TypeConverter.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/onnx/TypeConverter.java
index 35ec1d8c54a..2c008dbb922 100644
--- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/onnx/TypeConverter.java
+++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/onnx/TypeConverter.java
@@ -56,21 +56,27 @@ class TypeConverter {
                                                    tensor.getDimsList());
     }
 
-    private static TensorType.Value toValueType(Onnx.TensorProto.DataType dataType) {
-        switch (dataType) {
-            case FLOAT: return TensorType.Value.FLOAT;
-            case DOUBLE: return TensorType.Value.DOUBLE;
-            // Imperfect conversion, for now:
-            case BOOL: return TensorType.Value.FLOAT;
-            case INT8: return TensorType.Value.FLOAT;
-            case INT16: return TensorType.Value.FLOAT;
-            case INT32: return TensorType.Value.FLOAT;
-            case INT64: return TensorType.Value.FLOAT;
-            case UINT8: return TensorType.Value.FLOAT;
-            case UINT16: return TensorType.Value.FLOAT;
-            case UINT32: return TensorType.Value.FLOAT;
-            case UINT64: return TensorType.Value.FLOAT;
-            default: throw new IllegalArgumentException("A ONNX tensor with data type " + dataType +
+    private static TensorType.Value toValueType(Onnx.TensorProto.DataType onnxType) {
+        // NOTE:
+        // should match best_cell_type in onnx_wrapper.cpp
+        switch (onnxType) {
+            case BOOL: // Imperfect conversion fallthrough
+            case INT8:
+                return TensorType.Value.INT8;
+            case BFLOAT16:
+                return TensorType.Value.BFLOAT16;
+            case UINT8: // Imperfect conversion fallthrough
+            case INT16: // Imperfect conversion fallthrough
+            case UINT16: // Imperfect conversion fallthrough
+            case FLOAT:
+                return TensorType.Value.FLOAT;
+            case INT32: // Imperfect conversion fallthrough
+            case INT64: // Imperfect conversion fallthrough
+            case UINT32: // Imperfect conversion fallthrough
+            case UINT64: // Imperfect conversion fallthrough
+            case DOUBLE:
+                return TensorType.Value.DOUBLE;
+            default: throw new IllegalArgumentException("A ONNX tensor with data type " + onnxType +
                                                         " cannot be converted to a Vespa tensor type");
         }
     }
diff --git a/model-integration/src/main/protobuf/onnx.proto b/model-integration/src/main/protobuf/onnx.proto
index dc6542867e0..27f1fdef4b3 100644
--- a/model-integration/src/main/protobuf/onnx.proto
+++ b/model-integration/src/main/protobuf/onnx.proto
@@ -298,6 +298,10 @@ message TensorProto {
     UINT64 = 13;
     COMPLEX64 = 14;     // complex with float32 real and imaginary components
     COMPLEX128 = 15;    // complex with float64 real and imaginary components
+    // Non-IEEE floating-point format based on IEEE754 single-precision
+    // floating-point number truncated to 16 bits.
+    // This format has 1 sign bit, 8 exponent bits, and 7 mantissa bits.
+    BFLOAT16 = 16;
     // Future extensions go here.
   }
 
@@ -461,4 +465,4 @@ message OperatorSetIdProto {
   // The version of the operator set being identified.
   // This field MUST be present in this version of the IR.
   optional int64 version = 2;
-}
\ No newline at end of file
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Applications.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Applications.java
index 9cfed5d046c..8daac029c7b 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Applications.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Applications.java
@@ -73,4 +73,9 @@ public class Applications {
         return db.lock(application, timeout);
     }
 
+    /** Create a lock which provides exclusive rights to perform a maintenance deployment */
+    public Mutex lockMaintenance(ApplicationId application) {
+        return db.lockMaintenance(application);
+    }
+
 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java
index b3ff0c42547..50eee9e33b3 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java
@@ -46,15 +46,18 @@ public abstract class ApplicationMaintainer extends NodeRepositoryMaintainer {
         return 1.0;
     }
 
+    protected final Deployer deployer() { return deployer; }
+
     /** Returns the number of deployments that are pending execution */
     public int pendingDeployments() {
         return pendingDeployments.size();
     }
 
     /** Returns whether given application should be deployed at this moment in time */
-    protected boolean canDeployNow(ApplicationId application) {
-        return true;
-    }
+    protected abstract boolean canDeployNow(ApplicationId application);
+
+    /** Returns the applications that should be maintained by this now. */
+    protected abstract Map<ApplicationId, String> applicationsNeedingMaintenance();
 
     /**
      * Redeploy this application.
@@ -64,19 +67,14 @@ public abstract class ApplicationMaintainer extends NodeRepositoryMaintainer {
      */
     protected void deploy(ApplicationId application, String reason) {
         if (pendingDeployments.addIfAbsent(application)) { // Avoid queuing multiple deployments for same application
-            deploymentExecutor.execute(() -> deployWithLock(application, reason));
+            deploymentExecutor.execute(() -> deployNow(application, reason));
         }
     }
 
-    protected Deployer deployer() { return deployer; }
-
-    /** Returns the applications that should be maintained by this now. */
-    protected abstract Map<ApplicationId, String> applicationsNeedingMaintenance();
-
     /**
      * Redeploy this application. A lock will be taken for the duration of the deployment activation
      */
-    protected final void deployWithLock(ApplicationId application, String reason) {
+    protected final void deployNow(ApplicationId application, String reason) {
         try (MaintenanceDeployment deployment = new MaintenanceDeployment(application, deployer, metric, nodeRepository())) {
             if ( ! deployment.isValid()) return; // this will be done at another config server
             if ( ! canDeployNow(application)) return; // redeployment is no longer needed
@@ -97,7 +95,7 @@ public abstract class ApplicationMaintainer extends NodeRepositoryMaintainer {
     @Override
     public void shutdown() {
         super.shutdown();
-        this.deploymentExecutor.shutdownNow();
+        deploymentExecutor.shutdownNow();
     }
 
     @Override
@@ -105,7 +103,9 @@ public abstract class ApplicationMaintainer extends NodeRepositoryMaintainer {
         super.awaitShutdown();
         try {
             // Give deployments in progress some time to complete
-            this.deploymentExecutor.awaitTermination(1, TimeUnit.MINUTES);
+            if (!deploymentExecutor.awaitTermination(1, TimeUnit.MINUTES)) {
+                log.log(Level.WARNING, "Failed to shut down deployment executor within deadline");
+            }
         } catch (InterruptedException e) {
             throw new RuntimeException(e);
         }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java
index 9fffdcf34e1..4c9fab748d1 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java
@@ -6,7 +6,6 @@ import com.yahoo.config.provision.ApplicationLockException;
 import com.yahoo.config.provision.ClusterResources;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.Deployer;
-import com.yahoo.config.provision.Environment;
 import com.yahoo.jdisc.Metric;
 import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.NodeList;
@@ -19,6 +18,7 @@ import com.yahoo.vespa.hosted.provision.autoscale.Autoscaler;
 import com.yahoo.vespa.hosted.provision.autoscale.Autoscaling;
 import com.yahoo.vespa.hosted.provision.autoscale.NodeMetricSnapshot;
 import com.yahoo.vespa.hosted.provision.node.History;
+
 import java.time.Duration;
 import java.time.Instant;
 import java.util.Map;
@@ -68,6 +68,7 @@ public class AutoscalingMaintainer extends NodeRepositoryMaintainer {
      * @return true if an autoscaling decision was made or nothing should be done, false if there was an error
      */
     private boolean autoscale(ApplicationId applicationId, ClusterSpec.Id clusterId) {
+        boolean redeploy = false;
         try (var lock = nodeRepository().applications().lock(applicationId)) {
             Optional<Application> application = nodeRepository().applications().get(applicationId);
             if (application.isEmpty()) return true;
@@ -93,16 +94,10 @@ public class AutoscalingMaintainer extends NodeRepositoryMaintainer {
                 applications().put(application.get().with(cluster), lock);
 
             // Attempt to perform the autoscaling immediately, and log it regardless
-            if (autoscaling != null
-                && autoscaling.resources().isPresent()
-                && !current.equals(autoscaling.resources().get())) {
-                try (MaintenanceDeployment deployment = new MaintenanceDeployment(applicationId, deployer, metric, nodeRepository())) {
-                    if (deployment.isValid())
-                        deployment.activate();
-                    logAutoscaling(current, autoscaling.resources().get(), applicationId, clusterNodes.not().retired());
-                }
+            if (autoscaling != null && autoscaling.resources().isPresent() && !current.equals(autoscaling.resources().get())) {
+                redeploy = true;
+                logAutoscaling(current, autoscaling.resources().get(), applicationId, clusterNodes.not().retired());
             }
-            return true;
         }
         catch (ApplicationLockException e) {
             return false;
@@ -110,6 +105,13 @@ public class AutoscalingMaintainer extends NodeRepositoryMaintainer {
         catch (IllegalArgumentException e) {
             throw new IllegalArgumentException("Illegal arguments for " + applicationId + " cluster " + clusterId, e);
         }
+        if (redeploy) {
+            try (MaintenanceDeployment deployment = new MaintenanceDeployment(applicationId, deployer, metric, nodeRepository())) {
+                if (deployment.isValid())
+                    deployment.activate();
+            }
+        }
+        return true;
     }
 
     private Applications applications() {
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ExpeditedChangeApplicationMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ExpeditedChangeApplicationMaintainer.java
index d048f43973a..ffa125230a8 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ExpeditedChangeApplicationMaintainer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ExpeditedChangeApplicationMaintainer.java
@@ -16,7 +16,6 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
-import java.util.stream.Collectors;
 
 /**
  * This maintainer detects changes to nodes that must be expedited, and redeploys affected applications.
@@ -40,25 +39,21 @@ public class ExpeditedChangeApplicationMaintainer extends ApplicationMaintainer
 
     @Override
     protected Map<ApplicationId, String> applicationsNeedingMaintenance() {
-        var applications = new HashMap<ApplicationId, String>();
-
-        nodeRepository().nodes()
-                        .list()
-                        .nodeType(NodeType.tenant, NodeType.proxy)
-                        .matching(node -> node.allocation().isPresent())
-                        .groupingBy(node -> node.allocation().get().owner())
-                        .forEach((applicationId, nodes) -> {
-                            hasNodesWithChanges(applicationId, nodes)
-                                    .ifPresent(reason -> applications.put(applicationId, reason));
-                        });
-
+        NodeList allNodes = nodeRepository().nodes().list();
+        Map<ApplicationId, String> applications = new HashMap<>();
+        allNodes.nodeType(NodeType.tenant, NodeType.proxy)
+                .matching(node -> node.allocation().isPresent())
+                .groupingBy(node -> node.allocation().get().owner())
+                .forEach((applicationId, nodes) -> {
+                    hasNodesWithChanges(applicationId, nodes)
+                            .ifPresent(reason -> applications.put(applicationId, reason));
+                });
         // A ready proxy node should trigger a redeployment as it will activate the node.
-        if (!nodeRepository().nodes().list(Node.State.ready, Node.State.reserved).nodeType(NodeType.proxy).isEmpty()) {
+        if (!allNodes.state(Node.State.ready, Node.State.reserved).nodeType(NodeType.proxy).isEmpty()) {
             applications.merge(ApplicationId.from("hosted-vespa", "routing", "default"),
                                "nodes being ready",
                                (oldValue, newValue) -> oldValue + ", " + newValue);
         }
-
         return applications;
     }
 
@@ -68,7 +63,7 @@ public class ExpeditedChangeApplicationMaintainer extends ApplicationMaintainer
      */
     @Override
     protected void deploy(ApplicationId application, String reason) {
-        deployWithLock(application, reason);
+        deployNow(application, reason);
     }
 
     /** Returns the reason for doing an expedited deploy. */
@@ -78,11 +73,11 @@ public class ExpeditedChangeApplicationMaintainer extends ApplicationMaintainer
 
         List<String> reasons = nodes.stream()
                                     .flatMap(node -> node.history()
-                                                            .events()
-                                                            .stream()
-                                                            .filter(event -> expediteChangeBy(event.agent()))
-                                                            .filter(event -> lastDeployTime.get().isBefore(event.at()))
-                                                            .map(event -> event.type() + (event.agent() == Agent.system ? "" : " by " + event.agent())))
+                                                         .events()
+                                                         .stream()
+                                                         .filter(event -> expediteChangeBy(event.agent()))
+                                                         .filter(event -> lastDeployTime.get().isBefore(event.at()))
+                                                         .map(event -> event.type() + (event.agent() == Agent.system ? "" : " by " + event.agent())))
                                     .sorted()
                                     .distinct()
                                     .toList();
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceDeployment.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceDeployment.java
index bcc571355e3..189238a1c11 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceDeployment.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceDeployment.java
@@ -1,8 +1,8 @@
 // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.provision.maintenance;
 
+import com.yahoo.concurrent.UncheckedTimeoutException;
 import com.yahoo.config.provision.ApplicationId;
-import com.yahoo.config.provision.ApplicationLockException;
 import com.yahoo.config.provision.Deployer;
 import com.yahoo.config.provision.Deployment;
 import com.yahoo.config.provision.TransientException;
@@ -16,7 +16,6 @@ import com.yahoo.vespa.hosted.provision.node.Agent;
 import com.yahoo.yolean.Exceptions;
 
 import java.io.Closeable;
-import java.time.Duration;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
@@ -63,18 +62,12 @@ class MaintenanceDeployment implements Closeable {
         return deployment.isPresent();
     }
 
-    /**
-     * Returns the application lock held by this, or empty if it is not held.
-     *
-     * @throws IllegalStateException id this is called when closed
-     */
-    public Optional<Mutex> applicationLock() {
-        if (closed) throw new IllegalStateException(this + " is closed");
-        return lock;
-    }
-
+    /** Prepare this deployment. Returns whether prepare was successful */
     public boolean prepare() {
-        return doStep(() -> { deployment.get().prepare(); return 0L; }).isPresent();
+        return doStep(() -> {
+            deployment.get().prepare();
+            return 0L;
+        }).isPresent();
     }
 
     /**
@@ -104,13 +97,10 @@ class MaintenanceDeployment implements Closeable {
     }
 
     private Optional<Mutex> tryLock(ApplicationId application, NodeRepository nodeRepository) {
-        Duration timeout = Duration.ofSeconds(3);
         try {
-            // Use a short lock to avoid interfering with change deployments
-            return Optional.of(nodeRepository.applications().lock(application, timeout));
-        }
-        catch (ApplicationLockException e) {
-            log.log(Level.INFO, () -> "Could not lock " + application + " for maintenance deployment within " + timeout);
+            return Optional.of(nodeRepository.applications().lockMaintenance(application));
+        } catch (UncheckedTimeoutException e) {
+            log.log(Level.INFO, () -> "Could not lock " + application + " for maintenance deployment within timeout");
             return Optional.empty();
         }
     }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java
index 84a45de39d7..afea08711fa 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java
@@ -182,27 +182,26 @@ public class NodeFailer extends NodeRepositoryMaintainer {
      * Called when a node should be moved to the failed state: Do that if it seems safe,
      * which is when the node repo has available capacity to replace the node (and all its tenant nodes if host).
      * Otherwise not replacing the node ensures (by Orchestrator check) that no further action will be taken.
-     *
-     * @return whether node was successfully failed
      */
-    private boolean failActive(FailingNode failing) {
+    private void failActive(FailingNode failing) {
         Optional<Deployment> deployment =
             deployer.deployFromLocalActive(failing.node().allocation().get().owner(), Duration.ofMinutes(5));
-        if (deployment.isEmpty()) return false;
+        if (deployment.isEmpty()) return;
 
         // If the active node that we are trying to fail is of type host, we need to successfully fail all
         // the children nodes running on it before we fail the host.  Failing a child node in a dynamically
         // provisioned zone may require provisioning new hosts that require the host application lock to be held,
         // so we must release ours before failing the children.
         List<FailingNode> activeChildrenToFail = new ArrayList<>();
+        boolean redeploy = false;
         try (NodeMutex lock = nodeRepository().nodes().lockAndGetRequired(failing.node())) {
             // Now that we have gotten the node object under the proper lock, sanity-check it still makes sense to fail
             if (!Objects.equals(failing.node().allocation().map(Allocation::owner), lock.node().allocation().map(Allocation::owner)))
-                return false;
+                return;
             if (lock.node().state() == Node.State.failed)
-                return true;
+                return;
             if (!Objects.equals(failing.node().state(), lock.node().state()))
-                return false;
+                return;
             failing = new FailingNode(lock.node(), failing.reason);
 
             String reasonForChildFailure = "Failing due to parent host " + failing.node().hostname() + " failure: " + failing.reason();
@@ -216,36 +215,46 @@ public class NodeFailer extends NodeRepositoryMaintainer {
 
             if (activeChildrenToFail.isEmpty()) {
                 log.log(Level.INFO, "Failing out " + failing.node + ": " + failing.reason);
-                wantToFail(failing.node(), true, lock);
-                try {
-                    deployment.get().activate();
-                    return true;
-                } catch (TransientException | UncheckedTimeoutException e) {
-                    log.log(Level.INFO, "Failed to redeploy " + failing.node().allocation().get().owner() +
-                                        " with a transient error, will be retried by application maintainer: " +
-                                        Exceptions.toMessageString(e));
-                    return true;
-                } catch (RuntimeException e) {
-                    // Reset want to fail: We'll retry failing unless it heals in the meantime
-                    nodeRepository().nodes().node(failing.node().hostname())
-                                    .ifPresent(n -> wantToFail(n, false, lock));
-                    log.log(Level.WARNING, "Could not fail " + failing.node() + " for " + failing.node().allocation().get().owner() +
-                                           " for " + failing.reason() + ": " + Exceptions.toMessageString(e));
-                    return false;
-                }
+                markWantToFail(failing.node(), true, lock);
+                redeploy = true;
             }
         }
 
+        // Redeploy to replace failing node
+        if (redeploy) {
+            redeploy(deployment.get(), failing);
+            return;
+        }
+
         // In a dynamically provisioned zone the failing of the first child may require a new host to be provisioned,
         // so failActive() may take a long time to complete, but the remaining children should be fast.
         activeChildrenToFail.forEach(this::failActive);
 
-        return false;
     }
 
-    private void wantToFail(Node node, boolean wantToFail, Mutex lock) {
-        if (!node.status().wantToFail())
+    private void redeploy(Deployment deployment, FailingNode failing) {
+        try {
+            deployment.activate();
+        } catch (TransientException | UncheckedTimeoutException e) {
+            log.log(Level.INFO, "Failed to redeploy " + failing.node().allocation().get().owner() +
+                                " with a transient error, will be retried by application maintainer: " +
+                                Exceptions.toMessageString(e));
+        } catch (RuntimeException e) {
+            // Reset want to fail: We'll retry failing unless it heals in the meantime
+            Optional<NodeMutex> optionalNodeMutex = nodeRepository().nodes().lockAndGet(failing.node());
+            if (optionalNodeMutex.isEmpty()) return;
+            try (var nodeMutex = optionalNodeMutex.get()) {
+                markWantToFail(nodeMutex.node(), false, nodeMutex);
+                log.log(Level.WARNING, "Could not fail " + failing.node() + " for " + failing.node().allocation().get().owner() +
+                                       " for " + failing.reason() + ": " + Exceptions.toMessageString(e));
+            }
+        }
+    }
+
+    private void markWantToFail(Node node, boolean wantToFail, Mutex lock) {
+        if (node.status().wantToFail() != wantToFail) {
             nodeRepository().nodes().write(node.withWantToFail(wantToFail, Agent.NodeFailer, clock().instant()), lock);
+        }
     }
 
     /** Returns true if node failing should be throttled */
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainer.java
index 036c46479d1..10a828c887a 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainer.java
@@ -47,13 +47,12 @@ public class PeriodicApplicationMaintainer extends ApplicationMaintainer {
                 .orElse(false);
     }
 
-    // Returns the applications that need to be redeployed by this config server at this point in time.
     @Override
     protected Map<ApplicationId, String> applicationsNeedingMaintenance() {
         if (deployer().bootstrapping()) return Map.of();
 
         // Collect all deployment times before sorting as deployments may happen while we build the set, breaking
-        // the comparable contract. Stale times are fine as the time is rechecked in ApplicationMaintainer#deployWithLock
+        // the comparable contract. Stale times are fine as the time is rechecked in ApplicationMaintainer#deployNow
         Map<ApplicationId, Instant> deploymentTimes = nodesNeedingMaintenance().stream()
                                                                                .map(node -> node.allocation().get().owner())
                                                                                .distinct()
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirer.java
index 87af8c05b14..1ae9b00d794 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirer.java
@@ -14,9 +14,10 @@ import com.yahoo.vespa.orchestrator.OrchestrationException;
 import com.yahoo.yolean.Exceptions;
 
 import java.time.Duration;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Map;
 import java.util.Optional;
-import java.util.OptionalLong;
 
 /**
  * Maintenance job which deactivates retired nodes, if given permission by orchestrator, or
@@ -47,40 +48,55 @@ public class RetiredExpirer extends NodeRepositoryMaintainer {
     protected double maintain() {
         int attempts = 0;
         int successes = 0;
+        List<ApplicationId> applicationsWithRetiredNodes = nodeRepository().nodes().list(Node.State.active)
+                                                                           .retired()
+                                                                           .stream()
+                                                                           .map(node -> node.allocation().get().owner())
+                                                                           .distinct()
+                                                                           .toList();
+        for (var application : applicationsWithRetiredNodes) {
+            attempts++;
+            if (removeRetiredNodes(application)) {
+                successes++;
+            }
+        }
+        return attempts == 0 ? 1.0 : ((double)successes / attempts);
+    }
 
-        NodeList activeNodes = nodeRepository().nodes().list(Node.State.active);
-        Map<ApplicationId, NodeList> retiredNodesByApplication = activeNodes.retired().groupingBy(node -> node.allocation().get().owner());
-        for (Map.Entry<ApplicationId, NodeList> entry : retiredNodesByApplication.entrySet()) {
-            ApplicationId application = entry.getKey();
-            NodeList retiredNodes = entry.getValue();
-            Map<Removal, NodeList> nodesByRemovalReason = retiredNodes.groupingBy(node -> removalOf(node, activeNodes));
-            if (nodesByRemovalReason.isEmpty()) continue;
-
-            for (var kv : nodesByRemovalReason.entrySet()) {
-                Removal removal = kv.getKey();
-                if (removal.equals(Removal.none())) continue;
-
-                NodeList nodes = kv.getValue();
-                attempts++;
-                try (MaintenanceDeployment deployment = new MaintenanceDeployment(application, deployer, metric, nodeRepository())) {
-                    if (!deployment.isValid()) {
-                        log.info("Skipping invalid deployment for " + application);
-                        continue;
-                    }
-
-                    nodeRepository().nodes().setRemovable(application, nodes.asList(), removal.isReusable());
-                    Optional<Long> session = deployment.activate();
-                    String nodeList = String.join(", ", nodes.mapToList(Node::hostname));
-                    if (session.isEmpty()) {
-                        log.info("Failed to redeploy " + application);
-                        continue;
-                    }
-                    log.info("Redeployed " + application + " at session " + session.get() + " to deactivate retired nodes: " + nodeList);
-                    successes++;
+    /** Mark retired nodes as removable and redeploy application */
+    private boolean removeRetiredNodes(ApplicationId application) {
+        try (MaintenanceDeployment deployment = new MaintenanceDeployment(application, deployer, metric, nodeRepository())) {
+            if (!deployment.isValid()) {
+                log.info("Skipping invalid deployment for " + application);
+                return false;
+            }
+            boolean redeploy = false;
+            List<String> nodesToDeactivate = new ArrayList<>();
+            try (var lock = nodeRepository().applications().lock(application)) {
+                NodeList activeNodes = nodeRepository().nodes().list(Node.State.active);
+                Map<Removal, NodeList> nodesByRemovalReason = activeNodes.owner(application)
+                                                                         .retired()
+                                                                         .groupingBy(node -> removalOf(node, activeNodes));
+                for (var kv : nodesByRemovalReason.entrySet()) {
+                    Removal reason = kv.getKey();
+                    if (reason.equals(Removal.none())) continue;
+                    redeploy = true;
+                    nodesToDeactivate.addAll(kv.getValue().hostnames());
+                    nodeRepository().nodes().setRemovable(kv.getValue(), reason.isReusable());
                 }
             }
+            if (!redeploy) {
+                return true;
+            }
+            Optional<Long> session = deployment.activate();
+            String nodeList = String.join(", ", nodesToDeactivate);
+            if (session.isEmpty()) {
+                log.info("Failed to redeploy " + application);
+                return false;
+            }
+            log.info("Redeployed " + application + " at session " + session.get() + " to deactivate retired nodes: " + nodeList);
+            return true;
         }
-        return attempts == 0 ? 1.0 : ((double)successes / attempts);
     }
 
     /**
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainer.java
index 5ce88346178..dcdcbf09175 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainer.java
@@ -8,6 +8,7 @@ import com.yahoo.config.provision.NodeResources;
 import com.yahoo.jdisc.Metric;
 import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.NodeList;
+import com.yahoo.vespa.hosted.provision.NodeMutex;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
 import com.yahoo.vespa.hosted.provision.maintenance.MaintenanceDeployment.Move;
 import com.yahoo.vespa.hosted.provision.node.Agent;
@@ -49,9 +50,7 @@ public class SpareCapacityMaintainer extends NodeRepositoryMaintainer {
                                    NodeRepository nodeRepository,
                                    Metric metric,
                                    Duration interval) {
-        this(deployer, nodeRepository, metric, interval,
-             10_000 // Should take less than a few minutes
-            );
+        this(deployer, nodeRepository, metric, interval, 10_000 /* Should take less than a few minutes */);
     }
 
     public SpareCapacityMaintainer(Deployer deployer,
@@ -160,22 +159,32 @@ public class SpareCapacityMaintainer extends NodeRepositoryMaintainer {
                 .filter(node -> node.state() == Node.State.active)
                 .min(this::retireOvercomittedComparator);
         if (nodeToRetire.isEmpty()) return;
+        retire(nodeToRetire.get());
+    }
 
-        ApplicationId application = nodeToRetire.get().allocation().get().owner();
-        try (MaintenanceDeployment deployment = new MaintenanceDeployment(application, deployer, metric, nodeRepository())) {
-            if ( ! deployment.isValid()) return;
-
-            Optional<Node> nodeWithWantToRetire = nodeRepository().nodes().node(nodeToRetire.get().hostname())
-                    .map(node -> node.withWantToRetire(true, Agent.SpareCapacityMaintainer, nodeRepository().clock().instant()));
-            if (nodeWithWantToRetire.isEmpty()) return;
-
-            nodeRepository().nodes().write(nodeWithWantToRetire.get(), deployment.applicationLock().get());
-            log.log(Level.INFO, String.format("Redeploying %s to move %s from overcommitted host",
-                                              application, nodeToRetire.get().hostname()));
+    /** Mark node for retirement and redeploy its application */
+    private void retire(Node node) {
+        ApplicationId owner = node.allocation().get().owner();
+        try (MaintenanceDeployment deployment = new MaintenanceDeployment(owner, deployer, metric, nodeRepository())) {
+            if (!deployment.isValid()) return;
+            if (!markWantToRetire(node.hostname())) return;
+            log.log(Level.INFO, String.format("Redeploying %s to move %s from over-committed host",
+                                              owner, node.hostname()));
             deployment.activate();
         }
     }
 
+    private boolean markWantToRetire(String hostname) {
+        Optional<NodeMutex> optionalNodeMutex = nodeRepository().nodes().lockAndGet(hostname);
+        if (optionalNodeMutex.isEmpty()) return false;
+        try (var nodeMutex = optionalNodeMutex.get()) {
+            Node retiredNode = nodeMutex.node().withWantToRetire(true, Agent.SpareCapacityMaintainer,
+                                                                 nodeRepository().clock().instant());
+            nodeRepository().nodes().write(retiredNode, nodeMutex);
+            return true;
+        }
+    }
+
     private static class CapacitySolver {
 
         private final HostCapacity hostCapacity;
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java
index bb3d288e555..10f0c8aa554 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java
@@ -203,17 +203,12 @@ public class Nodes {
     /**
      * Sets a list of nodes to have their allocation removable (active to inactive) in the node repository.
      *
-     * @param application the application the nodes belong to
      * @param nodes the nodes to make removable. These nodes MUST be in the active state
      * @param reusable move the node directly to {@link Node.State#dirty} after removal
      */
-    public void setRemovable(ApplicationId application, List<Node> nodes, boolean reusable) {
-        try (Mutex lock = applications.lock(application)) {
-            List<Node> removableNodes = nodes.stream()
-                                             .map(node -> node.with(node.allocation().get().removable(true, reusable)))
-                                             .toList();
-            write(removableNodes, lock);
-        }
+    public void setRemovable(NodeList nodes, boolean reusable) {
+        performOn(nodes, (node, mutex) -> write(node.with(node.allocation().get().removable(true, reusable)),
+                                                mutex));
     }
 
     /**
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDb.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDb.java
index c1ab8489f40..cec413cf4e3 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDb.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDb.java
@@ -391,6 +391,11 @@ public class CuratorDb {
         return db.lock(lockPath.append("archiveUris"), defaultLockTimeout);
     }
 
+    public Lock lockMaintenance(ApplicationId application) {
+        return db.lock(lockPath.append("maintenanceDeployment").append(application.serializedForm()),
+                       Duration.ofSeconds(3));
+    }
+
     // Load balancers -----------------------------------------------------------
 
     public List<LoadBalancerId> readLoadBalancerIds() {
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainerTest.java
index 880a69b61e5..e075995c89e 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainerTest.java
@@ -402,8 +402,8 @@ public class HostCapacityMaintainerTest {
         // Config server becomes removable (done by RetiredExpirer in a real system) and redeployment moves it
         // to parked
         int removedIndex = nodeToRemove.get().allocation().get().membership().index();
-        tester.nodeRepository().nodes().setRemovable(configSrvApp, List.of(nodeToRemove.get()), true);
-        tester.nodeRepository().nodes().setRemovable(hostApp, List.of(hostToRemove.get()), true);
+        tester.nodeRepository().nodes().setRemovable(NodeList.of(nodeToRemove.get()), true);
+        tester.nodeRepository().nodes().setRemovable(NodeList.of(hostToRemove.get()), true);
         tester.prepareAndActivateInfraApplication(configSrvApp, hostType.childNodeType());
         tester.prepareAndActivateInfraApplication(hostApp, hostType);
         tester.nodeRepository().nodes().markNodeAvailableForNewAllocation(nodeToRemove.get().hostname(), Agent.operator, "Readied by host-admin");
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java
index b5735cfae84..98c17eb4d5e 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java
@@ -515,9 +515,9 @@ public class OsVersionsTest {
     private void replaceNodes(ApplicationId application) {
         // Deploy to retire nodes
         deployApplication(application);
-        List<Node> retired = tester.nodeRepository().nodes().list().owner(application).retired().asList();
+        NodeList retired = tester.nodeRepository().nodes().list().owner(application).retired();
         assertFalse("At least one node is retired", retired.isEmpty());
-        tester.nodeRepository().nodes().setRemovable(application, retired, false);
+        tester.nodeRepository().nodes().setRemovable(retired, false);
 
         // Redeploy to deactivate removable nodes and allocate new ones
         deployApplication(application);
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImplTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImplTest.java
index 9a38cbbba44..9cd5adef5f4 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImplTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImplTest.java
@@ -10,8 +10,8 @@ import com.yahoo.config.provision.HostSpec;
 import com.yahoo.config.provision.NodeType;
 import com.yahoo.config.provision.Provisioner;
 import com.yahoo.config.provision.Zone;
-import com.yahoo.vespa.flags.InMemoryFlagSource;
 import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.NodeList;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
 import com.yahoo.vespa.hosted.provision.NodeRepositoryTester;
 import com.yahoo.vespa.hosted.provision.maintenance.InfrastructureVersions;
@@ -122,7 +122,7 @@ public class InfraDeployerImplTest {
         addNode(5, Node.State.dirty, Optional.empty());
         addNode(6, Node.State.ready, Optional.empty());
         Node node7 = addNode(7, Node.State.active, Optional.of(target));
-        nodeRepository.nodes().setRemovable(application.getApplicationId(), List.of(node7), false);
+        nodeRepository.nodes().setRemovable(NodeList.of(node7), false);
 
         infraDeployer.getDeployment(application.getApplicationId()).orElseThrow().activate();
 
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java
index c1c4630f431..fb773f19b8a 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java
@@ -636,7 +636,7 @@ public class VirtualNodeProvisioningTest {
         tester.activate(app1, cluster1, Capacity.from(new ClusterResources(2, 1, r)));
 
         // Deactivate any retired nodes - usually done by the RetiredExpirer
-        tester.nodeRepository().nodes().setRemovable(app1, tester.getNodes(app1).retired().asList(), false);
+        tester.nodeRepository().nodes().setRemovable(tester.getNodes(app1).retired(), false);
         tester.activate(app1, cluster1, Capacity.from(new ClusterResources(2, 1, r)));
 
         if (expectedReuse) {
diff --git a/searchlib/src/vespa/searchlib/attribute/enumattributesaver.cpp b/searchlib/src/vespa/searchlib/attribute/enumattributesaver.cpp
index 74a37ca8394..a2090185158 100644
--- a/searchlib/src/vespa/searchlib/attribute/enumattributesaver.cpp
+++ b/searchlib/src/vespa/searchlib/attribute/enumattributesaver.cpp
@@ -8,16 +8,13 @@
 
 namespace search {
 
-EnumAttributeSaver::
-EnumAttributeSaver(const IEnumStore &enumStore)
+EnumAttributeSaver::EnumAttributeSaver(IEnumStore &enumStore)
     : _enumStore(enumStore),
       _enumerator(enumStore.make_enumerator())
 {
 }
 
-EnumAttributeSaver::~EnumAttributeSaver()
-{
-}
+EnumAttributeSaver::~EnumAttributeSaver() = default;
 
 void
 EnumAttributeSaver::writeUdat(IAttributeSaveTarget &saveTarget)
diff --git a/searchlib/src/vespa/searchlib/attribute/enumattributesaver.h b/searchlib/src/vespa/searchlib/attribute/enumattributesaver.h
index d6dceb4772a..47af8c0452e 100644
--- a/searchlib/src/vespa/searchlib/attribute/enumattributesaver.h
+++ b/searchlib/src/vespa/searchlib/attribute/enumattributesaver.h
@@ -24,7 +24,7 @@ private:
     std::unique_ptr<Enumerator> _enumerator;
 
 public:
-    EnumAttributeSaver(const IEnumStore &enumStore);
+    EnumAttributeSaver(IEnumStore &enumStore);
     ~EnumAttributeSaver();
 
     void writeUdat(IAttributeSaveTarget &saveTarget);
diff --git a/searchlib/src/vespa/searchlib/attribute/enumstore.h b/searchlib/src/vespa/searchlib/attribute/enumstore.h
index df3a595ae34..59524f3788a 100644
--- a/searchlib/src/vespa/searchlib/attribute/enumstore.h
+++ b/searchlib/src/vespa/searchlib/attribute/enumstore.h
@@ -212,7 +212,7 @@ public:
     void inc_compaction_count() override {
         _store.get_allocator().get_data_store().inc_compaction_count();
     }
-    std::unique_ptr<Enumerator> make_enumerator() const override;
+    std::unique_ptr<Enumerator> make_enumerator() override;
     std::unique_ptr<EntryComparator> allocate_comparator() const override;
 
     // Methods below are only relevant for strings, and are templated to only be instantiated on demand.
diff --git a/searchlib/src/vespa/searchlib/attribute/enumstore.hpp b/searchlib/src/vespa/searchlib/attribute/enumstore.hpp
index b863e56fb4a..bc767a296eb 100644
--- a/searchlib/src/vespa/searchlib/attribute/enumstore.hpp
+++ b/searchlib/src/vespa/searchlib/attribute/enumstore.hpp
@@ -259,7 +259,7 @@ EnumStoreT<EntryT>::consider_compact_dictionary(const CompactionStrategy& compac
 
 template <typename EntryT>
 std::unique_ptr<IEnumStore::Enumerator>
-EnumStoreT<EntryT>::make_enumerator() const
+EnumStoreT<EntryT>::make_enumerator()
 {
     return std::make_unique<Enumerator>(*_dict, _store.get_data_store(), false);
 }
diff --git a/searchlib/src/vespa/searchlib/attribute/i_enum_store.h b/searchlib/src/vespa/searchlib/attribute/i_enum_store.h
index 57886511221..2157db3e5ed 100644
--- a/searchlib/src/vespa/searchlib/attribute/i_enum_store.h
+++ b/searchlib/src/vespa/searchlib/attribute/i_enum_store.h
@@ -72,7 +72,7 @@ public:
     enumstore::EnumeratedLoader make_enumerated_loader();
     enumstore::EnumeratedPostingsLoader make_enumerated_postings_loader();
 
-    virtual std::unique_ptr<Enumerator> make_enumerator() const = 0;
+    virtual std::unique_ptr<Enumerator> make_enumerator() = 0;
     virtual std::unique_ptr<vespalib::datastore::EntryComparator> allocate_comparator() const = 0;
 };
 
diff --git a/searchlib/src/vespa/searchlib/attribute/multienumattributesaver.cpp b/searchlib/src/vespa/searchlib/attribute/multienumattributesaver.cpp
index 083f0409821..87326f3628f 100644
--- a/searchlib/src/vespa/searchlib/attribute/multienumattributesaver.cpp
+++ b/searchlib/src/vespa/searchlib/attribute/multienumattributesaver.cpp
@@ -80,7 +80,7 @@ MultiValueEnumAttributeSaver<MultiValueT>::
 MultiValueEnumAttributeSaver(GenerationHandler::Guard &&guard,
                              const attribute::AttributeHeader &header,
                              const MultiValueMapping &mvMapping,
-                             const IEnumStore &enumStore)
+                             IEnumStore &enumStore)
     : Parent(std::move(guard), header, mvMapping),
       _mvMapping(mvMapping),
       _enumSaver(enumStore),
diff --git a/searchlib/src/vespa/searchlib/attribute/multienumattributesaver.h b/searchlib/src/vespa/searchlib/attribute/multienumattributesaver.h
index 44c45567733..7c127ac0781 100644
--- a/searchlib/src/vespa/searchlib/attribute/multienumattributesaver.h
+++ b/searchlib/src/vespa/searchlib/attribute/multienumattributesaver.h
@@ -35,7 +35,7 @@ public:
     MultiValueEnumAttributeSaver(GenerationHandler::Guard &&guard,
                                  const attribute::AttributeHeader &header,
                                  const MultiValueMapping &mvMapping,
-                                 const IEnumStore &enumStore);
+                                 IEnumStore &enumStore);
     ~MultiValueEnumAttributeSaver() override;
 };
 
diff --git a/searchlib/src/vespa/searchlib/attribute/reference_attribute_saver.cpp b/searchlib/src/vespa/searchlib/attribute/reference_attribute_saver.cpp
index 0c3180875a3..a8f13df0a85 100644
--- a/searchlib/src/vespa/searchlib/attribute/reference_attribute_saver.cpp
+++ b/searchlib/src/vespa/searchlib/attribute/reference_attribute_saver.cpp
@@ -16,7 +16,7 @@ namespace search::attribute {
 ReferenceAttributeSaver::ReferenceAttributeSaver(GenerationHandler::Guard &&guard,
                                                  const AttributeHeader &header,
                                                  EntryRefVector&& indices,
-                                                 const Store &store)
+                                                 Store &store)
     : AttributeSaver(std::move(guard), header),
       _indices(std::move(indices)),
       _store(store),
@@ -25,9 +25,7 @@ ReferenceAttributeSaver::ReferenceAttributeSaver(GenerationHandler::Guard &&guar
 }
 
 
-ReferenceAttributeSaver::~ReferenceAttributeSaver()
-{
-}
+ReferenceAttributeSaver::~ReferenceAttributeSaver() = default;
 
 namespace {
 
diff --git a/searchlib/src/vespa/searchlib/attribute/reference_attribute_saver.h b/searchlib/src/vespa/searchlib/attribute/reference_attribute_saver.h
index c413d01c386..fa3fafc3254 100644
--- a/searchlib/src/vespa/searchlib/attribute/reference_attribute_saver.h
+++ b/searchlib/src/vespa/searchlib/attribute/reference_attribute_saver.h
@@ -41,9 +41,9 @@ public:
     ReferenceAttributeSaver(vespalib::GenerationHandler::Guard &&guard,
                             const AttributeHeader &header,
                             EntryRefVector&& indices,
-                            const Store &store);
+                            Store &store);
 
-    virtual ~ReferenceAttributeSaver();
+    ~ReferenceAttributeSaver() override;
 };
 
 }
diff --git a/searchlib/src/vespa/searchlib/attribute/singleenumattributesaver.cpp b/searchlib/src/vespa/searchlib/attribute/singleenumattributesaver.cpp
index 1857e942136..1f3f3e104d1 100644
--- a/searchlib/src/vespa/searchlib/attribute/singleenumattributesaver.cpp
+++ b/searchlib/src/vespa/searchlib/attribute/singleenumattributesaver.cpp
@@ -4,7 +4,6 @@
 #include "iattributesavetarget.h"
 #include <vespa/searchlib/util/bufferwriter.h>
 
-
 using search::attribute::EntryRefVector;
 using vespalib::GenerationHandler;
 
@@ -14,25 +13,20 @@ SingleValueEnumAttributeSaver::
 SingleValueEnumAttributeSaver(GenerationHandler::Guard &&guard,
                               const attribute::AttributeHeader &header,
                               EntryRefVector &&indices,
-                              const IEnumStore &enumStore)
+                              IEnumStore &enumStore)
     : AttributeSaver(std::move(guard), header),
       _indices(std::move(indices)),
       _enumSaver(enumStore)
 {
 }
 
-
-SingleValueEnumAttributeSaver::~SingleValueEnumAttributeSaver()
-{
-}
-
+SingleValueEnumAttributeSaver::~SingleValueEnumAttributeSaver() = default;
 
 bool
 SingleValueEnumAttributeSaver::onSave(IAttributeSaveTarget &saveTarget)
 {
     _enumSaver.writeUdat(saveTarget);
-    std::unique_ptr<search::BufferWriter> datWriter(saveTarget.datWriter().
-                                                    allocBufferWriter());
+    std::unique_ptr<search::BufferWriter> datWriter(saveTarget.datWriter().allocBufferWriter());
     assert(saveTarget.getEnumerated());
     auto &enumerator = _enumSaver.get_enumerator();
     enumerator.enumerateValues();
@@ -49,5 +43,4 @@ SingleValueEnumAttributeSaver::onSave(IAttributeSaveTarget &saveTarget)
     return true;
 }
 
-
 }  // namespace search
diff --git a/searchlib/src/vespa/searchlib/attribute/singleenumattributesaver.h b/searchlib/src/vespa/searchlib/attribute/singleenumattributesaver.h
index af83d36cbbb..7f1c62c7720 100644
--- a/searchlib/src/vespa/searchlib/attribute/singleenumattributesaver.h
+++ b/searchlib/src/vespa/searchlib/attribute/singleenumattributesaver.h
@@ -22,7 +22,7 @@ public:
     SingleValueEnumAttributeSaver(vespalib::GenerationHandler::Guard &&guard,
                                   const attribute::AttributeHeader &header,
                                   attribute::EntryRefVector &&indices,
-                                  const IEnumStore &enumStore);
+                                  IEnumStore &enumStore);
 
     ~SingleValueEnumAttributeSaver() override;
 };
diff --git a/searchlib/src/vespa/searchlib/bitcompression/compression.h b/searchlib/src/vespa/searchlib/bitcompression/compression.h
index 7c5ba3e94ca..a77d82d9e8f 100644
--- a/searchlib/src/vespa/searchlib/bitcompression/compression.h
+++ b/searchlib/src/vespa/searchlib/bitcompression/compression.h
@@ -1248,9 +1248,7 @@ public:
     void setReadContext(search::ComprFileReadContext *readContext) {
         _readContext = readContext;
     }
-    search::ComprFileReadContext *getReadContext() const {
-        return _readContext;
-    }
+
     void readComprBuffer() {
         _readContext->readComprBuffer();
     }
diff --git a/searchlib/src/vespa/searchlib/memoryindex/feature_store.h b/searchlib/src/vespa/searchlib/memoryindex/feature_store.h
index 5f5e782a382..53588fa2894 100644
--- a/searchlib/src/vespa/searchlib/memoryindex/feature_store.h
+++ b/searchlib/src/vespa/searchlib/memoryindex/feature_store.h
@@ -154,12 +154,8 @@ public:
     void setupForReadFeatures(vespalib::datastore::EntryRef ref, DecodeContextCooked &decoder) const {
         const uint8_t * bits = getBits(ref);
         decoder.setByteCompr(bits);
-        uint32_t bufferId = RefType(ref).bufferId();
-        const vespalib::datastore::BufferState &state = _store.getBufferState(bufferId);
-        decoder.setEnd(
-                ((_store.getEntryArray<uint8_t>(RefType(0, bufferId), buffer_array_size) + state.size() -
-                  bits) + 7) / 8,
-                false);
+        constexpr uint32_t maxOffset = RefType::offsetSize() * buffer_array_size;
+        decoder.setEnd(maxOffset, false);
     }
 
     /**
diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java
index 373928a3e22..fe1e2b46830 100644
--- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java
+++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java
@@ -171,6 +171,8 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler {
     private static final String SLICES = "slices";
     private static final String SLICE_ID = "sliceId";
     private static final String DRY_RUN = "dryRun";
+    private static final String FROM_TIMESTAMP = "fromTimestamp";
+    private static final String TO_TIMESTAMP = "toTimestamp";
 
     private final Clock clock;
     private final Duration handlerTimeout;
@@ -1227,6 +1229,12 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler {
         getProperty(request, CONTINUATION).map(ProgressToken::fromSerializedString).ifPresent(parameters::setResumeToken);
         parameters.setPriority(DocumentProtocol.Priority.NORMAL_4);
 
+        getProperty(request, FROM_TIMESTAMP, unsignedLongParser).ifPresent(parameters::setFromTimestamp);
+        getProperty(request, TO_TIMESTAMP, unsignedLongParser).ifPresent(parameters::setToTimestamp);
+        if (Long.compareUnsigned(parameters.getFromTimestamp(), parameters.getToTimestamp()) > 0) {
+            throw new IllegalArgumentException("toTimestamp must be greater than, or equal to, fromTimestamp");
+        }
+
         StorageCluster storageCluster = resolveCluster(cluster, clusters);
         parameters.setRoute(storageCluster.name());
         parameters.setBucketSpace(resolveBucket(storageCluster,
diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/DocumentV1ApiTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/DocumentV1ApiTest.java
index b6ad7ba5570..851a0949266 100644
--- a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/DocumentV1ApiTest.java
+++ b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/DocumentV1ApiTest.java
@@ -223,6 +223,8 @@ public class DocumentV1ApiTest {
             assertEquals("(all the things)", parameters.getDocumentSelection());
             assertEquals(6000, parameters.getSessionTimeoutMs());
             assertEquals(9, parameters.getTraceLevel());
+            assertEquals(1_000_000, parameters.getFromTimestamp());
+            assertEquals(2_000_000, parameters.getToTimestamp());
             // Put some documents in the response
             parameters.getLocalDataHandler().onMessage(new PutDocumentMessage(new DocumentPut(doc1)), tokens.get(0));
             parameters.getLocalDataHandler().onMessage(new PutDocumentMessage(new DocumentPut(doc2)), tokens.get(1));
@@ -234,7 +236,7 @@ public class DocumentV1ApiTest {
             parameters.getControlHandler().onDone(VisitorControlHandler.CompletionCode.TIMEOUT, "timeout is OK");
         });
         response = driver.sendRequest("http://localhost/document/v1?cluster=content&bucketSpace=default&wantedDocumentCount=1025&concurrency=123" +
-                                      "&selection=all%20the%20things&fieldSet=[id]&timeout=6&tracelevel=9");
+                                      "&selection=all%20the%20things&fieldSet=[id]&timeout=6&tracelevel=9&fromTimestamp=1000000&toTimestamp=2000000");
         assertSameJson("""
                        {
                          "pathId": "/document/v1",
@@ -284,6 +286,8 @@ public class DocumentV1ApiTest {
             assertEquals(6000, parameters.getTimeoutMs());
             assertEquals(4, parameters.getSlices());
             assertEquals(1, parameters.getSliceId());
+            assertEquals(0, parameters.getFromTimestamp()); // not set; 0 is default
+            assertEquals(0, parameters.getToTimestamp()); // not set; 0 is default
             // Put some documents in the response
             parameters.getLocalDataHandler().onMessage(new PutDocumentMessage(new DocumentPut(doc1)), tokens.get(0));
             parameters.getLocalDataHandler().onMessage(new PutDocumentMessage(new DocumentPut(doc2)), tokens.get(1));
@@ -500,6 +504,15 @@ public class DocumentV1ApiTest {
                        "}", response.readAll());
         assertEquals(200, response.getStatus());
 
+        // GET with from timestamp > to timestamp is an error
+        access.expect(parameters -> { fail("unreachable"); });
+        response = driver.sendRequest("http://localhost/document/v1/?cluster=content&fromTimestamp=100&toTimestamp=99");
+        assertSameJson("{" +
+                "  \"pathId\": \"/document/v1/\"," +
+                "  \"message\": \"toTimestamp must be greater than, or equal to, fromTimestamp\"" +
+                "}", response.readAll());
+        assertEquals(400, response.getStatus());
+
         // GET with full document ID is a document get operation which returns 404 when no document is found
         access.session.expect((id, parameters) -> {
             assertEquals(doc1.getId(), id);
diff --git a/vespalib/src/tests/datastore/array_store/array_store_test.cpp b/vespalib/src/tests/datastore/array_store/array_store_test.cpp
index 3ebe8fdba1a..97e1ddb985d 100644
--- a/vespalib/src/tests/datastore/array_store/array_store_test.cpp
+++ b/vespalib/src/tests/datastore/array_store/array_store_test.cpp
@@ -97,13 +97,13 @@ struct ArrayStoreTest : public TestT
     uint32_t getBufferId(EntryRef ref) const {
         return EntryRefType(ref).bufferId();
     }
-    void assertBufferState(EntryRef ref, const MemStats& expStats) const {
+    void assertBufferState(EntryRef ref, const MemStats& expStats) {
         EXPECT_EQ(expStats._used, store.bufferState(ref).size());
         EXPECT_EQ(expStats._hold, store.bufferState(ref).stats().hold_elems());
         EXPECT_EQ(expStats._dead, store.bufferState(ref).stats().dead_elems());
     }
-    void assert_buffer_stats(EntryRef ref, const TestBufferStats& exp_stats) const {
-        auto& state = store.bufferState(ref);
+    void assert_buffer_stats(EntryRef ref, const TestBufferStats& exp_stats) {
+        const auto& state = store.bufferState(ref);
         EXPECT_EQ(exp_stats._used, state.size());
         EXPECT_EQ(exp_stats._hold, state.stats().hold_elems());
         EXPECT_EQ(exp_stats._dead, state.stats().dead_elems());
diff --git a/vespalib/src/tests/datastore/unique_store_string_allocator/unique_store_string_allocator_test.cpp b/vespalib/src/tests/datastore/unique_store_string_allocator/unique_store_string_allocator_test.cpp
index 64e0c1599bd..7d4451556c8 100644
--- a/vespalib/src/tests/datastore/unique_store_string_allocator/unique_store_string_allocator_test.cpp
+++ b/vespalib/src/tests/datastore/unique_store_string_allocator/unique_store_string_allocator_test.cpp
@@ -56,11 +56,11 @@ struct TestBase : public ::testing::Test {
     uint32_t get_buffer_id(EntryRef ref) const {
         return EntryRefType(ref).bufferId();
     }
-    const BufferState &buffer_state(EntryRef ref) const {
+    BufferState &buffer_state(EntryRef ref) {
         return allocator.get_data_store().getBufferState(get_buffer_id(ref));
     }
-    void assert_buffer_state(EntryRef ref, const TestBufferStats expStats) const {
-        const auto & stats = buffer_state(ref).stats();
+    void assert_buffer_state(EntryRef ref, const TestBufferStats expStats) {
+        auto & stats = buffer_state(ref).stats();
         EXPECT_EQ(expStats._used, buffer_state(ref).size());
         EXPECT_EQ(expStats._hold, stats.hold_elems());
         EXPECT_EQ(expStats._dead, stats.dead_elems());
diff --git a/vespalib/src/vespa/vespalib/btree/btreenodeallocator.h b/vespalib/src/vespa/vespalib/btree/btreenodeallocator.h
index 3fa5f1188cd..784e95e3817 100644
--- a/vespalib/src/vespa/vespalib/btree/btreenodeallocator.h
+++ b/vespalib/src/vespa/vespalib/btree/btreenodeallocator.h
@@ -163,7 +163,7 @@ public:
     vespalib::string toString(BTreeNode::Ref ref) const;
     vespalib::string toString(const BTreeNode * node) const;
 
-    bool getCompacting(EntryRef ref) const { return _nodeStore.getCompacting(ref); }
+    bool getCompacting(EntryRef ref) { return _nodeStore.getCompacting(ref); }
 
     std::unique_ptr<vespalib::datastore::CompactingBuffers> start_compact_worst(const CompactionStrategy& compaction_strategy) { return _nodeStore.start_compact_worst(compaction_strategy); }
 
diff --git a/vespalib/src/vespa/vespalib/btree/btreenodestore.h b/vespalib/src/vespa/vespalib/btree/btreenodestore.h
index 8fef0185674..73e68c3579f 100644
--- a/vespalib/src/vespa/vespalib/btree/btreenodestore.h
+++ b/vespalib/src/vespa/vespalib/btree/btreenodestore.h
@@ -182,7 +182,7 @@ public:
     }
 
     // Inherit doc from DataStoreT
-    bool getCompacting(EntryRef ref) const {
+    bool getCompacting(EntryRef ref) {
         return _store.getCompacting(ref);
     }
 
diff --git a/vespalib/src/vespa/vespalib/btree/btreestore.h b/vespalib/src/vespa/vespalib/btree/btreestore.h
index c228c084e6d..9d98a9ca514 100644
--- a/vespalib/src/vespa/vespalib/btree/btreestore.h
+++ b/vespalib/src/vespa/vespalib/btree/btreestore.h
@@ -95,202 +95,90 @@ protected:
 
 public:
     BTreeStore();
-
     BTreeStore(bool init);
-
     ~BTreeStore();
 
     const NodeAllocatorType &getAllocator() const { return _allocator; }
 
-    void
-    disableFreeLists() {
+    void disableFreeLists() {
         _store.disableFreeLists();
         _allocator.disableFreeLists();
     }
 
-    void
-    disableElemHoldList()
-    {
+    void disableElemHoldList() {
         _store.disableElemHoldList();
         _allocator.disableElemHoldList();
     }
 
-    BTreeTypeRefPair
-    allocNewBTree() {
+    BTreeTypeRefPair allocNewBTree() {
         return _store.allocator<BTreeType>(BUFFERTYPE_BTREE).alloc();
     }
 
-    BTreeTypeRefPair
-    allocBTree() {
+    BTreeTypeRefPair allocBTree() {
         return _store.freeListAllocator<BTreeType, TreeReclaimer>(BUFFERTYPE_BTREE).alloc();
     }
 
-    BTreeTypeRefPair
-    allocNewBTreeCopy(const BTreeType &rhs) {
+    BTreeTypeRefPair allocNewBTreeCopy(const BTreeType &rhs) {
         return _store.allocator<BTreeType>(BUFFERTYPE_BTREE).alloc(rhs);
     }
 
-    BTreeTypeRefPair
-    allocBTreeCopy(const BTreeType &rhs) {
+    BTreeTypeRefPair allocBTreeCopy(const BTreeType &rhs) {
         return _store.freeListAllocator<BTreeType, datastore::DefaultReclaimer<BTreeType> >(BUFFERTYPE_BTREE).alloc(rhs);
     }
 
-    KeyDataTypeRefPair
-    allocNewKeyData(uint32_t clusterSize);
-
-    KeyDataTypeRefPair
-    allocKeyData(uint32_t clusterSize);
-
-    KeyDataTypeRefPair
-    allocNewKeyDataCopy(const KeyDataType *rhs, uint32_t clusterSize);
-
-    KeyDataTypeRefPair
-    allocKeyDataCopy(const KeyDataType *rhs, uint32_t clusterSize);
-
-    const KeyDataType *
-    lower_bound(const KeyDataType *b, const KeyDataType *e,
-                const KeyType &key, CompareT comp);
-
-    void
-    makeTree(EntryRef &ref,
-             const KeyDataType *array, uint32_t clusterSize);
-
-    void
-    makeArray(EntryRef &ref, EntryRef leafRef, LeafNodeType *leafNode);
-
-    bool
-    insert(EntryRef &ref,
-           const KeyType &key, const DataType &data,
-           CompareT comp = CompareT());
-
-    bool
-    remove(EntryRef &ref,
-           const KeyType &key,
-           CompareT comp = CompareT());
-
-    uint32_t
-    getNewClusterSize(const KeyDataType *o,
-                      const KeyDataType *oe,
-                      AddIter a,
-                      AddIter ae,
-                      RemoveIter r,
-                      RemoveIter re,
-                      CompareT comp);
-
-    void
-    applyCluster(const KeyDataType *o,
-                 const KeyDataType *oe,
-                 KeyDataType *d,
-                 const KeyDataType *de,
-                 AddIter a,
-                 AddIter ae,
-                 RemoveIter r,
-                 RemoveIter re,
-                 CompareT comp);
-
-
-    void
-    applyModifyTree(BTreeType *tree,
-                    AddIter a,
-                    AddIter ae,
-                    RemoveIter r,
-                    RemoveIter re,
-                    CompareT comp);
-
-    void
-    applyBuildTree(BTreeType *tree,
-                   AddIter a,
-                   AddIter ae,
-                   RemoveIter r,
-                   RemoveIter re,
-                   CompareT comp);
-
-    void
-    applyNewArray(EntryRef &ref,
-                  AddIter aOrg,
-                  AddIter ae);
-
-    void
-    applyNewTree(EntryRef &ref,
-                 AddIter a,
-                 AddIter ae,
-                 CompareT comp);
-
-    void
-    applyNew(EntryRef &ref,
-             AddIter a,
-             AddIter ae,
-             CompareT comp);
-
-
-    bool
-    applyCluster(EntryRef &ref,
-                 uint32_t clusterSize,
-                 AddIter a,
-                 AddIter ae,
-                 RemoveIter r,
-                 RemoveIter re,
-                 CompareT comp);
-
-    void
-    applyTree(BTreeType *tree,
-              AddIter a,
-              AddIter ae,
-              RemoveIter r,
-              RemoveIter re,
-              CompareT comp);
-
-    void
-    normalizeTree(EntryRef &ref,
-                  BTreeType *tree,
-                  bool wasArray);
+    KeyDataTypeRefPair allocNewKeyData(uint32_t clusterSize);
+    KeyDataTypeRefPair allocKeyData(uint32_t clusterSize);
+    KeyDataTypeRefPair allocNewKeyDataCopy(const KeyDataType *rhs, uint32_t clusterSize);
+    KeyDataTypeRefPair allocKeyDataCopy(const KeyDataType *rhs, uint32_t clusterSize);
+
+    const KeyDataType * lower_bound(const KeyDataType *b, const KeyDataType *e, const KeyType &key, CompareT comp);
+
+    void makeTree(EntryRef &ref, const KeyDataType *array, uint32_t clusterSize);
+    void makeArray(EntryRef &ref, EntryRef leafRef, LeafNodeType *leafNode);
+    bool insert(EntryRef &ref, const KeyType &key, const DataType &data, CompareT comp = CompareT());
+
+    bool remove(EntryRef &ref, const KeyType &key,CompareT comp = CompareT());
+
+    uint32_t getNewClusterSize(const KeyDataType *o, const KeyDataType *oe, AddIter a, AddIter ae,
+                               RemoveIter r, RemoveIter re, CompareT comp);
+
+    void applyCluster(const KeyDataType *o, const KeyDataType *oe, KeyDataType *d, const KeyDataType *de,
+                      AddIter a, AddIter ae, RemoveIter r, RemoveIter re, CompareT comp);
+
+    void applyModifyTree(BTreeType *tree, AddIter a, AddIter ae, RemoveIter r, RemoveIter re, CompareT comp);
+    void applyBuildTree(BTreeType *tree, AddIter a, AddIter ae, RemoveIter r, RemoveIter re, CompareT comp);
+    void applyNewArray(EntryRef &ref, AddIter aOrg, AddIter ae);
+    void applyNewTree(EntryRef &ref, AddIter a, AddIter ae, CompareT comp);
+    void applyNew(EntryRef &ref, AddIter a, AddIter ae, CompareT comp);
+
+    bool applyCluster(EntryRef &ref, uint32_t clusterSize, AddIter a, AddIter ae,
+                      RemoveIter r, RemoveIter re, CompareT comp);
+
+    void applyTree(BTreeType *tree, AddIter a, AddIter ae, RemoveIter r, RemoveIter re, CompareT comp);
+
+    void normalizeTree(EntryRef &ref, BTreeType *tree, bool wasArray);
     /**
      * Apply multiple changes at once.
      *
      * additions and removals should be sorted on key without duplicates.
      * Overlap between additions and removals indicates updates.
      */
-    void
-    apply(EntryRef &ref,
-          AddIter a,
-          AddIter ae,
-          RemoveIter r,
-          RemoveIter re,
-          CompareT comp = CompareT());
-
-    void
-    clear(const EntryRef ref);
-
-    size_t
-    size(const EntryRef ref) const;
+    void apply(EntryRef &ref, AddIter a, AddIter ae, RemoveIter r, RemoveIter re, CompareT comp = CompareT());
 
-    size_t
-    frozenSize(const EntryRef ref) const;
+    void clear(const EntryRef ref);
+    size_t size(const EntryRef ref) const;
+    size_t frozenSize(const EntryRef ref) const;
+    Iterator begin(const EntryRef ref) const;
+    ConstIterator beginFrozen(const EntryRef ref) const;
 
-    Iterator
-    begin(const EntryRef ref) const;
+    void beginFrozen(const EntryRef ref, std::vector<ConstIterator> &where) const;
 
-    ConstIterator
-    beginFrozen(const EntryRef ref) const;
-
-    void
-    beginFrozen(const EntryRef ref, std::vector<ConstIterator> &where) const;
-
-    uint32_t
-    getTypeId(RefType ref) const
-    {
-        return _store.getBufferState(ref.bufferId()).getTypeId();
+    uint32_t getTypeId(RefType ref) const {
+        return _store.getBufferMeta(ref.bufferId()).getTypeId();
     }
 
-    static bool
-    isSmallArray(uint32_t typeId)
-    {
-        return typeId < clusterLimit;
-    }
-
-    bool
-    isSmallArray(const EntryRef ref) const;
-
+    static bool isSmallArray(uint32_t typeId) { return typeId < clusterLimit; }
+    bool isSmallArray(const EntryRef ref) const;
     static bool isBTree(uint32_t typeId) { return typeId == BUFFERTYPE_BTREE; }
     bool isBTree(RefType ref) const { return isBTree(getTypeId(ref)); }
 
@@ -299,9 +187,7 @@ public:
      * Cluster size == 0 means we have a tree for the given reference.
      * The reference must be valid.
      **/
-    static uint32_t
-    getClusterSize(uint32_t typeId)
-    {
+    static uint32_t getClusterSize(uint32_t typeId) {
         return (typeId < clusterLimit) ? typeId + 1 : 0;
     }
 
@@ -310,11 +196,7 @@ public:
      * Cluster size == 0 means we have a tree for the given reference.
      * The reference must be valid.
      **/
-    uint32_t
-    getClusterSize(RefType ref) const
-    {
-        return getClusterSize(getTypeId(ref));
-    }
+    uint32_t getClusterSize(RefType ref) const { return getClusterSize(getTypeId(ref)); }
 
     const BTreeType * getTreeEntry(RefType ref) const {
         return _store.getEntry<BTreeType>(ref);
@@ -329,24 +211,18 @@ public:
     }
 
     // Inherit doc from DataStoreBase
-    void
-    reclaim_memory(generation_t oldest_used_gen)
-    {
+    void reclaim_memory(generation_t oldest_used_gen) {
         _allocator.reclaim_memory(oldest_used_gen);
         _store.reclaim_memory(oldest_used_gen);
     }
 
     // Inherit doc from DataStoreBase
-    void
-    assign_generation(generation_t current_gen)
-    {
+    void assign_generation(generation_t current_gen) {
         _allocator.assign_generation(current_gen);
         _store.assign_generation(current_gen);
     }
 
-    void
-    reclaim_all_memory()
-    {
+    void reclaim_all_memory() {
         _allocator.reclaim_all_memory();
         _store.reclaim_all_memory();
     }
@@ -360,30 +236,23 @@ public:
         return usage;
     }
 
-    void
-    clearBuilder()
-    {
+    void clearBuilder() {
         _builder.clear();
     }
 
-    AggregatedType
-    getAggregated(const EntryRef ref) const;
+    AggregatedType getAggregated(const EntryRef ref) const;
 
     template <typename FunctionType>
-    void
-    foreach_unfrozen_key(EntryRef ref, FunctionType func) const;
+    void foreach_unfrozen_key(EntryRef ref, FunctionType func) const;
 
     template <typename FunctionType>
-    void
-    foreach_frozen_key(EntryRef ref, FunctionType func) const;
+    void foreach_frozen_key(EntryRef ref, FunctionType func) const;
 
     template <typename FunctionType>
-    void
-    foreach_unfrozen(EntryRef ref, FunctionType func) const;
+    void foreach_unfrozen(EntryRef ref, FunctionType func) const;
 
     template <typename FunctionType>
-    void
-    foreach_frozen(EntryRef ref, FunctionType func) const;
+    void foreach_frozen(EntryRef ref, FunctionType func) const;
 
     std::unique_ptr<vespalib::datastore::CompactingBuffers> start_compact_worst_btree_nodes(const CompactionStrategy& compaction_strategy);
     void move_btree_nodes(const std::vector<EntryRef>& refs);
@@ -394,12 +263,10 @@ public:
 private:
     static constexpr size_t MIN_BUFFER_ARRAYS = 128u;
     template <typename FunctionType, bool Frozen>
-    void
-    foreach_key(EntryRef ref, FunctionType func) const;
+    void foreach_key(EntryRef ref, FunctionType func) const;
 
     template <typename FunctionType, bool Frozen>
-    void
-    foreach(EntryRef ref, FunctionType func) const;
+    void foreach(EntryRef ref, FunctionType func) const;
 };
 
 template <typename KeyT, typename DataT, typename AggrT, typename CompareT,
diff --git a/vespalib/src/vespa/vespalib/btree/btreestore.hpp b/vespalib/src/vespa/vespalib/btree/btreestore.hpp
index 6b2c4d924cd..a19d0b34aa6 100644
--- a/vespalib/src/vespa/vespalib/btree/btreestore.hpp
+++ b/vespalib/src/vespa/vespalib/btree/btreestore.hpp
@@ -850,7 +850,7 @@ isSmallArray(const EntryRef ref) const
     if (!ref.valid())
         return true;
     RefType iRef(ref);
-    uint32_t typeId(_store.getBufferState(iRef.bufferId()).getTypeId());
+    uint32_t typeId(_store.getBufferMeta(iRef.bufferId()).getTypeId());
     return typeId < clusterLimit;
 }
 
diff --git a/vespalib/src/vespa/vespalib/datastore/array_store.h b/vespalib/src/vespa/vespalib/datastore/array_store.h
index 43bd469fca4..dd786e5f2e2 100644
--- a/vespalib/src/vespa/vespalib/datastore/array_store.h
+++ b/vespalib/src/vespa/vespalib/datastore/array_store.h
@@ -139,7 +139,7 @@ public:
     static DataStoreBase& get_data_store_base(ArrayStore &self) { return self._store; }
 
     // Should only be used for unit testing
-    const BufferState &bufferState(EntryRef ref) const;
+    const BufferState &bufferState(EntryRef ref);
 
     bool has_free_lists_enabled() const { return _store.has_free_lists_enabled(); }
     bool has_held_buffers() const noexcept { return _store.has_held_buffers(); }
diff --git a/vespalib/src/vespa/vespalib/datastore/array_store.hpp b/vespalib/src/vespa/vespalib/datastore/array_store.hpp
index 64aed0bd541..301cff1e414 100644
--- a/vespalib/src/vespa/vespalib/datastore/array_store.hpp
+++ b/vespalib/src/vespa/vespalib/datastore/array_store.hpp
@@ -197,7 +197,7 @@ ArrayStore<EntryT, RefT, TypeMapperT>::update_stat(const CompactionStrategy& com
 
 template <typename EntryT, typename RefT, typename TypeMapperT>
 const BufferState &
-ArrayStore<EntryT, RefT, TypeMapperT>::bufferState(EntryRef ref) const
+ArrayStore<EntryT, RefT, TypeMapperT>::bufferState(EntryRef ref)
 {
     RefT internalRef(ref);
     return _store.getBufferState(internalRef.bufferId());
diff --git a/vespalib/src/vespa/vespalib/datastore/bufferstate.h b/vespalib/src/vespa/vespalib/datastore/bufferstate.h
index 3f023b41c51..aa7f6dfdfa4 100644
--- a/vespalib/src/vespa/vespalib/datastore/bufferstate.h
+++ b/vespalib/src/vespa/vespalib/datastore/bufferstate.h
@@ -40,14 +40,14 @@ public:
 
 private:
     InternalBufferStats _stats;
-    BufferFreeList _free_list;
+    BufferFreeList      _free_list;
     std::atomic<BufferTypeBase*> _typeHandler;
-    Alloc           _buffer;
-    uint32_t        _arraySize;
-    uint16_t        _typeId;
+    Alloc              _buffer;
+    uint32_t           _arraySize;
+    uint16_t           _typeId;
     std::atomic<State> _state;
-    bool            _disableElemHoldList : 1;
-    bool            _compacting : 1;
+    bool               _disableElemHoldList : 1;
+    bool               _compacting : 1;
 
 public:
     /**
@@ -130,7 +130,27 @@ public:
     BufferTypeBase *getTypeHandler() { return _typeHandler.load(std::memory_order_relaxed); }
 
     void resume_primary_buffer(uint32_t buffer_id);
+};
 
+class BufferAndMeta {
+public:
+    BufferAndMeta() : BufferAndMeta(nullptr, 0, 0) { }
+    BufferAndMeta(void* buffer, uint32_t typeId, uint32_t arraySize)
+        : _buffer(buffer),
+          _typeId(typeId),
+          _arraySize(arraySize)
+    { }
+    std::atomic<void*>& get_atomic_buffer() noexcept { return _buffer; }
+    void* get_buffer_relaxed() noexcept { return _buffer.load(std::memory_order_relaxed); }
+    const void* get_buffer_acquire() const noexcept { return _buffer.load(std::memory_order_acquire); }
+    uint32_t getTypeId() const { return _typeId; }
+    uint32_t getArraySize() const { return _arraySize; }
+    void setTypeId(uint32_t typeId) { _typeId = typeId; }
+    void setArraySize(uint32_t arraySize) { _arraySize = arraySize; }
+private:
+    std::atomic<void*> _buffer;
+    uint32_t   _typeId;
+    uint32_t   _arraySize;
 };
 
 }
diff --git a/vespalib/src/vespa/vespalib/datastore/datastore.h b/vespalib/src/vespa/vespalib/datastore/datastore.h
index f0ca9c90700..01b81d0fa58 100644
--- a/vespalib/src/vespa/vespalib/datastore/datastore.h
+++ b/vespalib/src/vespa/vespalib/datastore/datastore.h
@@ -49,7 +49,7 @@ public:
 
     void reclaim_all_entry_refs() override;
 
-    bool getCompacting(EntryRef ref) const {
+    bool getCompacting(EntryRef ref) {
         return getBufferState(RefType(ref).bufferId()).getCompacting();
     }
 
diff --git a/vespalib/src/vespa/vespalib/datastore/datastorebase.cpp b/vespalib/src/vespa/vespalib/datastore/datastorebase.cpp
index 42234194040..99bdb19576f 100644
--- a/vespalib/src/vespa/vespalib/datastore/datastorebase.cpp
+++ b/vespalib/src/vespa/vespalib/datastore/datastorebase.cpp
@@ -106,15 +106,8 @@ DataStoreBase::~DataStoreBase()
 void
 DataStoreBase::switch_primary_buffer(uint32_t typeId, size_t elemsNeeded)
 {
-    size_t buffer_id = primary_buffer_id(typeId);
-    for (size_t i = 0; i < getNumBuffers(); ++i) {
-        // start using next buffer
-        buffer_id = nextBufferId(buffer_id);
-        if (getBufferState(buffer_id).isFree()) {
-            break;
-        }
-    }
-    if (!getBufferState(buffer_id).isFree()) {
+    size_t buffer_id = getFirstFreeBufferId();
+    if ((buffer_id < _states.size()) && !getBufferState(buffer_id).isFree()) {
         LOG_ABORT(vespalib::make_string("switch_primary_buffer(%u, %zu): did not find a free buffer",
                                         typeId, elemsNeeded).c_str());
     }
@@ -164,6 +157,23 @@ DataStoreBase::consider_grow_active_buffer(uint32_t type_id, size_t elems_needed
     return true;
 }
 
+uint32_t
+DataStoreBase::getFirstFreeBufferId() {
+    for (uint32_t buffer_id = 0; buffer_id < _states.size(); buffer_id++) {
+        if (getBufferState(buffer_id).isFree()) {
+            return buffer_id;
+        }
+    }
+    // Need next(new) buffer
+    return _states.size();
+}
+
+BufferState &
+DataStoreBase::getBufferState(uint32_t buffer_id) noexcept {
+    assert(buffer_id < _states.size());
+    return _states[buffer_id];
+}
+
 void
 DataStoreBase::switch_or_grow_primary_buffer(uint32_t typeId, size_t elemsNeeded)
 {
@@ -191,15 +201,8 @@ DataStoreBase::init_primary_buffers()
 {
     uint32_t numTypes = _primary_buffer_ids.size();
     for (uint32_t typeId = 0; typeId < numTypes; ++typeId) {
-        size_t buffer_id = 0;
-        for (size_t i = 0; i < getNumBuffers(); ++i) {
-            if (getBufferState(buffer_id).isFree()) {
-                 break;
-            }
-            // start using next buffer
-            buffer_id = nextBufferId(buffer_id);
-        }
-        assert(getBufferState(buffer_id).isFree());
+        size_t buffer_id = getFirstFreeBufferId();
+        assert((buffer_id == _states.size()) || getBufferState(buffer_id).isFree());
         onActive(buffer_id, typeId, 0u);
         _primary_buffer_ids[typeId] = buffer_id;
     }
@@ -273,14 +276,14 @@ vespalib::MemoryUsage
 DataStoreBase::getMemoryUsage() const {
     auto usage = getDynamicMemoryUsage();
     size_t extra_allocated = 0;
-    extra_allocated += _buffers.capacity() * sizeof(BufferAndTypeId);
+    extra_allocated += _buffers.capacity() * sizeof(BufferAndMeta);
     extra_allocated += _primary_buffer_ids.capacity() * sizeof(uint32_t);
     extra_allocated += _states.capacity() * sizeof(BufferState);
     extra_allocated += _typeHandlers.capacity() * sizeof(BufferTypeBase *);
     extra_allocated += _free_lists.capacity() * sizeof(FreeList);
 
     size_t extra_used = 0;
-    extra_used += _buffers.size() * sizeof(BufferAndTypeId);
+    extra_used += _buffers.size() * sizeof(BufferAndMeta);
     extra_used += _primary_buffer_ids.size() * sizeof(uint32_t);
     extra_used += _states.size() * sizeof(BufferState);
     extra_used += _typeHandlers.size() * sizeof(BufferTypeBase *);
@@ -398,9 +401,11 @@ DataStoreBase::onActive(uint32_t bufferId, uint32_t typeId, size_t elemsNeeded)
 {
     assert(typeId < _typeHandlers.size());
     assert(bufferId < _numBuffers);
-    _buffers[bufferId].setTypeId(typeId);
     BufferState &state = getBufferState(bufferId);
-    state.onActive(bufferId, typeId, _typeHandlers[typeId], elemsNeeded, _buffers[bufferId].get_atomic_buffer());
+    BufferAndMeta & bufferMeta = _buffers[bufferId];
+    state.onActive(bufferId, typeId, _typeHandlers[typeId], elemsNeeded, bufferMeta.get_atomic_buffer());
+    bufferMeta.setTypeId(typeId);
+    bufferMeta.setArraySize(state.getArraySize());
     enableFreeList(bufferId);
 }
 
@@ -461,7 +466,7 @@ DataStoreBase::start_compact_worst_buffers(CompactionSpec compaction_spec, const
                                           compaction_strategy.get_active_buffers_ratio(),
                                           compaction_strategy.getMaxDeadAddressSpaceRatio() / 2,
                                           CompactionStrategy::DEAD_ADDRESS_SPACE_SLACK);
-    uint32_t free_buffers = 0;
+    uint32_t free_buffers = _buffers.size() - _states.size();
     for (uint32_t bufferId = 0; bufferId < _numBuffers; ++bufferId) {
         const auto &state = getBufferState(bufferId);
         if (state.isActive()) {
diff --git a/vespalib/src/vespa/vespalib/datastore/datastorebase.h b/vespalib/src/vespa/vespalib/datastore/datastorebase.h
index 950e1967ee2..8749f2a27e6 100644
--- a/vespalib/src/vespa/vespalib/datastore/datastorebase.h
+++ b/vespalib/src/vespa/vespalib/datastore/datastorebase.h
@@ -73,8 +73,8 @@ public:
      * Get the primary buffer id for the given type id.
      */
     uint32_t primary_buffer_id(uint32_t typeId) const { return _primary_buffer_ids[typeId]; }
-    const BufferState &getBufferState(uint32_t bufferId) const { return _states[bufferId]; }
-    BufferState &getBufferState(uint32_t bufferId) { return _states[bufferId]; }
+    BufferState &getBufferState(uint32_t buffer_id) noexcept;
+    const BufferAndMeta & getBufferMeta(uint32_t buffer_id) const { return _buffers[buffer_id]; }
     uint32_t getNumBuffers() const { return _numBuffers; }
 
     /**
@@ -211,15 +211,6 @@ private:
 
     class BufferHold;
 
-    /**
-     * Get the next buffer id after the given buffer id.
-     */
-    uint32_t nextBufferId(uint32_t bufferId) {
-        uint32_t ret = bufferId + 1;
-        if (ret == _numBuffers)
-            ret = 0;
-        return ret;
-    }
     bool consider_grow_active_buffer(uint32_t type_id, size_t elems_needed);
     void switch_or_grow_primary_buffer(uint32_t typeId, size_t elemsNeeded);
     void markCompacting(uint32_t bufferId);
@@ -243,24 +234,11 @@ private:
 
     void inc_hold_buffer_count();
     void fallbackResize(uint32_t bufferId, size_t elementsNeeded);
+    uint32_t getFirstFreeBufferId();
 
     virtual void reclaim_all_entry_refs() = 0;
 
-    class BufferAndTypeId {
-    public:
-        BufferAndTypeId() : BufferAndTypeId(nullptr, 0) { }
-        BufferAndTypeId(void* buffer, uint32_t typeId) : _buffer(buffer), _typeId(typeId) { }
-        std::atomic<void*>& get_atomic_buffer() noexcept { return _buffer; }
-        void* get_buffer_relaxed() noexcept { return _buffer.load(std::memory_order_relaxed); }
-        const void* get_buffer_acquire() const noexcept { return _buffer.load(std::memory_order_acquire); }
-        uint32_t getTypeId() const { return _typeId; }
-        void setTypeId(uint32_t typeId) { _typeId = typeId; }
-    private:
-        std::atomic<void*> _buffer;
-        uint32_t   _typeId;
-    };
-
-    std::vector<BufferAndTypeId>  _buffers; // For fast mapping with known types
+    std::vector<BufferAndMeta>  _buffers; // For fast mapping with known types
 
     // Provides a mapping from typeId -> primary buffer for that type.
     // The primary buffer is used for allocations of new element(s) if no available slots are found in free lists.
diff --git a/vespalib/src/vespa/vespalib/datastore/entryref.h b/vespalib/src/vespa/vespalib/datastore/entryref.h
index a0016f4fdcb..752d660a097 100644
--- a/vespalib/src/vespa/vespalib/datastore/entryref.h
+++ b/vespalib/src/vespa/vespalib/datastore/entryref.h
@@ -38,8 +38,8 @@ public:
     EntryRefT(const EntryRef & ref_) noexcept : EntryRef(ref_.ref()) {}
     size_t offset() const noexcept { return _ref & (offsetSize() - 1); }
     uint32_t bufferId() const noexcept { return _ref >> OffsetBits; }
-    static size_t offsetSize() noexcept { return 1ul << OffsetBits; }
-    static uint32_t numBuffers() noexcept { return 1 << BufferBits; }
+    static constexpr size_t offsetSize() noexcept { return 1ul << OffsetBits; }
+    static constexpr uint32_t numBuffers() noexcept { return 1 << BufferBits; }
 };
 
 vespalib::asciistream& operator<<(vespalib::asciistream& os, const EntryRef& ref);
diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store.h b/vespalib/src/vespa/vespalib/datastore/unique_store.h
index 1313d57fbab..f2d62020ab4 100644
--- a/vespalib/src/vespa/vespalib/datastore/unique_store.h
+++ b/vespalib/src/vespa/vespalib/datastore/unique_store.h
@@ -67,7 +67,8 @@ public:
     Allocator& get_allocator() { return _allocator; }
     const Allocator& get_allocator() const { return _allocator; }
     IUniqueStoreDictionary& get_dictionary() { return *_dict; }
-    inline const DataStoreType& get_data_store() const noexcept { return _allocator.get_data_store(); }
+    const DataStoreType& get_data_store() const noexcept { return _allocator.get_data_store(); }
+    DataStoreType& get_data_store() noexcept { return _allocator.get_data_store(); }
 
     // Pass on hold list management to underlying store
     void assign_generation(generation_t current_gen);
@@ -78,7 +79,7 @@ public:
     uint32_t getNumUniques() const;
 
     Builder getBuilder(uint32_t uniqueValuesHint);
-    Enumerator getEnumerator(bool sort_unique_values) const;
+    Enumerator getEnumerator(bool sort_unique_values);
 
     // Should only be used for unit testing
     const BufferState &bufferState(EntryRef ref) const;
diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store.hpp b/vespalib/src/vespa/vespalib/datastore/unique_store.hpp
index d9d5f9fee7f..aef6ea07290 100644
--- a/vespalib/src/vespa/vespalib/datastore/unique_store.hpp
+++ b/vespalib/src/vespa/vespalib/datastore/unique_store.hpp
@@ -221,7 +221,7 @@ UniqueStore<EntryT, RefT, Compare, Allocator>::getBuilder(uint32_t uniqueValuesH
 
 template <typename EntryT, typename RefT, typename Compare, typename Allocator>
 typename UniqueStore<EntryT, RefT, Compare, Allocator>::Enumerator
-UniqueStore<EntryT, RefT, Compare, Allocator>::getEnumerator(bool sort_unique_values) const
+UniqueStore<EntryT, RefT, Compare, Allocator>::getEnumerator(bool sort_unique_values)
 {
     return Enumerator(*_dict, _store, sort_unique_values);
 }
diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_enumerator.h b/vespalib/src/vespa/vespalib/datastore/unique_store_enumerator.h
index c4baff2206b..a6e7c2c974e 100644
--- a/vespalib/src/vespa/vespalib/datastore/unique_store_enumerator.h
+++ b/vespalib/src/vespa/vespalib/datastore/unique_store_enumerator.h
@@ -30,9 +30,9 @@ private:
     EnumValues _enumValues;
     uint32_t _next_enum_val;
 
-    void allocate_enum_values();
+    void allocate_enum_values(DataStoreBase &store);
 public:
-    UniqueStoreEnumerator(const IUniqueStoreDictionary &dict, const DataStoreBase &store, bool sort_unique_values);
+    UniqueStoreEnumerator(const IUniqueStoreDictionary &dict, DataStoreBase &store, bool sort_unique_values);
     ~UniqueStoreEnumerator();
     void enumerateValue(EntryRef ref);
     void enumerateValues();
diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_enumerator.hpp b/vespalib/src/vespa/vespalib/datastore/unique_store_enumerator.hpp
index 52437fc765c..6d08a027bf1 100644
--- a/vespalib/src/vespa/vespalib/datastore/unique_store_enumerator.hpp
+++ b/vespalib/src/vespa/vespalib/datastore/unique_store_enumerator.hpp
@@ -9,7 +9,7 @@
 namespace vespalib::datastore {
 
 template <typename RefT>
-UniqueStoreEnumerator<RefT>::UniqueStoreEnumerator(const IUniqueStoreDictionary &dict, const DataStoreBase &store, bool sort_unique_values)
+UniqueStoreEnumerator<RefT>::UniqueStoreEnumerator(const IUniqueStoreDictionary &dict, DataStoreBase &store, bool sort_unique_values)
     : _dict_snapshot(dict.get_read_snapshot()),
       _store(store),
       _enumValues(),
@@ -19,7 +19,7 @@ UniqueStoreEnumerator<RefT>::UniqueStoreEnumerator(const IUniqueStoreDictionary
     if (sort_unique_values) {
         _dict_snapshot->sort();
     }
-    allocate_enum_values();
+    allocate_enum_values(store);
 }
 
 template <typename RefT>
@@ -40,11 +40,11 @@ UniqueStoreEnumerator<RefT>::enumerateValue(EntryRef ref)
 
 template <typename RefT>
 void
-UniqueStoreEnumerator<RefT>::allocate_enum_values()
+UniqueStoreEnumerator<RefT>::allocate_enum_values(DataStoreBase & store)
 {
     _enumValues.resize(RefType::numBuffers());
     for (uint32_t bufferId = 0; bufferId < RefType::numBuffers(); ++bufferId) {
-        const BufferState &state = _store.getBufferState(bufferId);
+        const BufferState &state = store.getBufferState(bufferId);
         if (state.isActive()) {
             _enumValues[bufferId].resize(state.get_used_arrays());
         }
diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_string_allocator.h b/vespalib/src/vespa/vespalib/datastore/unique_store_string_allocator.h
index 265478fbaf5..a85b73f423d 100644
--- a/vespalib/src/vespa/vespalib/datastore/unique_store_string_allocator.h
+++ b/vespalib/src/vespa/vespalib/datastore/unique_store_string_allocator.h
@@ -114,7 +114,7 @@ public:
     EntryRef move_on_compact(EntryRef ref) override;
     const UniqueStoreEntryBase& get_wrapped(EntryRef ref) const {
         RefType iRef(ref);
-        auto &state = _store.getBufferState(iRef.bufferId());
+        auto &state = _store.getBufferMeta(iRef.bufferId());
         auto type_id = state.getTypeId();
         if (type_id != 0) {
             return *reinterpret_cast<const UniqueStoreEntryBase *>(_store.template getEntryArray<char>(iRef, state.getArraySize()));
@@ -124,7 +124,7 @@ public:
     }
     const char *get(EntryRef ref) const {
         RefType iRef(ref);
-        auto &state = _store.getBufferState(iRef.bufferId());
+        auto &state = _store.getBufferMeta(iRef.bufferId());
         auto type_id = state.getTypeId();
         if (type_id != 0) {
             return reinterpret_cast<const UniqueStoreSmallStringEntry *>(_store.template getEntryArray<char>(iRef, state.getArraySize()))->value();
diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_string_comparator.h b/vespalib/src/vespa/vespalib/datastore/unique_store_string_comparator.h
index 73a812ccd0b..e507132a085 100644
--- a/vespalib/src/vespa/vespalib/datastore/unique_store_string_comparator.h
+++ b/vespalib/src/vespa/vespalib/datastore/unique_store_string_comparator.h
@@ -25,10 +25,10 @@ protected:
     const char *get(EntryRef ref) const {
         if (ref.valid()) {
             RefType iRef(ref);
-            auto &state = _store.getBufferState(iRef.bufferId());
-            auto type_id = state.getTypeId();
+            const auto &meta = _store.getBufferMeta(iRef.bufferId());
+            auto type_id = meta.getTypeId();
             if (type_id != 0) {
-                return reinterpret_cast<const UniqueStoreSmallStringEntry *>(_store.template getEntryArray<char>(iRef, state.getArraySize()))->value();
+                return reinterpret_cast<const UniqueStoreSmallStringEntry *>(_store.template getEntryArray<char>(iRef, meta.getArraySize()))->value();
             } else {
                 return _store.template getEntry<WrappedExternalEntryType>(iRef)->value().c_str();
             }