Merge branch 'master' into bjormel/aws-main-controller

17 files changed, 318 insertions, 213 deletions

diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
index 9533f04107d..e675e00b642 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
@@ -41,16 +41,18 @@ import com.yahoo.slime.Slime;
 import com.yahoo.transaction.NestedTransaction;
 import com.yahoo.transaction.Transaction;
 import com.yahoo.vespa.applicationmodel.InfrastructureApplication;
+import com.yahoo.vespa.config.server.application.ActiveTokenFingerprints.Token;
+import com.yahoo.vespa.config.server.application.ActiveTokenFingerprintsClient;
 import com.yahoo.vespa.config.server.application.Application;
 import com.yahoo.vespa.config.server.application.ApplicationCuratorDatabase;
 import com.yahoo.vespa.config.server.application.ApplicationData;
 import com.yahoo.vespa.config.server.application.ApplicationReindexing;
-import com.yahoo.vespa.config.server.application.ApplicationReindexing.Status;
 import com.yahoo.vespa.config.server.application.ApplicationVersions;
 import com.yahoo.vespa.config.server.application.ClusterReindexing;
 import com.yahoo.vespa.config.server.application.ClusterReindexingStatusClient;
 import com.yahoo.vespa.config.server.application.CompressedApplicationInputStream;
 import com.yahoo.vespa.config.server.application.ConfigConvergenceChecker;
+import com.yahoo.vespa.config.server.application.ActiveTokenFingerprints;
 import com.yahoo.vespa.config.server.application.DefaultClusterReindexingStatusClient;
 import com.yahoo.vespa.config.server.application.FileDistributionStatus;
 import com.yahoo.vespa.config.server.application.HttpProxy;
@@ -129,7 +131,6 @@ import static com.yahoo.vespa.config.server.tenant.TenantRepository.HOSTED_VESPA
 import static com.yahoo.vespa.curator.Curator.CompletionWaiter;
 import static com.yahoo.yolean.Exceptions.uncheck;
 import static java.nio.file.Files.readAttributes;
-import static java.util.Comparator.naturalOrder;
 
 /**
  * The API for managing applications.
@@ -159,6 +160,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
     private final Metric metric;
     private final SecretStoreValidator secretStoreValidator;
     private final ClusterReindexingStatusClient clusterReindexingStatusClient;
+    private final ActiveTokenFingerprints activeTokenFingerprints;
     private final FlagSource flagSource;
 
     @Inject
@@ -188,6 +190,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
              metric,
              new SecretStoreValidator(secretStore),
              new DefaultClusterReindexingStatusClient(),
+             new ActiveTokenFingerprintsClient(),
              flagSource);
     }
 
@@ -205,6 +208,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
                                   Metric metric,
                                   SecretStoreValidator secretStoreValidator,
                                   ClusterReindexingStatusClient clusterReindexingStatusClient,
+                                  ActiveTokenFingerprints activeTokenFingerprints,
                                   FlagSource flagSource) {
         this.tenantRepository = Objects.requireNonNull(tenantRepository);
         this.hostProvisioner = Objects.requireNonNull(hostProvisioner);
@@ -219,7 +223,8 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
         this.testerClient = Objects.requireNonNull(testerClient);
         this.metric = Objects.requireNonNull(metric);
         this.secretStoreValidator = Objects.requireNonNull(secretStoreValidator);
-        this.clusterReindexingStatusClient = clusterReindexingStatusClient;
+        this.clusterReindexingStatusClient = Objects.requireNonNull(clusterReindexingStatusClient);
+        this.activeTokenFingerprints = Objects.requireNonNull(activeTokenFingerprints);
         this.flagSource = flagSource;
     }
 
@@ -237,6 +242,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
         private SecretStoreValidator secretStoreValidator = new SecretStoreValidator(new SecretStoreProvider().get());
         private FlagSource flagSource = new InMemoryFlagSource();
         private ConfigConvergenceChecker configConvergenceChecker = new ConfigConvergenceChecker();
+        private Map<String, List<Token>> activeTokens = Map.of();
 
         public Builder withTenantRepository(TenantRepository tenantRepository) {
             this.tenantRepository = tenantRepository;
@@ -298,6 +304,11 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
             return this;
         }
 
+        public Builder withActiveTokens(Map<String, List<Token>> tokens) {
+            this.activeTokens = tokens;
+            return this;
+        }
+
         public ApplicationRepository build() {
             return new ApplicationRepository(tenantRepository,
                                              tenantRepository.hostProvisionerProvider().getHostProvisioner(),
@@ -313,6 +324,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
                                              metric,
                                              secretStoreValidator,
                                              ClusterReindexingStatusClient.DUMMY_INSTANCE,
+                                             __ -> activeTokens,
                                              flagSource);
         }
 
@@ -612,6 +624,10 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
         return uncheck(() -> clusterReindexingStatusClient.getReindexingStatus(getApplication(applicationId)));
     }
 
+    public Map<String, List<Token>> activeTokenFingerprints(ApplicationId applicationId) {
+        return activeTokenFingerprints.get(getApplication(applicationId));
+    }
+
     public Long getApplicationGeneration(ApplicationId applicationId) {
         return getApplication(applicationId).getApplicationGeneration();
     }
@@ -1030,21 +1046,21 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
     private Session validateThatLocalSessionIsNotActive(Tenant tenant, long sessionId) {
         Session session = getLocalSession(tenant, sessionId);
         if (Session.Status.ACTIVATE.equals(session.getStatus())) {
-            throw new IllegalArgumentException("Session is active: " + sessionId);
+            throw new IllegalArgumentException("Session " + sessionId + " for '" + tenant.getName() + "' is active");
         }
         return session;
     }
 
     private Session getLocalSession(Tenant tenant, long sessionId) {
         Session session = tenant.getSessionRepository().getLocalSession(sessionId);
-        if (session == null) throw new NotFoundException("Session " + sessionId + " was not found");
+        if (session == null) throw new NotFoundException("Local session " + sessionId + " for '" + tenant.getName() + "' was not found");
 
         return session;
     }
 
     private RemoteSession getRemoteSession(Tenant tenant, long sessionId) {
         RemoteSession session = tenant.getSessionRepository().getRemoteSession(sessionId);
-        if (session == null) throw new NotFoundException("Session " + sessionId + " was not found");
+        if (session == null) throw new NotFoundException("Remote session " + sessionId + " for '" + tenant.getName() + "' was not found");
 
         return session;
     }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/FallbackOnnxModelCostProvider.java b/configserver/src/main/java/com/yahoo/vespa/config/server/FallbackOnnxModelCostProvider.java
new file mode 100644
index 00000000000..57cfb1cd43b
--- /dev/null
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/FallbackOnnxModelCostProvider.java
@@ -0,0 +1,16 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.vespa.config.server;
+
+import com.yahoo.config.model.api.OnnxModelCost;
+import com.yahoo.container.di.componentgraph.Provider;
+
+/**
+ * Default provider that provides a disabled {@link OnnxModelCost} instance.
+ *
+ * @author bjorncs
+ */
+public class FallbackOnnxModelCostProvider implements Provider<OnnxModelCost> {
+    @Override public OnnxModelCost get() { return OnnxModelCost.disabled(); }
+    @Override public void deconstruct() {}
+}
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ActiveTokenFingerprints.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ActiveTokenFingerprints.java
new file mode 100644
index 00000000000..9cde5e38302
--- /dev/null
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ActiveTokenFingerprints.java
@@ -0,0 +1,18 @@
+package com.yahoo.vespa.config.server.application;
+
+import com.yahoo.vespa.config.server.modelfactory.ModelResult;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * @author jonmv
+ */
+public interface ActiveTokenFingerprints {
+
+    /** Lists all active tokens and their fingerprints for each token-enabled container host in the application, that is currently up. */
+    Map<String, List<Token>> get(ModelResult application);
+
+    record Token(String id, List<String> fingerprints) { }
+
+}
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ActiveTokenFingerprintsClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ActiveTokenFingerprintsClient.java
new file mode 100644
index 00000000000..4e9eac7a9a6
--- /dev/null
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ActiveTokenFingerprintsClient.java
@@ -0,0 +1,123 @@
+package com.yahoo.vespa.config.server.application;
+
+import ai.vespa.http.DomainName;
+import ai.vespa.http.HttpURL;
+import ai.vespa.http.HttpURL.Path;
+import ai.vespa.http.HttpURL.Scheme;
+import ai.vespa.util.http.hc5.VespaAsyncHttpClientBuilder;
+import com.yahoo.config.model.api.ApplicationClusterEndpoint.AuthMethod;
+import com.yahoo.config.model.api.ServiceInfo;
+import com.yahoo.slime.Inspector;
+import com.yahoo.vespa.config.server.modelfactory.ModelResult;
+import org.apache.hc.client5.http.async.methods.SimpleHttpResponse;
+import org.apache.hc.client5.http.async.methods.SimpleRequestBuilder;
+import org.apache.hc.client5.http.config.ConnectionConfig;
+import org.apache.hc.client5.http.config.RequestConfig;
+import org.apache.hc.client5.http.impl.async.CloseableHttpAsyncClient;
+import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManagerBuilder;
+import org.apache.hc.core5.concurrent.FutureCallback;
+import org.apache.hc.core5.io.CloseMode;
+import org.apache.hc.core5.reactor.IOReactorConfig;
+import org.apache.hc.core5.util.TimeValue;
+import org.apache.hc.core5.util.Timeout;
+
+import java.net.URI;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.Phaser;
+
+import static com.yahoo.config.model.api.container.ContainerServiceType.CONTAINER;
+import static com.yahoo.config.model.api.container.ContainerServiceType.QRSERVER;
+import static com.yahoo.slime.SlimeUtils.entriesStream;
+import static com.yahoo.slime.SlimeUtils.jsonToSlime;
+import static java.util.stream.Collectors.toMap;
+import static java.util.stream.Collectors.toSet;
+
+/**
+ * @author jonmv
+ */
+public class ActiveTokenFingerprintsClient implements ActiveTokenFingerprints, AutoCloseable {
+
+    private final CloseableHttpAsyncClient httpClient = createHttpClient();
+
+    public ActiveTokenFingerprintsClient() {
+        httpClient.start();
+    }
+
+    @Override
+    public Map<String, List<Token>> get(ModelResult application) {
+        Set<String> containersWithTokenFilter = application.getModel().applicationClusterInfo().stream()
+                                                           .flatMap(cluster -> cluster.endpoints().stream())
+                                                           .filter(endpoint -> endpoint.authMethod() == AuthMethod.token)
+                                                           .flatMap(endpoint -> endpoint.hostNames().stream())
+                                                           .collect(toSet());
+        return getFingerprints(application.getModel().getHosts().stream()
+                                          .filter(host -> containersWithTokenFilter.contains(host.getHostname()))
+                                          .flatMap(host -> host.getServices().stream())
+                                          .filter(service ->    service.getServiceType().equals(CONTAINER.serviceName)
+                                                             || service.getServiceType().equals(QRSERVER.serviceName))
+                                          .toList());
+    }
+
+    private Map<String, List<Token>> getFingerprints(List<ServiceInfo> services) {
+        Map<String, List<Token>> tokens = new ConcurrentHashMap<>();
+        Phaser phaser = new Phaser(services.size() + 1);
+        for (ServiceInfo service : services) getFingerprints(tokens, service, phaser);
+        phaser.arriveAndAwaitAdvance();
+        return tokens;
+    }
+
+    // A container may be unable to provide its fingerprints for a number of reasons, which may be OK, so
+    // we only track those containers which return an OK response, but we do require at least one such response.
+    private void getFingerprints(Map<String, List<Token>> hostTokens, ServiceInfo service, Phaser phaser) {
+        URI uri = HttpURL.create(Scheme.http,
+                                 DomainName.of(service.getHostName()),
+                                 service.getPorts().stream().filter(port -> port.getTags().stream().anyMatch("http"::equals)).findAny().get().getPort(),
+                                 Path.parse("/data-plane-tokens/v1"))
+                         .asURI();
+        httpClient.execute(SimpleRequestBuilder.get(uri).build(), new FutureCallback<>() {
+            @Override public void completed(SimpleHttpResponse result) {
+                if (result.getCode() == 200) hostTokens.put(service.getHostName(), parseTokens(result));
+                phaser.arrive();
+            }
+            @Override public void failed(Exception ex) { phaser.arrive(); }
+            @Override public void cancelled() { phaser.arrive(); }
+        });
+    }
+
+    private static List<Token> parseTokens(SimpleHttpResponse response) {
+        return entriesStream(jsonToSlime(response.getBodyBytes()).get().field("tokens"))
+                .map(entry -> new Token(entry.field("id").asString(),
+                                        entriesStream(entry.field("fingerprints")).map(Inspector::asString).toList()))
+                .toList();
+    }
+
+    private static CloseableHttpAsyncClient createHttpClient() {
+        return VespaAsyncHttpClientBuilder
+                .create(tlsStrategy -> PoolingAsyncClientConnectionManagerBuilder.create()
+                                                                                 .setTlsStrategy(tlsStrategy)
+                                                                                 .setDefaultConnectionConfig(ConnectionConfig.custom()
+                                                                                                                             .setConnectTimeout(Timeout.ofSeconds(2))
+                                                                                                                             .build())
+                                                                                 .build())
+                .setIOReactorConfig(IOReactorConfig.custom()
+                                                   .setSoTimeout(Timeout.ofSeconds(2))
+                                                   .build())
+                .setDefaultRequestConfig(
+                        RequestConfig.custom()
+                                     .setConnectionRequestTimeout(Timeout.ofSeconds(2))
+                                     .setResponseTimeout(Timeout.ofSeconds(2))
+                                     .build())
+                .setUserAgent("data-plane-token-client")
+                .build();
+    }
+
+    @Override
+    public void close() throws Exception {
+        httpClient.close(CloseMode.GRACEFUL);
+        httpClient.awaitShutdown(TimeValue.ofSeconds(10));
+    }
+
+}
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java
index 693252da43a..ff2c137c11c 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java
@@ -23,7 +23,6 @@ import com.yahoo.vespa.config.server.monitoring.MetricUpdater;
 import com.yahoo.vespa.config.server.monitoring.Metrics;
 import com.yahoo.vespa.config.server.rpc.ConfigResponseFactory;
 import com.yahoo.vespa.config.server.tenant.TenantRepository;
-import com.yahoo.vespa.curator.CompletionTimeoutException;
 import com.yahoo.vespa.curator.Curator;
 import com.yahoo.vespa.curator.Lock;
 import com.yahoo.vespa.curator.transaction.CuratorTransaction;
@@ -37,7 +36,6 @@ import java.nio.file.Files;
 import java.nio.file.Paths;
 import java.time.Clock;
 import java.time.Duration;
-import java.time.Instant;
 import java.util.Collection;
 import java.util.LinkedHashSet;
 import java.util.List;
@@ -62,6 +60,8 @@ import static java.util.stream.Collectors.toSet;
 public class TenantApplications implements RequestHandler, HostValidator {
 
     private static final Logger log = Logger.getLogger(TenantApplications.class.getName());
+    /* Time to wait for all config servers to get event when an application is removed */
+    private static final Duration waitForAll = Duration.ofSeconds(5);
 
     private final Curator curator;
     private final ApplicationCuratorDatabase database;
@@ -430,146 +430,17 @@ public class TenantApplications implements RequestHandler, HostValidator {
     public TenantFileSystemDirs getTenantFileSystemDirs() { return tenantFileSystemDirs; }
 
     public CompletionWaiter createRemoveApplicationWaiter(ApplicationId applicationId) {
-        return RemoveApplicationWaiter.createAndInitialize(curator, applicationId, serverId);
+        return curator.createCompletionWaiter(barrierPath(applicationId), serverId, waitForAll);
     }
 
     public CompletionWaiter getRemoveApplicationWaiter(ApplicationId applicationId) {
-        return RemoveApplicationWaiter.create(curator, applicationId, serverId);
+        return curator.getCompletionWaiter(barrierPath(applicationId), serverId, waitForAll);
     }
 
-    /**
-     * Waiter for removing application. Will wait for some time for all servers to remove application,
-     * but will accept the majority of servers to have removed app if it takes a long time.
-     */
-    // TODO: Merge with CuratorCompletionWaiter
-    static class RemoveApplicationWaiter implements CompletionWaiter {
-
-        private static final java.util.logging.Logger log = Logger.getLogger(RemoveApplicationWaiter.class.getName());
-        private static final Duration waitForAllDefault = Duration.ofSeconds(5);
-
-        private final Curator curator;
-        private final Path barrierPath;
-        private final Path waiterNode;
-        private final Duration waitForAll;
-        private final Clock clock = Clock.systemUTC();
-
-        RemoveApplicationWaiter(Curator curator, ApplicationId applicationId, String serverId) {
-            this(curator, applicationId, serverId, waitForAllDefault);
-        }
-
-        RemoveApplicationWaiter(Curator curator, ApplicationId applicationId, String serverId, Duration waitForAll) {
-            this.barrierPath = TenantRepository.getBarriersPath().append(applicationId.tenant().value())
-                                               .append("delete-application")
-                                               .append(applicationId.serializedForm());
-            this.waiterNode = barrierPath.append(serverId);
-            this.curator = curator;
-            this.waitForAll = waitForAll;
-        }
-
-        @Override
-        public void awaitCompletion(Duration timeout) {
-            List<String> respondents;
-            try {
-                respondents = awaitInternal(timeout);
-            } catch (Exception e) {
-                throw new RuntimeException(e);
-            }
-            if (respondents.size() < barrierMemberCount()) {
-                throw new CompletionTimeoutException("Timed out waiting for peer config servers to remove application " +
-                                                     "(waited for barrier " + barrierPath + ")." +
-                                                     "Got response from " + respondents + ", but need response from " +
-                                                     "at least " + barrierMemberCount() + " server(s). " +
-                                                     "Timeout passed as argument was " + timeout.toMillis() + " ms");
-            }
-        }
-
-        private List<String> awaitInternal(Duration timeout) throws Exception {
-            Instant startTime = clock.instant();
-            Instant endTime = startTime.plus(timeout);
-            Instant gotQuorumTime = Instant.EPOCH;
-            List<String> respondents;
-            do {
-                respondents = curator.framework().getChildren().forPath(barrierPath.getAbsolute());
-                if (log.isLoggable(Level.FINE)) {
-                    log.log(Level.FINE, respondents.size() + "/" + curator.zooKeeperEnsembleCount() + " responded: " +
-                                        respondents + ", all participants: " + curator.zooKeeperEnsembleConnectionSpec());
-                }
-
-                // If all config servers responded, return
-                if (respondents.size() == curator.zooKeeperEnsembleCount()) {
-                    logBarrierCompleted(respondents, startTime);
-                    break;
-                }
-
-                // If some are missing, quorum is enough, but wait for all up to 5 seconds before returning
-                if (respondents.size() >= barrierMemberCount()) {
-                    if (gotQuorumTime.isBefore(startTime))
-                        gotQuorumTime = clock.instant();
-
-                    // Give up if more than some time has passed since we got quorum, otherwise continue
-                    if (Duration.between(clock.instant(), gotQuorumTime.plus(waitForAll)).isNegative()) {
-                        logBarrierCompleted(respondents, startTime);
-                        break;
-                    }
-                }
-
-                Thread.sleep(100);
-            } while (clock.instant().isBefore(endTime));
-
-            return respondents;
-        }
-
-        private void logBarrierCompleted(List<String> respondents, Instant startTime) {
-            Duration duration = Duration.between(startTime, Instant.now());
-            Level level = (duration.minus(Duration.ofSeconds(5))).isNegative() ? Level.FINE : Level.INFO;
-            log.log(level, () -> barrierCompletedMessage(respondents, duration));
-        }
-
-        private String barrierCompletedMessage(List<String> respondents, Duration duration) {
-            return barrierPath + " completed in " + duration.toString() +
-                   ", " + respondents.size() + "/" + curator.zooKeeperEnsembleCount() + " responded: " + respondents;
-        }
-
-        @Override
-        public void notifyCompletion() {
-            try {
-                curator.framework().create().forPath(waiterNode.getAbsolute());
-            } catch (Exception e) {
-                throw new RuntimeException(e);
-            }
-        }
-
-        @Override
-        public String toString() { return "'" + barrierPath + "', " + barrierMemberCount() + " members"; }
-
-        public static CompletionWaiter create(Curator curator, ApplicationId applicationId, String serverId) {
-            return new RemoveApplicationWaiter(curator, applicationId, serverId);
-        }
-
-        public static CompletionWaiter create(Curator curator, ApplicationId applicationId, String serverId, Duration waitForAll) {
-            return new RemoveApplicationWaiter(curator, applicationId, serverId, waitForAll);
-        }
-
-        public static CompletionWaiter createAndInitialize(Curator curator, ApplicationId applicationId, String serverId) {
-            return createAndInitialize(curator, applicationId, serverId, waitForAllDefault);
-        }
-
-        public static CompletionWaiter createAndInitialize(Curator curator, ApplicationId applicationId, String serverId, Duration waitForAll) {
-            RemoveApplicationWaiter waiter = new RemoveApplicationWaiter(curator, applicationId, serverId, waitForAll);
-
-            // Cleanup and create a new barrier path
-            Path barrierPath = waiter.barrierPath();
-            curator.delete(barrierPath);
-            curator.create(barrierPath.getParentPath());
-            curator.createAtomically(barrierPath);
-
-            return waiter;
-        }
-
-        private int barrierMemberCount() { return (curator.zooKeeperEnsembleCount() / 2) + 1; /* majority */ }
-
-        private Path barrierPath() { return barrierPath; }
-
+    private static Path barrierPath(ApplicationId applicationId) {
+        return TenantRepository.getBarriersPath().append(applicationId.tenant().value())
+                .append("delete-application")
+                .append(applicationId.serializedForm());
     }
 
 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLogger.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLogger.java
index 154d2d0f2f0..042aa2423f3 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLogger.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLogger.java
@@ -11,6 +11,7 @@ import com.yahoo.slime.Slime;
 import com.yahoo.vespa.config.server.session.PrepareParams;
 import com.yahoo.vespa.config.server.tenant.TenantRepository;
 
+import java.util.function.Supplier;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
@@ -35,15 +36,17 @@ public class DeployHandlerLogger implements DeployLogger {
         this.logroot = slime.setObject().setArray("log");
     }
 
+    @Override public void log(Level level, String message) { log(level, () -> message); }
+    @Override public void log(Level level, Supplier<String> message) { log(level, message, null); }
+
     @Override
     @SuppressWarnings("deprecation")
-    public void log(Level level, String message) {
-        if (level.intValue() <= LogLevel.DEBUG.intValue() && !verbose)
-            return;
+    public void log(Level level, Supplier<String> supplier, Throwable throwable) {
+        // Also tee to a normal log, Vespa log for example, but use level fine
+        log.log(Level.FINE, throwable, () -> prefix + supplier.get());
 
-        logJson(level, message);
-        // Also tee to a normal log, Vespa log for example, but use level fine 
-        log.log(Level.FINE, () -> prefix + message);
+        if (level.intValue() <= LogLevel.DEBUG.intValue() && !verbose) return;
+        logJson(level, supplier.get());
     }
 
     @Override
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java
index 142f98e13e3..96b0b03c832 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java
@@ -13,6 +13,7 @@ import com.yahoo.config.model.api.EndpointCertificateSecrets;
 import com.yahoo.config.model.api.HostProvisioner;
 import com.yahoo.config.model.api.Model;
 import com.yahoo.config.model.api.ModelContext;
+import com.yahoo.config.model.api.OnnxModelCost;
 import com.yahoo.config.model.api.Provisioned;
 import com.yahoo.config.model.api.Quota;
 import com.yahoo.config.model.api.Reindexing;
@@ -28,6 +29,7 @@ import com.yahoo.config.provision.Zone;
 import com.yahoo.container.jdisc.secretstore.SecretStore;
 import com.yahoo.vespa.config.server.tenant.SecretStoreExternalIdRetriever;
 import com.yahoo.vespa.flags.FetchVector;
+import com.yahoo.vespa.flags.Flag;
 import com.yahoo.vespa.flags.FlagSource;
 import com.yahoo.vespa.flags.Flags;
 import com.yahoo.vespa.flags.PermanentFlags;
@@ -66,6 +68,7 @@ public class ModelContextImpl implements ModelContext {
     private final Optional<? extends Reindexing> reindexing;
     private final ModelContext.Properties properties;
     private final Optional<File> appDir;
+    private final OnnxModelCost onnxModelCost;
 
     private final Optional<DockerImage> wantedDockerImageRepository;
 
@@ -92,6 +95,7 @@ public class ModelContextImpl implements ModelContext {
                             Provisioned provisioned,
                             ModelContext.Properties properties,
                             Optional<File> appDir,
+                            OnnxModelCost onnxModelCost,
                             Optional<DockerImage> wantedDockerImageRepository,
                             Version modelVespaVersion,
                             Version wantedNodeVespaVersion) {
@@ -109,6 +113,7 @@ public class ModelContextImpl implements ModelContext {
         this.wantedDockerImageRepository = wantedDockerImageRepository;
         this.modelVespaVersion = modelVespaVersion;
         this.wantedNodeVespaVersion = wantedNodeVespaVersion;
+        this.onnxModelCost = onnxModelCost;
     }
 
     @Override
@@ -150,6 +155,8 @@ public class ModelContextImpl implements ModelContext {
     @Override
     public Optional<File> appDir() { return appDir; }
 
+    @Override public OnnxModelCost onnxModelCost() { return onnxModelCost; }
+
     @Override
     public Optional<DockerImage> wantedDockerImageRepo() { return wantedDockerImageRepository; }
 
@@ -201,6 +208,8 @@ public class ModelContextImpl implements ModelContext {
         private final boolean enableNestedMultivalueGrouping;
         private final boolean useReconfigurableDispatcher;
         private final int contentLayerMetadataFeatureLevel;
+        private final boolean dynamicHeapSize;
+        private final String unknownConfigDefinition;
 
         public FeatureFlags(FlagSource source, ApplicationId appId, Version version) {
             this.defaultTermwiseLimit = flagValue(source, appId, version, Flags.DEFAULT_TERM_WISE_LIMIT);
@@ -243,6 +252,8 @@ public class ModelContextImpl implements ModelContext {
             this.enableNestedMultivalueGrouping = flagValue(source, appId, version, Flags.ENABLE_NESTED_MULTIVALUE_GROUPING);
             this.useReconfigurableDispatcher = flagValue(source, appId, version, Flags.USE_RECONFIGURABLE_DISPATCHER);
             this.contentLayerMetadataFeatureLevel = flagValue(source, appId, version, Flags.CONTENT_LAYER_METADATA_FEATURE_LEVEL);
+            this.dynamicHeapSize = flagValue(source, appId, version, Flags.DYNAMIC_HEAP_SIZE);
+            this.unknownConfigDefinition = flagValue(source, appId, version, Flags.UNKNOWN_CONFIG_DEFINITION);
         }
 
         @Override public int heapSizePercentage() { return heapPercentage; }
@@ -293,10 +304,13 @@ public class ModelContextImpl implements ModelContext {
         @Override public boolean enableNestedMultivalueGrouping() { return enableNestedMultivalueGrouping; }
         @Override public boolean useReconfigurableDispatcher() { return useReconfigurableDispatcher; }
         @Override public int contentLayerMetadataFeatureLevel() { return contentLayerMetadataFeatureLevel; }
+        @Override public boolean dynamicHeapSize() { return dynamicHeapSize; }
+        @Override public String unknownConfigDefinition() { return unknownConfigDefinition; }
 
         private static <V> V flagValue(FlagSource source, ApplicationId appId, Version vespaVersion, UnboundFlag<? extends V, ?, ?> flag) {
             return flag.bindTo(source)
                     .with(FetchVector.Dimension.INSTANCE_ID, appId.serializedForm())
+                    .with(FetchVector.Dimension.APPLICATION_ID, appId.toSerializedFormWithoutInstance())
                     .with(FetchVector.Dimension.VESPA_VERSION, vespaVersion.toFullString())
                     .with(FetchVector.Dimension.TENANT_ID, appId.tenant().value())
                     .boxedValue();
@@ -309,6 +323,7 @@ public class ModelContextImpl implements ModelContext {
                                        UnboundFlag<? extends V, ?, ?> flag) {
             return flag.bindTo(source)
                        .with(FetchVector.Dimension.INSTANCE_ID, appId.serializedForm())
+                       .with(FetchVector.Dimension.APPLICATION_ID, appId.toSerializedFormWithoutInstance())
                        .with(FetchVector.Dimension.CLUSTER_TYPE, clusterType.name())
                        .with(FetchVector.Dimension.VESPA_VERSION, vespaVersion.toFullString())
                        .boxedValue();
@@ -321,6 +336,7 @@ public class ModelContextImpl implements ModelContext {
                                        UnboundFlag<? extends V, ?, ?> flag) {
             return flag.bindTo(source)
                        .with(FetchVector.Dimension.INSTANCE_ID, appId.serializedForm())
+                       .with(FetchVector.Dimension.APPLICATION_ID, appId.toSerializedFormWithoutInstance())
                        .with(FetchVector.Dimension.CLUSTER_ID, clusterId.value())
                        .with(FetchVector.Dimension.VESPA_VERSION, vespaVersion.toFullString())
                        .boxedValue();
@@ -397,21 +413,16 @@ public class ModelContextImpl implements ModelContext {
             this.tenantSecretStores = tenantSecretStores;
             this.secretStore = secretStore;
             this.jvmGCOptionsFlag = PermanentFlags.JVM_GC_OPTIONS.bindTo(flagSource)
-                    .with(FetchVector.Dimension.INSTANCE_ID, applicationId.serializedForm());
-            this.allowDisableMtls = PermanentFlags.ALLOW_DISABLE_MTLS.bindTo(flagSource)
-                    .with(FetchVector.Dimension.INSTANCE_ID, applicationId.serializedForm()).value();
+                    .with(FetchVector.Dimension.INSTANCE_ID, applicationId.serializedForm())
+                    .with(FetchVector.Dimension.APPLICATION_ID, applicationId.toSerializedFormWithoutInstance());
+            this.allowDisableMtls = flagValue(flagSource, applicationId, PermanentFlags.ALLOW_DISABLE_MTLS);
             this.operatorCertificates = operatorCertificates;
-            this.tlsCiphersOverride = PermanentFlags.TLS_CIPHERS_OVERRIDE.bindTo(flagSource)
-                    .with(FetchVector.Dimension.INSTANCE_ID, applicationId.serializedForm()).value();
+            this.tlsCiphersOverride = flagValue(flagSource, applicationId, PermanentFlags.TLS_CIPHERS_OVERRIDE);
             this.zoneDnsSuffixes = configserverConfig.zoneDnsSuffixes();
-            this.environmentVariables = PermanentFlags.ENVIRONMENT_VARIABLES.bindTo(flagSource)
-                    .with(FetchVector.Dimension.INSTANCE_ID, applicationId.serializedForm()).value();
+            this.environmentVariables = flagValue(flagSource, applicationId, PermanentFlags.ENVIRONMENT_VARIABLES);
             this.cloudAccount = cloudAccount;
-            this.allowUserFilters = PermanentFlags.ALLOW_USER_FILTERS.bindTo(flagSource)
-                    .with(FetchVector.Dimension.INSTANCE_ID, applicationId.serializedForm()).value();
-            this.endpointConnectionTtl = Duration.ofSeconds(
-                    PermanentFlags.ENDPOINT_CONNECTION_TTL.bindTo(flagSource)
-                            .with(FetchVector.Dimension.INSTANCE_ID, applicationId.serializedForm()).value());
+            this.allowUserFilters = flagValue(flagSource, applicationId, PermanentFlags.ALLOW_USER_FILTERS);
+            this.endpointConnectionTtl = Duration.ofSeconds(flagValue(flagSource, applicationId, PermanentFlags.ENDPOINT_CONNECTION_TTL));
             this.dataplaneTokens = dataplaneTokens;
         }
 
@@ -512,4 +523,10 @@ public class ModelContextImpl implements ModelContext {
         @Override public Duration endpointConnectionTtl() { return endpointConnectionTtl; }
     }
 
+    private static <V> V flagValue(FlagSource source, ApplicationId appId, UnboundFlag<? extends V, ?, ?> flag) {
+        return flag.bindTo(source)
+                .with(FetchVector.Dimension.INSTANCE_ID, appId.serializedForm())
+                .with(FetchVector.Dimension.APPLICATION_ID, appId.toSerializedFormWithoutInstance())
+                .boxedValue();
+    }
 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java
index bd6e0f90b54..f39feceeeb1 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java
@@ -95,9 +95,11 @@ public class ApplicationHandler extends HttpHandler {
         if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}")) return getApplicationResponse(applicationId(path));
         if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/content/{*}")) return content(applicationId(path), path.getRest(), request);
         if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/filedistributionstatus")) return filedistributionStatus(applicationId(path), request);
+        if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/active-token-fingerprints")) return activeTokenFingerprints(applicationId(path));
         if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/logs")) return logs(applicationId(path), request);
         if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/metrics/deployment")) return deploymentMetrics(applicationId(path));
         if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/metrics/searchnode")) return searchNodeMetrics(applicationId(path));
+        if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/quota")) return quotaUsage(applicationId(path));
         if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/reindexing")) return getReindexingStatus(applicationId(path));
         if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/service/{service}/{hostname}/status/{*}")) return serviceStatusPage(applicationId(path), path.get("service"), path.get("hostname"), path.getRest(), request);
         if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/service/{service}/{hostname}/state/v1/{*}")) return serviceStateV1(applicationId(path), path.get("service"), path.get("hostname"), path.getRest(), request);
@@ -105,7 +107,6 @@ public class ApplicationHandler extends HttpHandler {
         if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/serviceconverge/{hostAndPort}")) return checkServiceConverge(applicationId(path), path.get("hostAndPort"), request);
         if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/suspended")) return isSuspended(applicationId(path));
         if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/tester/{command}")) return testerRequest(applicationId(path), path.get("command"), request);
-        if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/quota")) return quotaUsage(applicationId(path));
         return ErrorResponse.notFoundError("Nothing at " + path);
     }
 
@@ -150,18 +151,11 @@ public class ApplicationHandler extends HttpHandler {
     }
 
     private HttpResponse serviceStatusPage(ApplicationId applicationId, String service, String hostname, HttpURL.Path pathSuffix, HttpRequest request) {
-        HttpURL.Path pathPrefix = HttpURL.Path.empty();
-        switch (service) {
-            case "container-clustercontroller":
-                pathPrefix = pathPrefix.append("clustercontroller-status").append("v1");
-                break;
-            case "distributor":
-            case "storagenode":
-                pathPrefix = pathPrefix.append("contentnode-status").append("v1");
-                break;
-            default:
-                throw new com.yahoo.vespa.config.server.NotFoundException("No status page for service: " + service);
-        }
+        HttpURL.Path pathPrefix = switch (service) {
+            case "container-clustercontroller" -> HttpURL.Path.empty().append("clustercontroller-status").append("v1");
+            case "distributor", "storagenode" -> HttpURL.Path.empty().append("contentnode-status").append("v1");
+            default -> throw new NotFoundException("No status page for service: " + service);
+        };
         return applicationRepository.proxyServiceHostnameRequest(applicationId, hostname, service, pathPrefix.append(pathSuffix), Query.empty().add(request.getJDiscRequest().parameters()), null);
     }
 
@@ -194,6 +188,22 @@ public class ApplicationHandler extends HttpHandler {
         return applicationRepository.fileDistributionStatus(applicationId, getTimeoutFromRequest(request));
     }
 
+    private HttpResponse activeTokenFingerprints(ApplicationId applicationId) {
+        Slime slime = new Slime();
+        Cursor hostsArray = slime.setObject().setArray("hosts");
+        applicationRepository.activeTokenFingerprints(applicationId).forEach((host, tokens) -> {
+            Cursor hostObject = hostsArray.addObject();
+            hostObject.setString("host", host);
+            Cursor tokensArray = hostObject.setArray("tokens");
+            tokens.forEach(token -> {
+                Cursor tokenObject = tokensArray.addObject();
+                tokenObject.setString("id", token.id());
+                token.fingerprints().forEach(tokenObject.setArray("fingerprints")::addString);
+            });
+        });
+        return new SlimeJsonResponse(slime);
+    }
+
     private HttpResponse logs(ApplicationId applicationId, HttpRequest request) {
         Optional<DomainName> hostname = Optional.ofNullable(request.getProperty("hostname")).map(DomainName::of);
         String apiParams = Optional.ofNullable(request.getUri().getQuery()).map(q -> "?" + q).orElse("");
@@ -213,19 +223,13 @@ public class ApplicationHandler extends HttpHandler {
     }
 
     private HttpResponse testerRequest(ApplicationId applicationId, String command, HttpRequest request) {
-        switch (command) {
-            case "status":
-                return applicationRepository.getTesterStatus(applicationId);
-            case "log":
-                Long after = Long.valueOf(request.getProperty("after"));
-                return applicationRepository.getTesterLog(applicationId, after);
-            case "ready":
-                return applicationRepository.isTesterReady(applicationId);
-            case "report":
-                return applicationRepository.getTestReport(applicationId);
-            default:
-                throw new IllegalArgumentException("Unknown tester command in request " + request.getUri().toString());
-        }
+        return switch (command) {
+            case "status" -> applicationRepository.getTesterStatus(applicationId);
+            case "log"    -> applicationRepository.getTesterLog(applicationId, Long.valueOf(request.getProperty("after")));
+            case "ready"  -> applicationRepository.isTesterReady(applicationId);
+            case "report" -> applicationRepository.getTestReport(applicationId);
+            default       -> throw new IllegalArgumentException("Unknown tester command in request " + request.getUri().toString());
+        };
     }
 
     private HttpResponse quotaUsage(ApplicationId applicationId) {
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java
index 328bd143d81..d302e0e8008 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java
@@ -9,6 +9,7 @@ import com.yahoo.config.model.api.ConfigDefinitionRepo;
 import com.yahoo.config.model.api.Model;
 import com.yahoo.config.model.api.ModelContext;
 import com.yahoo.config.model.api.ModelFactory;
+import com.yahoo.config.model.api.OnnxModelCost;
 import com.yahoo.config.model.api.Provisioned;
 import com.yahoo.config.model.application.provider.MockFileRegistry;
 import com.yahoo.config.provision.ApplicationId;
@@ -58,6 +59,7 @@ public class ActivatedModelsBuilder extends ModelsBuilder<Application> {
     private final FlagSource flagSource;
     private final SecretStore secretStore;
     private final ExecutorService executor;
+    private final OnnxModelCost onnxModelCost;
 
     public ActivatedModelsBuilder(TenantName tenant,
                                   long applicationGeneration,
@@ -72,7 +74,8 @@ public class ActivatedModelsBuilder extends ModelsBuilder<Application> {
                                   ConfigserverConfig configserverConfig,
                                   Zone zone,
                                   ModelFactoryRegistry modelFactoryRegistry,
-                                  ConfigDefinitionRepo configDefinitionRepo) {
+                                  ConfigDefinitionRepo configDefinitionRepo,
+                                  OnnxModelCost onnxModelCost) {
         super(modelFactoryRegistry, configserverConfig, zone, hostProvisionerProvider, new SilentDeployLogger());
         this.tenant = tenant;
         this.applicationGeneration = applicationGeneration;
@@ -84,6 +87,7 @@ public class ActivatedModelsBuilder extends ModelsBuilder<Application> {
         this.flagSource = flagSource;
         this.secretStore = secretStore;
         this.executor = executor;
+        this.onnxModelCost = onnxModelCost;
     }
 
     @Override
@@ -108,6 +112,7 @@ public class ActivatedModelsBuilder extends ModelsBuilder<Application> {
                 provisioned,
                 modelContextProperties,
                 Optional.empty(),
+                onnxModelCost,
                 wantedDockerImageRepository,
                 modelFactory.version(),
                 wantedNodeVespaVersion);
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java
index 4faa475fa08..57c766bb9c2 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java
@@ -207,11 +207,12 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> {
                 builtModelVersions.add(modelVersion);
             } catch (RuntimeException e) {
                 // allow failure to create old config models if there is a validation override that allow skipping old
-                // config models or we're manually deploying
+                // config models, or we're manually deploying
                 if (builtModelVersions.size() > 0 &&
                     ( builtModelVersions.get(0).getModel().skipOldConfigModels(now) || zone().environment().isManuallyDeployed()))
-                    log.log(Level.INFO, applicationId + ": Failed to build version " + version +
-                                        ", but allow failure due to validation override or manual deployment");
+                    log.log(Level.WARNING, applicationId + ": Failed to build version " + version +
+                            ", but allow failure due to validation override or manual deployment:"
+                            + Exceptions.toMessageString(e));
                 else {
                     log.log(Level.SEVERE, applicationId + ": Failed to build version " + version);
                     throw e;
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java
index af611b131f6..a3f0284890c 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java
@@ -16,6 +16,7 @@ import com.yahoo.config.model.api.Model;
 import com.yahoo.config.model.api.ModelContext;
 import com.yahoo.config.model.api.ModelCreateResult;
 import com.yahoo.config.model.api.ModelFactory;
+import com.yahoo.config.model.api.OnnxModelCost;
 import com.yahoo.config.model.api.Provisioned;
 import com.yahoo.config.model.api.ValidationParameters;
 import com.yahoo.config.model.api.ValidationParameters.IgnoreValidationErrors;
@@ -69,6 +70,7 @@ public class PreparedModelsBuilder extends ModelsBuilder<PreparedModelsBuilder.P
     private final Optional<ApplicationVersions> activeApplicationVersions;
     private final Curator curator;
     private final ExecutorService executor;
+    private final OnnxModelCost onnxModelCost;
 
     public PreparedModelsBuilder(ModelFactoryRegistry modelFactoryRegistry,
                                  FlagSource flagSource,
@@ -85,7 +87,8 @@ public class PreparedModelsBuilder extends ModelsBuilder<PreparedModelsBuilder.P
                                  PrepareParams params,
                                  Optional<ApplicationVersions> activeApplicationVersions,
                                  ConfigserverConfig configserverConfig,
-                                 Zone zone) {
+                                 Zone zone,
+                                 OnnxModelCost onnxModelCost) {
         super(modelFactoryRegistry, configserverConfig, zone, hostProvisionerProvider, deployLogger);
         this.flagSource = flagSource;
         this.secretStore = secretStore;
@@ -98,6 +101,7 @@ public class PreparedModelsBuilder extends ModelsBuilder<PreparedModelsBuilder.P
         this.params = params;
         this.activeApplicationVersions = activeApplicationVersions;
         this.executor = executor;
+        this.onnxModelCost = onnxModelCost;
     }
 
     @Override
@@ -123,6 +127,7 @@ public class PreparedModelsBuilder extends ModelsBuilder<PreparedModelsBuilder.P
                 provisioned,
                 createModelContextProperties(modelFactory.version(), applicationPackage),
                 getAppDir(applicationPackage),
+                onnxModelCost,
                 wantedDockerImageRepository,
                 modelVersion,
                 wantedNodeVespaVersion);
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java
index aeff97169f4..67872865106 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java
@@ -18,6 +18,7 @@ import com.yahoo.config.model.api.ContainerEndpoint;
 import com.yahoo.config.model.api.EndpointCertificateMetadata;
 import com.yahoo.config.model.api.EndpointCertificateSecrets;
 import com.yahoo.config.model.api.FileDistribution;
+import com.yahoo.config.model.api.OnnxModelCost;
 import com.yahoo.config.model.api.Quota;
 import com.yahoo.config.model.api.TenantSecretStore;
 import com.yahoo.config.provision.AllocatedHosts;
@@ -93,6 +94,7 @@ public class SessionPreparer {
     private final FlagSource flagSource;
     private final ExecutorService executor;
     private final BooleanFlag writeSessionData;
+    private final OnnxModelCost onnxModelCost;
 
     public SessionPreparer(ModelFactoryRegistry modelFactoryRegistry,
                            FileDistributionFactory fileDistributionFactory,
@@ -103,7 +105,8 @@ public class SessionPreparer {
                            Curator curator,
                            Zone zone,
                            FlagSource flagSource,
-                           SecretStore secretStore) {
+                           SecretStore secretStore,
+                           OnnxModelCost onnxModelCost) {
         this.modelFactoryRegistry = modelFactoryRegistry;
         this.fileDistributionFactory = fileDistributionFactory;
         this.hostProvisionerProvider = hostProvisionerProvider;
@@ -115,6 +118,7 @@ public class SessionPreparer {
         this.flagSource = flagSource;
         this.executor = executor;
         this.writeSessionData = Flags.WRITE_CONFIG_SERVER_SESSION_DATA_AS_ONE_BLOB.bindTo(flagSource);
+        this.onnxModelCost = onnxModelCost;
     }
 
     ExecutorService getExecutor() { return executor; }
@@ -134,7 +138,8 @@ public class SessionPreparer {
         ApplicationId applicationId = params.getApplicationId();
         Preparation preparation = new Preparation(hostValidator, logger, params, activeApplicationVersions,
                                                   TenantRepository.getTenantPath(applicationId.tenant()),
-                                                  serverDbSessionDir, applicationPackage, sessionZooKeeperClient);
+                                                  serverDbSessionDir, applicationPackage, sessionZooKeeperClient,
+                                                  onnxModelCost);
         preparation.preprocess();
         try {
             AllocatedHosts allocatedHosts = preparation.buildModels(now);
@@ -186,7 +191,7 @@ public class SessionPreparer {
         Preparation(HostValidator hostValidator, DeployLogger logger, PrepareParams params,
                     Optional<ApplicationVersions> activeApplicationVersions, Path tenantPath,
                     File serverDbSessionDir, ApplicationPackage applicationPackage,
-                    SessionZooKeeperClient sessionZooKeeperClient) {
+                    SessionZooKeeperClient sessionZooKeeperClient, OnnxModelCost onnxModelCost) {
             this.logger = logger;
             this.params = params;
             this.applicationPackage = applicationPackage;
@@ -219,7 +224,8 @@ public class SessionPreparer {
                                                                    params,
                                                                    activeApplicationVersions,
                                                                    configserverConfig,
-                                                                   zone);
+                                                                   zone,
+                                                                   onnxModelCost);
         }
 
         void checkTimeout(String step) {
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java
index 3b57945b21d..eb07e3010c6 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java
@@ -9,6 +9,7 @@ import com.yahoo.concurrent.StripedExecutor;
 import com.yahoo.config.application.api.ApplicationPackage;
 import com.yahoo.config.application.api.DeployLogger;
 import com.yahoo.config.model.api.ConfigDefinitionRepo;
+import com.yahoo.config.model.api.OnnxModelCost;
 import com.yahoo.config.model.application.provider.DeployData;
 import com.yahoo.config.model.application.provider.FilesApplicationPackage;
 import com.yahoo.config.provision.ApplicationId;
@@ -118,6 +119,7 @@ public class SessionRepository {
     private final SessionPreparer sessionPreparer;
     private final Path sessionsPath;
     private final TenantName tenantName;
+    private final OnnxModelCost onnxModelCost;
     private final SessionCounter sessionCounter;
     private final SecretStore secretStore;
     private final HostProvisionerProvider hostProvisionerProvider;
@@ -147,8 +149,10 @@ public class SessionRepository {
                              Clock clock,
                              ModelFactoryRegistry modelFactoryRegistry,
                              ConfigDefinitionRepo configDefinitionRepo,
-                             int maxNodeSize) {
+                             int maxNodeSize,
+                             OnnxModelCost onnxModelCost) {
         this.tenantName = tenantName;
+        this.onnxModelCost = onnxModelCost;
         sessionCounter = new SessionCounter(curator, tenantName);
         this.sessionsPath = TenantRepository.getSessionsPath(tenantName);
         this.clock = clock;
@@ -553,7 +557,8 @@ public class SessionRepository {
                                                                     configserverConfig,
                                                                     zone,
                                                                     modelFactoryRegistry,
-                                                                    configDefinitionRepo);
+                                                                    configDefinitionRepo,
+                                                                    onnxModelCost);
         return ApplicationVersions.fromList(builder.buildModels(session.getApplicationId(),
                                                                 session.getDockerImageRepository(),
                                                                 session.getVespaVersion(),
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java
index 2bc8cb5bc0a..378cd9bdb8c 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java
@@ -118,21 +118,21 @@ public class SessionZooKeeperClient {
 
     public long sessionId() { return sessionId; }
 
-    public CompletionWaiter createActiveWaiter() { return createCompletionWaiter(getWaiterPath(ACTIVE_BARRIER)); }
+    public CompletionWaiter createActiveWaiter() { return createCompletionWaiter(barrierPath(ACTIVE_BARRIER)); }
 
-    CompletionWaiter createPrepareWaiter() { return createCompletionWaiter(getWaiterPath(PREPARE_BARRIER)); }
+    CompletionWaiter createPrepareWaiter() { return createCompletionWaiter(barrierPath(PREPARE_BARRIER)); }
 
-    CompletionWaiter getPrepareWaiter() { return getCompletionWaiter(getWaiterPath(PREPARE_BARRIER)); }
+    CompletionWaiter getPrepareWaiter() { return getCompletionWaiter(barrierPath(PREPARE_BARRIER)); }
 
-    CompletionWaiter getActiveWaiter() { return getCompletionWaiter(getWaiterPath(ACTIVE_BARRIER)); }
+    CompletionWaiter getActiveWaiter() { return getCompletionWaiter(barrierPath(ACTIVE_BARRIER)); }
 
-    CompletionWaiter getUploadWaiter() { return getCompletionWaiter(getWaiterPath(UPLOAD_BARRIER)); }
+    CompletionWaiter getUploadWaiter() { return getCompletionWaiter(barrierPath(UPLOAD_BARRIER)); }
 
     private static final String PREPARE_BARRIER = "prepareBarrier";
     private static final String ACTIVE_BARRIER = "activeBarrier";
     private static final String UPLOAD_BARRIER = "uploadBarrier";
 
-    private Path getWaiterPath(String barrierName) {
+    private Path barrierPath(String barrierName) {
         return sessionPath.append(barrierName);
     }
 
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java
index ba09b3de365..ea53c8aa2bb 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java
@@ -11,6 +11,7 @@ import com.yahoo.concurrent.Locks;
 import com.yahoo.concurrent.StripedExecutor;
 import com.yahoo.concurrent.ThreadFactoryFactory;
 import com.yahoo.config.model.api.ConfigDefinitionRepo;
+import com.yahoo.config.model.api.OnnxModelCost;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.TenantName;
 import com.yahoo.config.provision.Zone;
@@ -119,6 +120,7 @@ public class TenantRepository {
             new ScheduledThreadPoolExecutor(1, new DaemonThreadFactory("check for removed applications"));
     private final Curator.DirectoryCache directoryCache;
     private final ZookeeperServerConfig zookeeperServerConfig;
+    private final OnnxModelCost onnxModelCost;
 
     /**
      * Creates a new tenant repository
@@ -138,7 +140,8 @@ public class TenantRepository {
                             ConfigActivationListener configActivationListener,
                             TenantListener tenantListener,
                             ZookeeperServerConfig zookeeperServerConfig,
-                            FileDirectory fileDirectory) {
+                            FileDirectory fileDirectory,
+                            OnnxModelCost onnxModelCost) {
         this(hostRegistry,
              curator,
              metrics,
@@ -157,7 +160,8 @@ public class TenantRepository {
              configDefinitionRepo,
              configActivationListener,
              tenantListener,
-             zookeeperServerConfig);
+             zookeeperServerConfig,
+             onnxModelCost);
     }
 
     public TenantRepository(HostRegistry hostRegistry,
@@ -178,7 +182,8 @@ public class TenantRepository {
                             ConfigDefinitionRepo configDefinitionRepo,
                             ConfigActivationListener configActivationListener,
                             TenantListener tenantListener,
-                            ZookeeperServerConfig zookeeperServerConfig) {
+                            ZookeeperServerConfig zookeeperServerConfig,
+                            OnnxModelCost onnxModelCost) {
         this.hostRegistry = hostRegistry;
         this.configserverConfig = configserverConfig;
         this.curator = curator;
@@ -201,6 +206,7 @@ public class TenantRepository {
         this.zookeeperServerConfig = zookeeperServerConfig;
         // This we should control with a feature flag.
         this.deployHelperExecutor = createModelBuilderExecutor();
+        this.onnxModelCost = onnxModelCost;
 
         curator.framework().getConnectionStateListenable().addListener(this::stateChanged);
 
@@ -353,7 +359,8 @@ public class TenantRepository {
                                                               curator,
                                                               zone,
                                                               flagSource,
-                                                              secretStore);
+                                                              secretStore,
+                                                              onnxModelCost);
         SessionRepository sessionRepository = new SessionRepository(tenantName,
                                                                     applicationRepo,
                                                                     sessionPreparer,
@@ -371,7 +378,8 @@ public class TenantRepository {
                                                                     clock,
                                                                     modelFactoryRegistry,
                                                                     configDefinitionRepo,
-                                                                    zookeeperServerConfig.juteMaxBuffer());
+                                                                    zookeeperServerConfig.juteMaxBuffer(),
+                                                                    onnxModelCost);
         log.log(Level.FINE, "Adding tenant '" + tenantName + "'" + ", created " + created +
                             ". Bootstrapping in " + Duration.between(start, clock.instant()));
         Tenant tenant = new Tenant(tenantName, sessionRepository, applicationRepo, created);
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplication.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplication.java
index 4c262379c35..1288b63cadd 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplication.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplication.java
@@ -111,6 +111,12 @@ public class ZKApplication {
         return getBytesInternal(getFullPath(path));
     }
 
+    public long getSize(Path path) {
+        return curator.getStat(path).map(stat -> (long)stat.getDataLength())
+                .orElseThrow(() -> new IllegalArgumentException(
+                        "Could not get size from '" + path + "' in zookeeper"));
+    }
+
     void putData(Path path, String data) {
         byte[] bytes = Utf8.toBytes(data);
         ensureDataIsNotTooLarge(bytes, path);
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFile.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFile.java
index 6bc29331efb..e51f8627de2 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFile.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFile.java
@@ -3,8 +3,9 @@ package com.yahoo.vespa.config.server.zookeeper;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.yahoo.config.application.api.ApplicationFile;
-import com.yahoo.path.Path;
 import com.yahoo.io.IOUtils;
+import com.yahoo.path.Path;
+import com.yahoo.vespa.config.util.ConfigUtils;
 
 import java.io.ByteArrayInputStream;
 import java.io.FileNotFoundException;
@@ -13,11 +14,9 @@ import java.io.InputStream;
 import java.io.Reader;
 import java.io.StringReader;
 import java.io.StringWriter;
-import java.util.logging.Level;
-import com.yahoo.vespa.config.util.ConfigUtils;
-
 import java.util.ArrayList;
 import java.util.List;
+import java.util.logging.Level;
 import java.util.logging.Logger;
 
 import static com.yahoo.vespa.config.server.zookeeper.ZKApplication.USERAPP_ZK_SUBPATH;
@@ -184,6 +183,8 @@ class ZKApplicationFile extends ApplicationFile {
         }
     }
 
+    @Override public long getSize() { return zkApp.getSize(getZKPath(path)); }
+
     @Override
     public int compareTo(ApplicationFile other) {
         if (other == this) return 0;