Merge branch 'master' into hmusum/throw-exception-when-specifyin-resource-limits

45 files changed, 269 insertions, 71 deletions

diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java
index 195f9dbf8a7..8805c339482 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java
@@ -37,6 +37,7 @@ import com.yahoo.vespa.flags.UnboundFlag;
 
 import java.io.File;
 import java.net.URI;
+import java.security.cert.X509Certificate;
 import java.util.List;
 import java.util.Optional;
 import java.util.Set;
@@ -167,17 +168,18 @@ public class ModelContextImpl implements ModelContext {
         private final boolean skipMbusReplyThread;
         private final boolean useAsyncMessageHandlingOnSchedule;
         private final double feedConcurrency;
-        private final boolean useBucketExecutorForPruneRemoved;
         private final boolean enableFeedBlockInDistributor;
         private final ToIntFunction<ClusterSpec.Type> metricsProxyMaxHeapSizeInMb;
         private final List<String> allowedAthenzProxyIdentities;
-        private final boolean tenantIamRole;
         private final int maxActivationInhibitedOutOfSyncGroups;
         private final ToIntFunction<ClusterSpec.Type> jvmOmitStackTraceInFastThrow;
         private final boolean enableCustomAclMapping;
         private final boolean useExternalRankExpression;
         private final boolean distributeExternalRankExpressions;
         private final int numDistributorStripes;
+        private final boolean requireConnectivityCheck;
+        private final int maxConcurrentMergesPerContentNode;
+        private final int maxMergeQueueSize;
 
         public FeatureFlags(FlagSource source, ApplicationId appId) {
             this.dedicatedClusterControllerFlavor = parseDedicatedClusterControllerFlavor(flagValue(source, appId, Flags.DEDICATED_CLUSTER_CONTROLLER_FLAVOR));
@@ -191,17 +193,18 @@ public class ModelContextImpl implements ModelContext {
             this.skipMbusReplyThread = flagValue(source, appId, Flags.SKIP_MBUS_REPLY_THREAD);
             this.useAsyncMessageHandlingOnSchedule = flagValue(source, appId, Flags.USE_ASYNC_MESSAGE_HANDLING_ON_SCHEDULE);
             this.feedConcurrency = flagValue(source, appId, Flags.FEED_CONCURRENCY);
-            this.useBucketExecutorForPruneRemoved = flagValue(source, appId, Flags.USE_BUCKET_EXECUTOR_FOR_PRUNE_REMOVED);
             this.enableFeedBlockInDistributor = flagValue(source, appId, Flags.ENABLE_FEED_BLOCK_IN_DISTRIBUTOR);
             this.metricsProxyMaxHeapSizeInMb = type -> Flags.METRICS_PROXY_MAX_HEAP_SIZE_IN_MB.bindTo(source).with(CLUSTER_TYPE, type.name()).value();
             this.allowedAthenzProxyIdentities = flagValue(source, appId, Flags.ALLOWED_ATHENZ_PROXY_IDENTITIES);
-            this.tenantIamRole = flagValue(source, appId.tenant(), Flags.TENANT_IAM_ROLE);
             this.maxActivationInhibitedOutOfSyncGroups = flagValue(source, appId, Flags.MAX_ACTIVATION_INHIBITED_OUT_OF_SYNC_GROUPS);
             this.jvmOmitStackTraceInFastThrow = type -> flagValueAsInt(source, appId, type, PermanentFlags.JVM_OMIT_STACK_TRACE_IN_FAST_THROW);
             this.enableCustomAclMapping = flagValue(source, appId, Flags.ENABLE_CUSTOM_ACL_MAPPING);
             this.numDistributorStripes = flagValue(source, appId, Flags.NUM_DISTRIBUTOR_STRIPES);
             this.useExternalRankExpression = flagValue(source, appId, Flags.USE_EXTERNAL_RANK_EXPRESSION);
             this.distributeExternalRankExpressions = flagValue(source, appId, Flags.DISTRIBUTE_EXTERNAL_RANK_EXPRESSION);
+            this.requireConnectivityCheck = flagValue(source, appId, Flags.REQUIRE_CONNECTIVITY_CHECK);
+            this.maxConcurrentMergesPerContentNode = flagValue(source, appId, Flags.MAX_CONCURRENT_MERGES_PER_NODE);
+            this.maxMergeQueueSize = flagValue(source, appId, Flags.MAX_MERGE_QUEUE_SIZE);
         }
 
         @Override public Optional<NodeResources> dedicatedClusterControllerFlavor() { return Optional.ofNullable(dedicatedClusterControllerFlavor); }
@@ -215,11 +218,9 @@ public class ModelContextImpl implements ModelContext {
         @Override public boolean skipMbusReplyThread() { return skipMbusReplyThread; }
         @Override public boolean useAsyncMessageHandlingOnSchedule() { return useAsyncMessageHandlingOnSchedule; }
         @Override public double feedConcurrency() { return feedConcurrency; }
-        @Override public boolean useBucketExecutorForPruneRemoved() { return useBucketExecutorForPruneRemoved; }
         @Override public boolean enableFeedBlockInDistributor() { return enableFeedBlockInDistributor; }
         @Override public int metricsProxyMaxHeapSizeInMb(ClusterSpec.Type type) { return metricsProxyMaxHeapSizeInMb.applyAsInt(type); }
         @Override public List<String> allowedAthenzProxyIdentities() { return allowedAthenzProxyIdentities; }
-        @Override public boolean tenantIamRole() { return tenantIamRole; }
         @Override public int maxActivationInhibitedOutOfSyncGroups() { return maxActivationInhibitedOutOfSyncGroups; }
         @Override public String jvmOmitStackTraceInFastThrowOption(ClusterSpec.Type type) {
             return translateJvmOmitStackTraceInFastThrowIntToString(jvmOmitStackTraceInFastThrow, type);
@@ -228,6 +229,9 @@ public class ModelContextImpl implements ModelContext {
         @Override public int numDistributorStripes() { return numDistributorStripes; }
         @Override public boolean useExternalRankExpressions() { return useExternalRankExpression; }
         @Override public boolean distributeExternalRankExpressions() { return distributeExternalRankExpressions; }
+        @Override public boolean requireConnectivityCheck() { return requireConnectivityCheck; }
+        @Override public int maxConcurrentMergesPerNode() { return maxConcurrentMergesPerContentNode; }
+        @Override public int maxMergeQueueSize() { return maxMergeQueueSize; }
 
         private static <V> V flagValue(FlagSource source, ApplicationId appId, UnboundFlag<? extends V, ?, ?> flag) {
             return flag.bindTo(source)
@@ -287,6 +291,7 @@ public class ModelContextImpl implements ModelContext {
         private final SecretStore secretStore;
         private final StringFlag jvmGCOptionsFlag;
         private final boolean allowDisableMtls;
+        private final List<X509Certificate> operatorCertificates;
 
         public Properties(ApplicationId applicationId,
                           ConfigserverConfig configserverConfig,
@@ -300,7 +305,8 @@ public class ModelContextImpl implements ModelContext {
                           Optional<ApplicationRoles> applicationRoles,
                           Optional<Quota> maybeQuota,
                           List<TenantSecretStore> tenantSecretStores,
-                          SecretStore secretStore) {
+                          SecretStore secretStore,
+                          List<X509Certificate> operatorCertificates) {
             this.featureFlags = new FeatureFlags(flagSource, applicationId);
             this.applicationId = applicationId;
             this.multitenant = configserverConfig.multitenant() || configserverConfig.hostedVespa() || Boolean.getBoolean("multitenant");
@@ -323,6 +329,7 @@ public class ModelContextImpl implements ModelContext {
                                                                  .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm());
             this.allowDisableMtls = PermanentFlags.ALLOW_DISABLE_MTLS.bindTo(flagSource)
                     .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value();
+            this.operatorCertificates = operatorCertificates;
         }
 
         @Override public ModelContext.FeatureFlags featureFlags() { return featureFlags; }
@@ -391,6 +398,11 @@ public class ModelContextImpl implements ModelContext {
             return allowDisableMtls;
         }
 
+        @Override
+        public List<X509Certificate> operatorCertificates() {
+            return operatorCertificates;
+        }
+
         public String flagValueForClusterType(StringFlag flag, Optional<ClusterSpec.Type> clusterType) {
             return clusterType.map(type -> flag.with(CLUSTER_TYPE, type.name()))
                               .orElse(flag)
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java
index 1ab99db8985..f0a63757477 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java
@@ -35,7 +35,7 @@ import static com.yahoo.vespa.config.server.zookeeper.ConfigCurator.USERAPP_ZK_S
 import static com.yahoo.vespa.config.server.zookeeper.ConfigCurator.USER_DEFCONFIGS_ZK_SUBPATH;
 
 /**
- * A class used for reading and writing application data to zookeeper.
+ * Reads and writes application package to and from ZooKeeper.
  *
  * @author hmusum
  */
@@ -76,13 +76,13 @@ public class ZooKeeperClient {
      *
      * @param app the application package to feed to zookeeper
      */
-    void write(ApplicationPackage app) {
+    void writeApplicationPackage(ApplicationPackage app) {
         try {
             writeUserDefs(app);
             writeSomeOf(app);
-            writeSearchDefinitions(app);
+            writeSchemas(app);
             writeUserIncludeDirs(app, app.getUserIncludeDirs());
-            write(app.getMetaData());
+            writeMetadata(app.getMetaData());
         } catch (Exception e) {
             throw new IllegalStateException("Unable to write vespa model to config server(s) " + System.getProperty("configsources") + "\n" +
                                             "Please ensure that config server is started " +
@@ -90,13 +90,11 @@ public class ZooKeeperClient {
         }
     }
 
-    private void writeSearchDefinitions(ApplicationPackage app) throws IOException {
-        Collection<NamedReader> sds = app.getSearchDefinitions();
+    private void writeSchemas(ApplicationPackage app) throws IOException {
+        Collection<NamedReader> sds = app.getSchemas();
         if (sds.isEmpty()) return;
 
-        // TODO: Change to SCHEMAS_DIR after March 2020
-        // TODO: When it does also check RankExpressionFile.sendTo
-        Path zkPath = getZooKeeperAppPath(USERAPP_ZK_SUBPATH).append(ApplicationPackage.SEARCH_DEFINITIONS_DIR);
+        Path zkPath = getZooKeeperAppPath(USERAPP_ZK_SUBPATH).append(SCHEMAS_DIR);
         configCurator.createNode(zkPath.getAbsolute());
         // Ensures that ranking expressions and other files are also written
         writeDir(app.getFile(ApplicationPackage.SEARCH_DEFINITIONS_DIR), zkPath, false);
@@ -155,7 +153,6 @@ public class ZooKeeperClient {
         for (ApplicationFile file : listFiles(dir, filenameFilter)) {
             String name = file.getPath().getName();
             if (name.startsWith(".")) continue; //.svn , .git ...
-            if ("CVS".equals(name)) continue;
             if (file.isDirectory()) {
                 configCurator.createNode(path.append(name).getAbsolute());
                 if (recurse) {
@@ -200,7 +197,6 @@ public class ZooKeeperClient {
     }
 
     private void writeUserIncludeDirs(ApplicationPackage applicationPackage, List<String> userIncludeDirs) throws IOException {
-        // User defined include directories
         for (String userInclude : userIncludeDirs) {
             ApplicationFile dir = applicationPackage.getFile(Path.fromString(userInclude));
             final List<ApplicationFile> files = dir.listFiles();
@@ -240,12 +236,12 @@ public class ZooKeeperClient {
     }
 
     /**
-     * Feeds application metadata to zookeeper. Used by vespamodel to create config
-     * for application metadata (used by ApplicationStatusHandler)
+     * Feeds application metadata to zookeeper. Used by config model to create config
+     * for application metadata
      *
      * @param metaData The application metadata.
      */
-    private void write(ApplicationMetaData metaData) {
+    private void writeMetadata(ApplicationMetaData metaData) {
         configCurator.putData(getZooKeeperAppPath(META_ZK_PATH).getAbsolute(), metaData.asJsonBytes());
     }
 
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperDeployer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperDeployer.java
index 12aa5b7cc35..8c7d6ea28dd 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperDeployer.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperDeployer.java
@@ -33,7 +33,7 @@ public class ZooKeeperDeployer {
     public void deploy(ApplicationPackage applicationPackage, Map<Version, FileRegistry> fileRegistryMap, 
                        AllocatedHosts allocatedHosts) throws IOException {
         zooKeeperClient.initialize();
-        zooKeeperClient.write(applicationPackage);
+        zooKeeperClient.writeApplicationPackage(applicationPackage);
         zooKeeperClient.write(fileRegistryMap);
         zooKeeperClient.write(allocatedHosts);
     }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java
index 5b520b10fcf..dfbce72d4ba 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java
@@ -82,7 +82,7 @@ public class ApplicationApiHandler extends SessionHandler {
                         .collect(Collectors.toMap(Part::getName, p -> p));
 
                 byte[] params = parts.get(MULTIPART_PARAMS).getInputStream().readAllBytes();
-                log.log(Level.FINE, "Deploy parameters: [{}]", new String(params, StandardCharsets.UTF_8));
+                log.log(Level.FINE, "Deploy parameters: [{0}]", new String(params, StandardCharsets.UTF_8));
                 prepareParams = PrepareParams.fromJson(params, tenantName, zookeeperBarrierTimeout);
                 Part appPackagePart = parts.get(MULTIPART_APPLICATION_PACKAGE);
                 compressedStream = createFromCompressedStream(appPackagePart.getInputStream(), appPackagePart.getContentType());
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java
index 5519ffc1bdc..003b4fbb345 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java
@@ -48,8 +48,9 @@ public class ApplicationPackageMaintainer extends ConfigServerMaintainer {
     }
 
     @Override
-    protected boolean maintain() {
-        boolean success = true;
+    protected double maintain() {
+        int attempts = 0;
+        int failures = 0;
 
         try (var fileDownloader = new FileDownloader(connectionPool, downloadDirectory)) {
             for (var applicationId : applicationRepository.listApplications()) {
@@ -62,11 +63,12 @@ public class ApplicationPackageMaintainer extends ConfigServerMaintainer {
                 log.fine(() -> "Verifying application package file reference " + applicationPackage + " for session " + sessionId);
 
                 if (applicationPackage != null) {
+                    attempts++;
                     if (! fileReferenceExistsOnDisk(downloadDirectory, applicationPackage)) {
                         log.fine(() -> "Downloading missing application package for application " + applicationId + " - session " + sessionId);
 
                         if (fileDownloader.getFile(applicationPackage).isEmpty()) {
-                            success = false;
+                            failures++;
                             log.warning("Failed to download application package for application " + applicationId + " - session " + sessionId);
                             continue;
                         }
@@ -75,7 +77,7 @@ public class ApplicationPackageMaintainer extends ConfigServerMaintainer {
                 }
             }
         }
-        return success;
+        return asSuccessFactor(attempts, failures);
     }
 
     @Override
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintainer.java
index 4938f34131e..e0f0a4b4099 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintainer.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintainer.java
@@ -35,14 +35,24 @@ public abstract class ConfigServerMaintainer extends Maintainer {
     ConfigServerMaintainer(ApplicationRepository applicationRepository, Curator curator, FlagSource flagSource,
                            Instant now, Duration interval) {
         super(null, interval, now, new JobControl(new JobControlFlags(curator, flagSource)),
-              jobMetrics(applicationRepository.metric()), cluster(curator), false);
+              new ConfigServerJobMetrics(applicationRepository.metric()), cluster(curator), false);
         this.applicationRepository = applicationRepository;
     }
 
-    private static JobMetrics jobMetrics(Metric metric) {
-        return new JobMetrics((job, consecutiveFailures) -> {
+    private static class ConfigServerJobMetrics extends JobMetrics {
+
+        private final Metric metric;
+
+        public ConfigServerJobMetrics(Metric metric) {
+            this.metric = metric;
+        }
+
+        @Override
+        protected void recordCompletion(String job, Long consecutiveFailures, double successFactor) {
             metric.set("maintenance.consecutiveFailures", consecutiveFailures, metric.createContext(Map.of("job", job)));
-        });
+            metric.set("maintenance.successFactor", successFactor, metric.createContext(Map.of("job", job)));
+        }
+
     }
 
     private static class JobControlFlags implements JobControlState {
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java
index b0876fb57e8..ca8db30c21f 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java
@@ -33,9 +33,9 @@ public class FileDistributionMaintainer extends ConfigServerMaintainer {
     }
 
     @Override
-    protected boolean maintain() {
+    protected double maintain() {
         applicationRepository.deleteUnusedFiledistributionReferences(fileReferencesDir, maxUnusedFileReferenceAge);
-        return true;
+        return 1.0;
     }
 
 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ReindexingMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ReindexingMaintainer.java
index 971c2c20ae9..af9ea917aaf 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ReindexingMaintainer.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ReindexingMaintainer.java
@@ -22,6 +22,7 @@ import java.util.Comparator;
 import java.util.Map;
 import java.util.Optional;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.function.Supplier;
 import java.util.logging.Level;
@@ -51,8 +52,9 @@ public class ReindexingMaintainer extends ConfigServerMaintainer {
     }
 
     @Override
-    protected boolean maintain() {
-        AtomicBoolean success = new AtomicBoolean(true);
+    protected double maintain() {
+        AtomicInteger attempts = new AtomicInteger(0);
+        AtomicInteger failures = new AtomicInteger(0);
         for (Tenant tenant : applicationRepository.tenantRepository().getAllTenants()) {
             ApplicationCuratorDatabase database = tenant.getApplicationRepo().database();
             for (ApplicationId id : database.activeApplications())
@@ -60,6 +62,7 @@ public class ReindexingMaintainer extends ConfigServerMaintainer {
                                      .map(application -> application.getForVersionOrLatest(Optional.empty(), clock.instant()))
                                      .ifPresent(application -> {
                                          try {
+                                             attempts.incrementAndGet();
                                              applicationRepository.modifyReindexing(id, reindexing -> {
                                                  reindexing = withNewReady(reindexing, lazyGeneration(application), clock.instant());
                                                  reindexing = withOnlyCurrentData(reindexing, application);
@@ -68,11 +71,11 @@ public class ReindexingMaintainer extends ConfigServerMaintainer {
                                          }
                                          catch (RuntimeException e) {
                                              log.log(Level.INFO, "Failed to update reindexing status for " + id + ": " + Exceptions.toMessageString(e));
-                                             success.set(false);
+                                             failures.incrementAndGet();
                                          }
                                      });
         }
-        return success.get();
+        return asSuccessFactor(attempts.get(), failures.get());
     }
 
     private Supplier<Long> lazyGeneration(Application application) {
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/SessionsMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/SessionsMaintainer.java
index 7482980e221..1f85dd4579d 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/SessionsMaintainer.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/SessionsMaintainer.java
@@ -25,7 +25,7 @@ public class SessionsMaintainer extends ConfigServerMaintainer {
     }
 
     @Override
-    protected boolean maintain() {
+    protected double maintain() {
         if (iteration % 10 == 0)
             log.log(Level.INFO, () -> "Running " + SessionsMaintainer.class.getSimpleName() + ", iteration "  + iteration);
 
@@ -38,7 +38,7 @@ public class SessionsMaintainer extends ConfigServerMaintainer {
         }
 
         iteration++;
-        return true;
+        return 1.0;
     }
 
 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainer.java
index 7c01045ee72..0a7df2c9d21 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainer.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainer.java
@@ -31,12 +31,12 @@ public class TenantsMaintainer extends ConfigServerMaintainer {
     }
 
     @Override
-    protected boolean maintain() {
-        if ( ! applicationRepository.configserverConfig().hostedVespa()) return true;
+    protected double maintain() {
+        if ( ! applicationRepository.configserverConfig().hostedVespa()) return 1.0;
 
         Set<TenantName> tenants = applicationRepository.deleteUnusedTenants(ttlForUnusedTenant, clock.instant());
         if (tenants.size() > 0) log.log(Level.INFO, "Deleted tenants " + tenants);
-        return true;
+        return 1.0;
     }
 
 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java
index 4e7afa7b3db..2b7ce234777 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java
@@ -38,7 +38,9 @@ import com.yahoo.vespa.config.server.tenant.TenantRepository;
 import com.yahoo.vespa.curator.Curator;
 import com.yahoo.vespa.flags.FlagSource;
 
+import java.security.cert.X509Certificate;
 import java.util.Comparator;
+import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.logging.Level;
@@ -164,7 +166,8 @@ public class ActivatedModelsBuilder extends ModelsBuilder<Application> {
                                                        .readApplicationRoles(applicationId),
                                                zkClient.readQuota(),
                                                zkClient.readTenantSecretStores(),
-                                               secretStore);
+                                               secretStore,
+                                               zkClient.readOperatorCertificates());
     }
 
 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java
index 707a828bdd6..2d4aa78bcf6 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java
@@ -1,7 +1,6 @@
 // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.config.server.modelfactory;
 
-import com.google.common.util.concurrent.UncheckedTimeoutException;
 import com.yahoo.cloud.config.ConfigserverConfig;
 import com.yahoo.component.Version;
 import com.yahoo.config.application.api.ApplicationPackage;
@@ -32,7 +31,6 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.List;
-import java.util.NoSuchElementException;
 import java.util.Optional;
 import java.util.Set;
 import java.util.logging.Level;
@@ -224,8 +222,7 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> {
     }
 
     private Set<Version> versionsToBuild(Set<Version> versions, Version wantedVersion, int majorVersion, AllocatedHosts allocatedHosts) {
-        if (configserverConfig.buildMinimalSetOfConfigModels())
-            versions = keepThoseUsedOn(allocatedHosts, versions);
+        versions = keepThoseUsedOn(allocatedHosts, versions);
 
         // Make sure we build wanted version if we are building models for this major version and we are on hosted vespa
         // If not on hosted vespa, we do not want to try to build this version, since we have only one version (the latest)
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java
index ea2a525b440..071a0dd8f0c 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java
@@ -1,6 +1,7 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.config.server.session;
 
+import com.google.common.collect.ImmutableList;
 import com.yahoo.component.Version;
 import com.yahoo.config.model.api.ApplicationRoles;
 import com.yahoo.config.model.api.ContainerEndpoint;
@@ -11,6 +12,8 @@ import com.yahoo.config.provision.AthenzDomain;
 import com.yahoo.config.provision.DockerImage;
 import com.yahoo.config.provision.TenantName;
 import com.yahoo.container.jdisc.HttpRequest;
+import com.yahoo.security.X509CertificateUtils;
+import com.yahoo.slime.ArrayTraverser;
 import com.yahoo.slime.Inspector;
 import com.yahoo.slime.Slime;
 import com.yahoo.slime.SlimeUtils;
@@ -20,14 +23,18 @@ import com.yahoo.vespa.config.server.http.SessionHandler;
 import com.yahoo.vespa.config.server.tenant.ContainerEndpointSerializer;
 import com.yahoo.vespa.config.server.tenant.EndpointCertificateMetadataSerializer;
 import com.yahoo.vespa.config.server.tenant.TenantSecretStoreSerializer;
+import org.eclipse.jetty.util.ssl.X509;
 
+import java.security.cert.X509Certificate;
 import java.time.Clock;
 import java.time.Duration;
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.function.Function;
+import java.util.stream.Collectors;
 
 /**
  * Parameters for preparing an application. Immutable.
@@ -52,6 +59,7 @@ public final class PrepareParams {
     static final String TENANT_SECRET_STORES_PARAM_NAME = "tenantSecretStores";
     static final String FORCE_PARAM_NAME = "force";
     static final String WAIT_FOR_RESOURCES_IN_PREPARE = "waitForResourcesInPrepare";
+    static final String OPERATOR_CERTIFICATES = "operatorCertificates";
 
     private final ApplicationId applicationId;
     private final TimeoutBudget timeoutBudget;
@@ -69,6 +77,7 @@ public final class PrepareParams {
     private final Optional<ApplicationRoles> applicationRoles;
     private final Optional<Quota> quota;
     private final List<TenantSecretStore> tenantSecretStores;
+    private final List<X509Certificate> operatorCertificates;
 
     private PrepareParams(ApplicationId applicationId, TimeoutBudget timeoutBudget, boolean ignoreValidationErrors,
                           boolean dryRun, boolean verbose, boolean isBootstrap, Optional<Version> vespaVersion,
@@ -76,7 +85,7 @@ public final class PrepareParams {
                           Optional<EndpointCertificateMetadata> endpointCertificateMetadata,
                           Optional<DockerImage> dockerImageRepository, Optional<AthenzDomain> athenzDomain,
                           Optional<ApplicationRoles> applicationRoles, Optional<Quota> quota, List<TenantSecretStore> tenantSecretStores,
-                          boolean force, boolean waitForResourcesInPrepare) {
+                          boolean force, boolean waitForResourcesInPrepare, List<X509Certificate> operatorCertificates) {
         this.timeoutBudget = timeoutBudget;
         this.applicationId = Objects.requireNonNull(applicationId);
         this.ignoreValidationErrors = ignoreValidationErrors;
@@ -93,6 +102,7 @@ public final class PrepareParams {
         this.tenantSecretStores = tenantSecretStores;
         this.force = force;
         this.waitForResourcesInPrepare = waitForResourcesInPrepare;
+        this.operatorCertificates = operatorCertificates;
     }
 
     public static class Builder {
@@ -113,6 +123,7 @@ public final class PrepareParams {
         private Optional<ApplicationRoles> applicationRoles = Optional.empty();
         private Optional<Quota> quota = Optional.empty();
         private List<TenantSecretStore> tenantSecretStores = List.of();
+        private List<X509Certificate> operatorCertificates = List.of();
 
         public Builder() { }
 
@@ -245,11 +256,17 @@ public final class PrepareParams {
             return this;
         }
 
+        public Builder withOperatorCertificates(List<X509Certificate> operatorCertificates) {
+            this.operatorCertificates = List.copyOf(operatorCertificates);
+            return this;
+        }
+
         public PrepareParams build() {
             return new PrepareParams(applicationId, timeoutBudget, ignoreValidationErrors, dryRun,
                                      verbose, isBootstrap, vespaVersion, containerEndpoints,
                                      endpointCertificateMetadata, dockerImageRepository, athenzDomain,
-                                     applicationRoles, quota, tenantSecretStores, force, waitForResourcesInPrepare);
+                                     applicationRoles, quota, tenantSecretStores, force, waitForResourcesInPrepare,
+                                     operatorCertificates);
         }
     }
 
@@ -289,9 +306,10 @@ public final class PrepareParams {
                 .athenzDomain(SlimeUtils.optionalString(params.field(ATHENZ_DOMAIN)).orElse(null))
                 .applicationRoles(ApplicationRoles.fromString(SlimeUtils.optionalString(params.field(APPLICATION_HOST_ROLE)).orElse(null), SlimeUtils.optionalString(params.field(APPLICATION_CONTAINER_ROLE)).orElse(null)))
                 .quota(deserialize(params.field(QUOTA_PARAM_NAME), Quota::fromSlime))
-                .tenantSecretStores(SlimeUtils.optionalString(params.field(TENANT_SECRET_STORES_PARAM_NAME)).orElse(null))
+                .tenantSecretStores(deserialize(params.field(TENANT_SECRET_STORES_PARAM_NAME), TenantSecretStoreSerializer::listFromSlime, List.of()))
                 .force(booleanValue(params, FORCE_PARAM_NAME))
                 .waitForResourcesInPrepare(booleanValue(params, WAIT_FOR_RESOURCES_IN_PREPARE))
+                .withOperatorCertificates(deserialize(params.field(OPERATOR_CERTIFICATES), PrepareParams::readOperatorCertificates, Collections.emptyList()))
                 .build();
     }
 
@@ -343,6 +361,13 @@ public final class PrepareParams {
         return Optional.ofNullable(request.getProperty(propertyName));
     }
 
+    private static List<X509Certificate> readOperatorCertificates(Inspector array) {
+        return SlimeUtils.entriesStream(array)
+                .map(Inspector::asString)
+                .map(X509CertificateUtils::fromPem)
+                .collect(Collectors.toList());
+    }
+
     public String getApplicationName() {
         return applicationId.application().value();
     }
@@ -400,4 +425,8 @@ public final class PrepareParams {
     public List<TenantSecretStore> tenantSecretStores() {
         return tenantSecretStores;
     }
+
+    public List<X509Certificate> operatorCertificates() {
+        return operatorCertificates;
+    }
 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java
index f1044b28049..542b54d877e 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java
@@ -17,6 +17,7 @@ import com.yahoo.transaction.Transaction;
 import com.yahoo.vespa.config.server.application.ApplicationSet;
 import com.yahoo.vespa.config.server.tenant.TenantRepository;
 
+import java.security.cert.X509Certificate;
 import java.time.Instant;
 import java.util.List;
 import java.util.Optional;
@@ -137,6 +138,10 @@ public abstract class Session implements Comparable<Session>  {
         sessionZooKeeperClient.writeTenantSecretStores(tenantSecretStores);
     }
 
+    public void setOperatorCertificates(List<X509Certificate> operatorCertificates) {
+        sessionZooKeeperClient.writeOperatorCertificates(operatorCertificates);
+    }
+
     /** Returns application id read from ZooKeeper. Will throw RuntimeException if not found */
     public ApplicationId getApplicationId() {
         return sessionZooKeeperClient.readApplicationId()
@@ -172,6 +177,10 @@ public abstract class Session implements Comparable<Session>  {
         return sessionZooKeeperClient.readTenantSecretStores();
     }
 
+    public List<X509Certificate> getOperatorCertificates() {
+        return sessionZooKeeperClient.readOperatorCertificates();
+    }
+
     private Transaction createSetStatusTransaction(Status status) {
         return sessionZooKeeperClient.createWriteStatusTransaction(status);
     }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java
index 50b9ac55bda..dedd9e08655 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java
@@ -49,6 +49,7 @@ import com.yahoo.vespa.flags.FlagSource;
 
 import java.io.File;
 import java.io.IOException;
+import java.security.cert.X509Certificate;
 import java.time.Instant;
 import java.util.Collection;
 import java.util.List;
@@ -202,7 +203,8 @@ public class SessionPreparer {
                                                               applicationRoles,
                                                               params.quota(),
                                                               params.tenantSecretStores(),
-                                                              secretStore);
+                                                              secretStore,
+                                                              params.operatorCertificates());
             this.fileDistributionProvider = fileDistributionFactory.createProvider(serverDbSessionDir);
             this.preparedModelsBuilder = new PreparedModelsBuilder(modelFactoryRegistry,
                                                                    permanentApplicationPackage,
@@ -275,7 +277,8 @@ public class SessionPreparer {
                                   prepareResult.allocatedHosts(),
                                   athenzDomain,
                                   params.quota(),
-                                  params.tenantSecretStores());
+                                  params.tenantSecretStores(),
+                                  params.operatorCertificates());
             checkTimeout("write state to zookeeper");
         }
 
@@ -325,7 +328,8 @@ public class SessionPreparer {
                                        AllocatedHosts allocatedHosts,
                                        Optional<AthenzDomain> athenzDomain,
                                        Optional<Quota> quota,
-                                       List<TenantSecretStore> tenantSecretStores) {
+                                       List<TenantSecretStore> tenantSecretStores,
+                                       List<X509Certificate> operatorCertificates) {
         ZooKeeperDeployer zkDeployer = zooKeeperClient.createDeployer(deployLogger);
         try {
             zkDeployer.deploy(applicationPackage, fileRegistryMap, allocatedHosts);
@@ -337,6 +341,7 @@ public class SessionPreparer {
             zooKeeperClient.writeAthenzDomain(athenzDomain);
             zooKeeperClient.writeQuota(quota);
             zooKeeperClient.writeTenantSecretStores(tenantSecretStores);
+            zooKeeperClient.writeOperatorCertificates(operatorCertificates);
         } catch (RuntimeException | IOException e) {
             zkDeployer.cleanup();
             throw new RuntimeException("Error preparing session", e);
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java
index 7d74b53fdff..ac350db5c21 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java
@@ -256,6 +256,7 @@ public class SessionRepository {
         session.setDockerImageRepository(existingSession.getDockerImageRepository());
         session.setAthenzDomain(existingSession.getAthenzDomain());
         session.setTenantSecretStores(existingSession.getTenantSecretStores());
+        session.setOperatorCertificates(existingSession.getOperatorCertificates());
         return session;
     }
 
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java
index c7c4f1926d7..c3d6bba0ac2 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java
@@ -21,6 +21,7 @@ import com.yahoo.transaction.Transaction;
 import com.yahoo.vespa.config.server.UserConfigDefinitionRepo;
 import com.yahoo.vespa.config.server.deploy.ZooKeeperClient;
 import com.yahoo.vespa.config.server.deploy.ZooKeeperDeployer;
+import com.yahoo.vespa.config.server.tenant.OperatorCertificateSerializer;
 import com.yahoo.vespa.config.server.tenant.TenantRepository;
 import com.yahoo.vespa.config.server.tenant.TenantSecretStoreSerializer;
 import com.yahoo.vespa.config.server.zookeeper.ConfigCurator;
@@ -29,6 +30,7 @@ import com.yahoo.vespa.curator.Curator;
 import com.yahoo.vespa.curator.transaction.CuratorOperations;
 import com.yahoo.vespa.curator.transaction.CuratorTransaction;
 
+import java.security.cert.X509Certificate;
 import java.time.Instant;
 import java.util.List;
 import java.util.Optional;
@@ -57,6 +59,7 @@ public class SessionZooKeeperClient {
     private static final String ATHENZ_DOMAIN = "athenzDomain";
     private static final String QUOTA_PATH = "quota";
     private static final String TENANT_SECRET_STORES_PATH = "tenantSecretStores";
+    private static final String OPERATOR_CERTIFICATES_PATH = "operatorCertificates";
 
     private final Curator curator;
     private final ConfigCurator configCurator;
@@ -191,6 +194,10 @@ public class SessionZooKeeperClient {
         return sessionPath.append(TENANT_SECRET_STORES_PATH).getAbsolute();
     }
 
+    private String operatorCertificatesPath() {
+        return sessionPath.append(OPERATOR_CERTIFICATES_PATH).getAbsolute();
+    }
+
     public void writeVespaVersion(Version version) {
         configCurator.putData(versionPath(), version.toString());
     }
@@ -282,6 +289,21 @@ public class SessionZooKeeperClient {
                 .orElse(List.of());
     }
 
+    public void writeOperatorCertificates(List<X509Certificate> certificates) {
+        if( ! certificates.isEmpty()) {
+            var bytes = uncheck(() -> SlimeUtils.toJsonBytes(OperatorCertificateSerializer.toSlime(certificates)));
+            configCurator.putData(operatorCertificatesPath(), bytes);
+        }
+    }
+
+    public List<X509Certificate> readOperatorCertificates() {
+        if ( ! configCurator.exists(operatorCertificatesPath())) return List.of();
+        return Optional.ofNullable(configCurator.getData(operatorCertificatesPath()))
+                .map(SlimeUtils::jsonToSlime)
+                .map(slime -> OperatorCertificateSerializer.fromSlime(slime.get()))
+                .orElse(List.of());
+    }
+
     /**
      * Create necessary paths atomically for a new session.
      *
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/OperatorCertificateSerializer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/OperatorCertificateSerializer.java
new file mode 100644
index 00000000000..3dbdf1380f1
--- /dev/null
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/OperatorCertificateSerializer.java
@@ -0,0 +1,37 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.vespa.config.server.tenant;
+
+import com.yahoo.config.model.api.ApplicationRoles;
+import com.yahoo.security.X509CertificateUtils;
+import com.yahoo.slime.Cursor;
+import com.yahoo.slime.Inspector;
+import com.yahoo.slime.Slime;
+import com.yahoo.slime.SlimeUtils;
+
+import java.security.cert.X509Certificate;
+import java.util.List;
+import java.util.stream.Collectors;
+
+public class OperatorCertificateSerializer {
+
+    private final static String certificateField = "certificates";
+
+
+    public static Slime toSlime(List<X509Certificate> certificateList) {
+        Slime slime = new Slime();
+        var root = slime.setObject();
+        Cursor array = root.setArray(certificateField);
+        certificateList.stream()
+                .map(X509CertificateUtils::toPem)
+                .forEach(array::addString);
+        return slime;
+    }
+
+    public static List<X509Certificate> fromSlime(Inspector object) {
+        return SlimeUtils.entriesStream(object.field(certificateField))
+                .map(Inspector::asString)
+                .map(X509CertificateUtils::fromPem)
+                .collect(Collectors.toList());
+    }
+}
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java
index f01bf1fe6ef..9f1bfa0b4e4 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java
@@ -132,16 +132,17 @@ public class ZKApplicationPackage implements ApplicationPackage {
     }
 
     @Override
-    public List<NamedReader> searchDefinitionContents() {
+    public List<NamedReader> getSchemas() {
         List<NamedReader> schemas = new ArrayList<>();
-        for (String sd : zkApplication.getChildren(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SEARCH_DEFINITIONS_DIR)) {
-            if (sd.endsWith(SD_NAME_SUFFIX))
-                schemas.add(new NamedReader(sd, new StringReader(zkApplication.getData(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SEARCH_DEFINITIONS_DIR, sd))));
-        }
         for (String sd : zkApplication.getChildren(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SCHEMAS_DIR)) {
             if (sd.endsWith(SD_NAME_SUFFIX))
                 schemas.add(new NamedReader(sd, new StringReader(zkApplication.getData(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SCHEMAS_DIR, sd))));
         }
+        // TODO: Remove when 7.414.19 is oldest version in use
+        for (String sd : zkApplication.getChildren(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SEARCH_DEFINITIONS_DIR)) {
+            if (sd.endsWith(SD_NAME_SUFFIX))
+                schemas.add(new NamedReader(sd, new StringReader(zkApplication.getData(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SEARCH_DEFINITIONS_DIR, sd))));
+        }
         return schemas;
     }
 
@@ -164,11 +165,6 @@ public class ZKApplicationPackage implements ApplicationPackage {
         return fileRegistry;
     }
 
-    @Override
-    public List<NamedReader> getSearchDefinitions() {
-        return searchDefinitionContents();
-    }
-
     private Reader retrieveConfigDefReader(String def) {
         try {
             return zkApplication.getDataReader(ConfigCurator.DEFCONFIGS_ZK_SUBPATH, def);
@@ -262,6 +258,7 @@ public class ZKApplicationPackage implements ApplicationPackage {
     @Override
     public Reader getRankingExpression(String name) {
         Optional<Reader> reader = zkApplication.getOptionalDataReader(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SCHEMAS_DIR, name);
+        // TODO: Remove when 7.414.19 is oldest version in use
         return reader.orElseGet(() -> zkApplication.getDataReader(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SEARCH_DEFINITIONS_DIR, name));
     }
 
diff --git a/configserver/src/test/apps/app-jdisc-only-restart/searchdefinitions/music.sd b/configserver/src/test/apps/app-jdisc-only-restart/schemas/music.sd
index a2d4614c657..a2d4614c657 100644
--- a/configserver/src/test/apps/app-jdisc-only-restart/searchdefinitions/music.sd
+++ b/configserver/src/test/apps/app-jdisc-only-restart/schemas/music.sd
diff --git a/configserver/src/test/apps/app-jdisc-only/searchdefinitions/music.sd b/configserver/src/test/apps/app-jdisc-only/schemas/music.sd
index a2d4614c657..a2d4614c657 100644
--- a/configserver/src/test/apps/app-jdisc-only/searchdefinitions/music.sd
+++ b/configserver/src/test/apps/app-jdisc-only/schemas/music.sd
diff --git a/configserver/src/test/apps/app-major-version-2/searchdefinitions/music.sd b/configserver/src/test/apps/app-major-version-2/schemas/music.sd
index 7670e78f22b..7670e78f22b 100644
--- a/configserver/src/test/apps/app-major-version-2/searchdefinitions/music.sd
+++ b/configserver/src/test/apps/app-major-version-2/schemas/music.sd
diff --git a/configserver/src/test/apps/app/searchdefinitions/music.sd b/configserver/src/test/apps/app/schemas/music.sd
index 7670e78f22b..7670e78f22b 100644
--- a/configserver/src/test/apps/app/searchdefinitions/music.sd
+++ b/configserver/src/test/apps/app/schemas/music.sd
diff --git a/configserver/src/test/apps/hosted-no-write-access-control/searchdefinitions/music.sd b/configserver/src/test/apps/hosted-no-write-access-control/schemas/music.sd
index 78d58b27d4a..78d58b27d4a 100644
--- a/configserver/src/test/apps/hosted-no-write-access-control/searchdefinitions/music.sd
+++ b/configserver/src/test/apps/hosted-no-write-access-control/schemas/music.sd
diff --git a/configserver/src/test/apps/hosted/searchdefinitions/music.sd b/configserver/src/test/apps/hosted/schemas/music.sd
index 78d58b27d4a..78d58b27d4a 100644
--- a/configserver/src/test/apps/hosted/searchdefinitions/music.sd
+++ b/configserver/src/test/apps/hosted/schemas/music.sd
diff --git a/configserver/src/test/apps/zkapp/searchdefinitions/bar.expression b/configserver/src/test/apps/zkapp/schemas/bar.expression
index eed496e6aeb..eed496e6aeb 100644
--- a/configserver/src/test/apps/zkapp/searchdefinitions/bar.expression
+++ b/configserver/src/test/apps/zkapp/schemas/bar.expression
diff --git a/configserver/src/test/apps/zkapp/searchdefinitions/foo.expression b/configserver/src/test/apps/zkapp/schemas/foo.expression
index ce26aa75dcb..ce26aa75dcb 100644
--- a/configserver/src/test/apps/zkapp/searchdefinitions/foo.expression
+++ b/configserver/src/test/apps/zkapp/schemas/foo.expression
diff --git a/configserver/src/test/apps/zkapp/searchdefinitions/laptop.sd b/configserver/src/test/apps/zkapp/schemas/laptop.sd
index 21a76ad605c..21a76ad605c 100644
--- a/configserver/src/test/apps/zkapp/searchdefinitions/laptop.sd
+++ b/configserver/src/test/apps/zkapp/schemas/laptop.sd
diff --git a/configserver/src/test/apps/zkapp/searchdefinitions/music.sd b/configserver/src/test/apps/zkapp/schemas/music.sd
index bdf4c060cf7..bdf4c060cf7 100644
--- a/configserver/src/test/apps/zkapp/searchdefinitions/music.sd
+++ b/configserver/src/test/apps/zkapp/schemas/music.sd
diff --git a/configserver/src/test/apps/zkapp/searchdefinitions/pc.sd b/configserver/src/test/apps/zkapp/schemas/pc.sd
index bdc90328cdb..bdc90328cdb 100644
--- a/configserver/src/test/apps/zkapp/searchdefinitions/pc.sd
+++ b/configserver/src/test/apps/zkapp/schemas/pc.sd
diff --git a/configserver/src/test/apps/zkapp/searchdefinitions/product.sd b/configserver/src/test/apps/zkapp/schemas/product.sd
index 588a1d544c6..588a1d544c6 100644
--- a/configserver/src/test/apps/zkapp/searchdefinitions/product.sd
+++ b/configserver/src/test/apps/zkapp/schemas/product.sd
diff --git a/configserver/src/test/apps/zkapp/searchdefinitions/sock.sd b/configserver/src/test/apps/zkapp/schemas/sock.sd
index 9331c0caf54..9331c0caf54 100644
--- a/configserver/src/test/apps/zkapp/searchdefinitions/sock.sd
+++ b/configserver/src/test/apps/zkapp/schemas/sock.sd
diff --git a/configserver/src/test/apps/zkfeed/searchdefinitions/bar.expression b/configserver/src/test/apps/zkfeed/schemas/bar.expression
index eed496e6aeb..eed496e6aeb 100644
--- a/configserver/src/test/apps/zkfeed/searchdefinitions/bar.expression
+++ b/configserver/src/test/apps/zkfeed/schemas/bar.expression
diff --git a/configserver/src/test/apps/zkfeed/searchdefinitions/foo.expression b/configserver/src/test/apps/zkfeed/schemas/foo.expression
index ce26aa75dcb..ce26aa75dcb 100644
--- a/configserver/src/test/apps/zkfeed/searchdefinitions/foo.expression
+++ b/configserver/src/test/apps/zkfeed/schemas/foo.expression
diff --git a/configserver/src/test/apps/zkfeed/searchdefinitions/laptop.sd b/configserver/src/test/apps/zkfeed/schemas/laptop.sd
index 21a76ad605c..21a76ad605c 100644
--- a/configserver/src/test/apps/zkfeed/searchdefinitions/laptop.sd
+++ b/configserver/src/test/apps/zkfeed/schemas/laptop.sd
diff --git a/configserver/src/test/apps/zkfeed/searchdefinitions/pc.sd b/configserver/src/test/apps/zkfeed/schemas/pc.sd
index bdc90328cdb..bdc90328cdb 100644
--- a/configserver/src/test/apps/zkfeed/searchdefinitions/pc.sd
+++ b/configserver/src/test/apps/zkfeed/schemas/pc.sd
diff --git a/configserver/src/test/apps/zkfeed/searchdefinitions/product.sd b/configserver/src/test/apps/zkfeed/schemas/product.sd
index 588a1d544c6..588a1d544c6 100644
--- a/configserver/src/test/apps/zkfeed/searchdefinitions/product.sd
+++ b/configserver/src/test/apps/zkfeed/schemas/product.sd
diff --git a/configserver/src/test/apps/zkfeed/searchdefinitions/sock.sd b/configserver/src/test/apps/zkfeed/schemas/sock.sd
index 9331c0caf54..9331c0caf54 100644
--- a/configserver/src/test/apps/zkfeed/searchdefinitions/sock.sd
+++ b/configserver/src/test/apps/zkfeed/schemas/sock.sd
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java
index 7b9420b6b9e..0acf4404326 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java
@@ -74,7 +74,8 @@ public class ModelContextImplTest {
                         Optional.empty(),
                         Optional.empty(),
                         List.of(),
-                        new SecretStoreProvider().get()),
+                        new SecretStoreProvider().get(),
+                        List.of()),
                 Optional.empty(),
                 Optional.empty(),
                 new Version(7),
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStreamTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStreamTest.java
index 016192e3281..2a18a1b2221 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStreamTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStreamTest.java
@@ -141,9 +141,9 @@ public class CompressedApplicationInputStreamTest {
         assertTrue(files.contains(new File(outApp, "services.xml")));
         assertTrue(files.contains(new File(outApp, "hosts.xml")));
         assertTrue(files.contains(new File(outApp, "deployment.xml")));
-        assertTrue(files.contains(new File(outApp, "searchdefinitions")));
+        assertTrue(files.contains(new File(outApp, "schemas")));
         assertTrue(files.contains(new File(outApp, "external")));
-        File sd = files.get(files.indexOf(new File(outApp, "searchdefinitions")));
+        File sd = files.get(files.indexOf(new File(outApp, "schemas")));
         assertTrue(sd.isDirectory());
         assertThat(sd.listFiles().length, is(1));
         assertThat(sd.listFiles()[0].getAbsolutePath(), is(new File(sd, "keyvalue.sd").getAbsolutePath()));
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java
index 7d14b1996b0..e20363af4e9 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java
@@ -64,7 +64,7 @@ public class ZooKeeperClientTest {
         Map<Version, FileRegistry> fileRegistries = createFileRegistries();
         app.writeMetaData();
         zkc.initialize();
-        zkc.write(app);
+        zkc.writeApplicationPackage(app);
         zkc.write(fileRegistries);
     }
 
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/PrepareParamsTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/PrepareParamsTest.java
index f50238f2b85..f68e79ae266 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/PrepareParamsTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/PrepareParamsTest.java
@@ -4,9 +4,16 @@ package com.yahoo.vespa.config.server.session;
 import com.yahoo.config.model.api.ApplicationRoles;
 import com.yahoo.config.model.api.ContainerEndpoint;
 import com.yahoo.config.model.api.EndpointCertificateMetadata;
+import com.yahoo.config.model.api.TenantSecretStore;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.TenantName;
 import com.yahoo.container.jdisc.HttpRequest;
+import com.yahoo.security.KeyAlgorithm;
+import com.yahoo.security.KeyUtils;
+import com.yahoo.security.SignatureAlgorithm;
+import com.yahoo.security.X509CertificateBuilder;
+import com.yahoo.security.X509CertificateUtils;
+import com.yahoo.security.X509CertificateWithKey;
 import com.yahoo.slime.ArrayInserter;
 import com.yahoo.slime.Cursor;
 import com.yahoo.slime.Injector;
@@ -18,12 +25,19 @@ import com.yahoo.slime.SlimeInserter;
 import com.yahoo.slime.SlimeUtils;
 import com.yahoo.vespa.config.server.tenant.ContainerEndpointSerializer;
 import com.yahoo.vespa.config.server.tenant.EndpointCertificateMetadataSerializer;
+import com.yahoo.vespa.config.server.tenant.TenantSecretStoreSerializer;
 import org.junit.Test;
 
+import javax.security.auth.x500.X500Principal;
 import java.io.IOException;
+import java.math.BigInteger;
 import java.net.URLEncoder;
 import java.nio.charset.StandardCharsets;
+import java.security.KeyPair;
+import java.security.cert.X509Certificate;
 import java.time.Duration;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -179,6 +193,38 @@ public class PrepareParamsTest {
         assertPrepareParamsEqual(urlPrepareParams, jsonPrepareParams);
     }
 
+    @Test
+    public void testOperatorCertificates() throws IOException {
+        Slime slime = SlimeUtils.jsonToSlime(json);
+        Cursor cursor = slime.get();
+        Cursor array = cursor.setArray(PrepareParams.OPERATOR_CERTIFICATES);
+        X509Certificate certificate = X509CertificateUtils.createSelfSigned("cn=myservice", Duration.ofDays(1)).certificate();
+        array.addString(X509CertificateUtils.toPem(certificate));
+        PrepareParams prepareParams = PrepareParams.fromJson(SlimeUtils.toJsonBytes(slime), TenantName.from("foo"), Duration.ofSeconds(60));
+        assertEquals(1, prepareParams.operatorCertificates().size());
+        assertEquals(certificate, prepareParams.operatorCertificates().get(0));
+    }
+
+    @Test
+    public void testSecretStores() throws  IOException {
+        List<TenantSecretStore> secretStores = List.of(new TenantSecretStore("name", "awsId", "role"));
+        Slime secretStoreSlime = TenantSecretStoreSerializer.toSlime(secretStores);
+        String secretStoreParam = new String(SlimeUtils.toJsonBytes(secretStoreSlime), StandardCharsets.UTF_8);
+
+        var prepareParams = createParams(request + "&" + PrepareParams.TENANT_SECRET_STORES_PARAM_NAME + "=" + URLEncoder.encode(secretStoreParam, StandardCharsets.UTF_8), TenantName.from("foo"));
+        assertEquals(1, prepareParams.tenantSecretStores().size());
+        TenantSecretStore tenantSecretStore = prepareParams.tenantSecretStores().get(0);
+        assertEquals("name", tenantSecretStore.getName());
+        assertEquals("awsId", tenantSecretStore.getAwsId());
+        assertEquals("role", tenantSecretStore.getRole());
+
+        // Verify using json object
+        var root = SlimeUtils.jsonToSlime(json);
+        new Injector().inject(secretStoreSlime.get(), new ObjectInserter(root.get(), PrepareParams.TENANT_SECRET_STORES_PARAM_NAME));
+        PrepareParams prepareParamsJson = PrepareParams.fromJson(SlimeUtils.toJsonBytes(root), TenantName.from("foo"), Duration.ofSeconds(60));
+        assertPrepareParamsEqual(prepareParams, prepareParamsJson);
+    }
+
     private void assertPrepareParamsEqual(PrepareParams urlParams, PrepareParams jsonParams) {
         assertEquals(urlParams.ignoreValidationErrors(), jsonParams.ignoreValidationErrors());
         assertEquals(urlParams.isDryRun(), jsonParams.isDryRun());
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/OperatorCertificateSerializerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/OperatorCertificateSerializerTest.java
new file mode 100644
index 00000000000..b77248f0840
--- /dev/null
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/OperatorCertificateSerializerTest.java
@@ -0,0 +1,30 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.vespa.config.server.tenant;
+
+import com.yahoo.security.X509CertificateUtils;
+import com.yahoo.security.X509CertificateWithKey;
+import com.yahoo.slime.Slime;
+import com.yahoo.slime.SlimeUtils;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.security.cert.X509Certificate;
+import java.time.Duration;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+
+public class OperatorCertificateSerializerTest {
+
+    @Test
+    public void testSerialization() {
+        X509Certificate certificate = X509CertificateUtils.createSelfSigned("cn=mycn", Duration.ofDays(1)).certificate();
+        Slime slime = OperatorCertificateSerializer.toSlime(List.of(certificate));
+        List<X509Certificate> deserialized = OperatorCertificateSerializer.fromSlime(slime.get());
+        assertEquals(1, deserialized.size());
+        assertEquals(certificate, deserialized.get(0));
+    }
+}
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java
index 80d01fa4d36..458cdb82066 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java
@@ -78,7 +78,7 @@ public class ZKApplicationPackageTest {
         assertTrue(Pattern.compile(".*<slobroks>.*",Pattern.MULTILINE+Pattern.DOTALL).matcher(IOUtils.readAll(zkApp.getFile(Path.fromString("services.xml")).createReader())).matches());
         DeployState deployState = new DeployState.Builder().applicationPackage(zkApp).build();
         assertEquals(deployState.getSchemas().size(), 5);
-        assertEquals(zkApp.searchDefinitionContents().size(), 5);
+        assertEquals(zkApp.getSchemas().size(), 5);
         assertEquals(IOUtils.readAll(zkApp.getRankingExpression("foo.expression")), "foo()+1\n");
         assertEquals(zkApp.getFiles(Path.fromString(""), "xml").size(), 3);
         assertEquals(zkApp.getFileReference(Path.fromString("components/file.txt")).getAbsolutePath(), "/home/vespa/test/file.txt");
@@ -124,8 +124,7 @@ public class ZKApplicationPackageTest {
     }
 
     /**
-     * Takes for instance the dir /app  and puts the contents into the given ZK path. Ignores files starting with dot,
-     * and dirs called CVS.
+     * Takes for instance the dir /app  and puts the contents into the given ZK path. Ignores files starting with dot.
      *
      * @param dir            directory which holds the summary class part files
      * @param path           zookeeper path
@@ -142,7 +141,6 @@ public class ZKApplicationPackageTest {
             }
             for (File file : listFiles(dir, filenameFilter)) {
                 if (file.getName().startsWith(".")) continue; //.svn , .git ...
-                if ("CVS".equals(file.getName())) continue;
                 if (file.isFile()) {
                     String contents = IOUtils.readFile(file);
                     zk.putData(path, file.getName(), contents);
diff --git a/configserver/src/test/resources/deploy/advancedapp/searchdefinitions/keyvalue.sd b/configserver/src/test/resources/deploy/advancedapp/schemas/keyvalue.sd
index 2d0f1ef0b75..2d0f1ef0b75 100644
--- a/configserver/src/test/resources/deploy/advancedapp/searchdefinitions/keyvalue.sd
+++ b/configserver/src/test/resources/deploy/advancedapp/schemas/keyvalue.sd