Merge branch 'master' into jonmv/cluster-controller-migration-cleanup-2

16 files changed, 116 insertions, 74 deletions

diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
index 98d0c32c5d5..5d46f3dc240 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
@@ -29,6 +29,7 @@ import com.yahoo.docproc.jdisc.metric.NullMetric;
 import com.yahoo.io.IOUtils;
 import com.yahoo.jdisc.Metric;
 import com.yahoo.path.Path;
+import com.yahoo.slime.Slime;
 import com.yahoo.transaction.NestedTransaction;
 import com.yahoo.transaction.Transaction;
 import com.yahoo.vespa.config.server.application.Application;
@@ -663,10 +664,10 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
         }
     }
 
-    public Set<ApplicationId> listApplications() {
+    public List<ApplicationId> listApplications() {
         return tenantRepository.getAllTenants().stream()
                 .flatMap(tenant -> tenant.getApplicationRepo().activeApplications().stream())
-                .collect(Collectors.toSet());
+                .collect(Collectors.toList());
     }
 
     private boolean isFileLastModifiedBefore(File fileReference, Instant instant) {
@@ -702,9 +703,9 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
                 : applicationSet.get().getAllVersions(applicationId);
     }
 
-    public HttpResponse validateSecretStore(ApplicationId applicationId, TenantSecretStore tenantSecretStore, String tenantSecretName) {
+    public HttpResponse validateSecretStore(ApplicationId applicationId, SystemName systemName, Slime slime) {
         Application application = getApplication(applicationId);
-        return secretStoreValidator.validateSecretStore(application, tenantSecretStore, tenantSecretName);
+        return secretStoreValidator.validateSecretStore(application, systemName, slime);
     }
 
     // ---------------- Convergence ----------------------------------------------------------------
@@ -867,8 +868,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
     public void deleteExpiredLocalSessions() {
         Map<Tenant, Collection<LocalSession>> sessionsPerTenant = new HashMap<>();
         tenantRepository.getAllTenants()
-                        .forEach(tenant -> sessionsPerTenant.put(tenant,
-                                                                 List.copyOf(tenant.getSessionRepository().getLocalSessions())));
+                        .forEach(tenant -> sessionsPerTenant.put(tenant, tenant.getSessionRepository().getLocalSessions()));
 
         Set<ApplicationId> applicationIds = new HashSet<>();
         sessionsPerTenant.values()
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java
index 3275dc42477..be26e880440 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java
@@ -17,10 +17,11 @@ import com.yahoo.yolean.Exceptions;
 import java.time.Duration;
 import java.time.Instant;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Optional;
-import java.util.Set;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
@@ -203,7 +204,8 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable
 
     private boolean redeployAllApplications() throws InterruptedException {
         Instant end = Instant.now().plus(maxDurationOfRedeployment);
-        Set<ApplicationId> applicationsNotRedeployed = applicationRepository.listApplications();
+        List<ApplicationId> applicationsNotRedeployed = applicationRepository.listApplications();
+        Collections.shuffle(applicationsNotRedeployed);
         long failCount = 0;
         do {
             applicationsNotRedeployed = redeployApplications(applicationsNotRedeployed);
@@ -225,7 +227,7 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable
     }
 
     // Returns the set of applications that failed to redeploy
-    private Set<ApplicationId> redeployApplications(Set<ApplicationId> applicationIds) throws InterruptedException {
+    private List<ApplicationId> redeployApplications(List<ApplicationId> applicationIds) throws InterruptedException {
         ExecutorService executor = Executors.newFixedThreadPool(configserverConfig.numParallelTenantLoaders(),
                                                                 new DaemonThreadFactory("redeploy apps"));
         // Keep track of deployment per application
@@ -235,12 +237,12 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable
                                                     executor.submit(() -> applicationRepository.deployFromLocalActive(appId, true /* bootstrap */)
                                 .ifPresent(Deployment::activate))));
 
-        Set<ApplicationId> failedDeployments =
+        List<ApplicationId> failedDeployments =
                 deployments.entrySet().stream()
                         .map(entry -> checkDeployment(entry.getKey(), entry.getValue()))
                         .filter(Optional::isPresent)
                         .map(Optional::get)
-                        .collect(Collectors.toSet());
+                        .collect(Collectors.toList());
 
         executor.shutdown();
         executor.awaitTermination(365, TimeUnit.DAYS); // Timeout should never happen
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/Application.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/Application.java
index 96dc334ba45..8e98bf5884b 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/Application.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/Application.java
@@ -146,7 +146,8 @@ public class Application implements ModelResult {
                 } catch (ConfigurationRuntimeException e) {
                     // This can happen in cases where services ask for config that no longer exist before they have been able
                     // to reconfigure themselves
-                    log.log(Level.INFO, "Error resolving instance for builder '" + builder.getClass().getName() +
+                    log.log(Level.INFO, TenantRepository.logPre(getId()) +
+                                        ": Error resolving instance for builder '" + builder.getClass().getName() +
                                         "', returning empty config: " + Exceptions.toMessageString(e));
                     payload = ConfigPayload.fromBuilder(new ConfigPayloadBuilder());
                 }
@@ -178,7 +179,7 @@ public class Application implements ModelResult {
     }
 
     private void debug(String message) {
-        log.log(Level.FINE, TenantRepository.logPre(getId())+message);
+        log.log(Level.FINE, TenantRepository.logPre(getId()) + message);
     }
 
     private ConfigDefinition getTargetDef(GetConfigRequest req) {
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationCuratorDatabase.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationCuratorDatabase.java
index 2edd0d9fc2a..519e53272dc 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationCuratorDatabase.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationCuratorDatabase.java
@@ -77,6 +77,8 @@ public class ApplicationCuratorDatabase {
         if ( ! id.tenant().equals(tenant))
             throw new IllegalArgumentException("Cannot write application id '" + id + "' for tenant '" + tenant + "'");
         try (Lock lock = lock(id)) {
+            if (curator.exists(applicationPath(id))) return;
+
             curator.create(applicationPath(id));
             modifyReindexing(id, ApplicationReindexing.empty(), UnaryOperator.identity());
         }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java
index 980134f8884..05e73e7f454 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java
@@ -219,7 +219,8 @@ public class Deployment implements com.yahoo.config.provision.Deployment {
                     .ignoreValidationErrors(ignoreValidationErrors)
                     .isBootstrap(isBootstrap)
                     .force(force)
-                    .waitForResourcesInPrepare(waitForResourcesInPrepare);
+                    .waitForResourcesInPrepare(waitForResourcesInPrepare)
+                    .tenantSecretStores(session.getTenantSecretStores());
             session.getDockerImageRepository().ifPresent(params::dockerImageRepository);
             session.getAthenzDomain().ifPresent(params::athenzDomain);
 
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SecretStoreValidator.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SecretStoreValidator.java
index 72f5747e15c..71eed450955 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SecretStoreValidator.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SecretStoreValidator.java
@@ -3,12 +3,16 @@ package com.yahoo.vespa.config.server.http;
 
 import ai.vespa.util.http.VespaHttpClientBuilder;
 import com.yahoo.config.model.api.HostInfo;
+import com.yahoo.config.provision.SystemName;
+import com.yahoo.config.provision.TenantName;
+import com.yahoo.container.jdisc.HttpRequest;
 import com.yahoo.container.jdisc.HttpResponse;
 import com.yahoo.container.jdisc.secretstore.SecretStore;
 import com.yahoo.slime.Slime;
 import com.yahoo.slime.SlimeUtils;
 import com.yahoo.vespa.config.server.application.Application;
 import com.yahoo.config.model.api.TenantSecretStore;
+import com.yahoo.vespa.config.server.tenant.SecretStoreExternalIdRetriever;
 import com.yahoo.yolean.Exceptions;
 import org.apache.http.client.methods.HttpPost;
 import org.apache.http.entity.ByteArrayEntity;
@@ -22,6 +26,7 @@ import static com.yahoo.yolean.Exceptions.uncheck;
 
 /**
  * @author olaa
+ * Takes the payload received from the controller, adds external ID and acts as a proxy for the AwsParameterStoreValidationHandler result
  */
 public class SecretStoreValidator {
 
@@ -35,22 +40,12 @@ public class SecretStoreValidator {
         this.secretStore = secretStore;
     }
 
-    public HttpResponse validateSecretStore(Application application, TenantSecretStore tenantSecretStore, String tenantSecretName) {
-        var slime = toSlime(tenantSecretStore, tenantSecretName);
+    public HttpResponse validateSecretStore(Application application, SystemName system, Slime slime) {
+        addExternalId(application.getId().tenant(), system, slime);
         var uri = getUri(application);
         return postRequest(uri, slime);
     }
 
-    private Slime toSlime(TenantSecretStore tenantSecretStore, String tenantSecretName) {
-        var slime = new Slime();
-        var cursor = slime.setObject();
-        cursor.setString("externalId", secretStore.getSecret(tenantSecretName));
-        cursor.setString("awsId", tenantSecretStore.getAwsId());
-        cursor.setString("name", tenantSecretStore.getName());
-        cursor.setString("role", tenantSecretStore.getRole());
-        return slime;
-    }
-
     private URI getUri(Application application) {
         var hostname = application.getModel().getHosts()
                 .stream()
@@ -78,4 +73,11 @@ public class SecretStoreValidator {
         }
     }
 
+    private void addExternalId(TenantName tenantName, SystemName system, Slime slime) {
+        var data = slime.get();
+        var name = data.field("name").asString();
+        var secretName = SecretStoreExternalIdRetriever.secretName(tenantName, system, name);
+        data.setString("externalId", secretStore.getSecret(secretName));
+    }
+
 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java
index ba2989164ee..3634a6825a3 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java
@@ -29,7 +29,6 @@ import com.yahoo.vespa.config.server.http.HttpHandler;
 import com.yahoo.vespa.config.server.http.JSONResponse;
 import com.yahoo.vespa.config.server.http.NotFoundException;
 import com.yahoo.vespa.config.server.tenant.Tenant;
-import com.yahoo.config.model.api.TenantSecretStore;
 
 import java.io.IOException;
 import java.net.URLDecoder;
@@ -47,7 +46,6 @@ import java.util.stream.Stream;
 
 import static com.yahoo.yolean.Exceptions.uncheck;
 import static java.nio.charset.StandardCharsets.UTF_8;
-import static java.util.Map.Entry.comparingByKey;
 import static java.util.stream.Collectors.toList;
 
 /**
@@ -70,7 +68,7 @@ public class ApplicationHandler extends HttpHandler {
             "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/metrics/*",
             "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/metrics/*",
             "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/logs",
-            "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/validate-secret-store/*",
+            "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/validate-secret-store",
             "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/tester/*/*",
             "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/tester/*",
             "http://*/application/v2/tenant/*/application/*/environment/*/region/*/instance/*/quota",
@@ -231,9 +229,8 @@ public class ApplicationHandler extends HttpHandler {
         }
 
         if (isValidateSecretStoreRequest(request)) {
-            var tenantSecretStore = tenantSecretStoreFromRequest(request);
-            var tenantSecretName = tenantSecretNameFromRequest(request);
-            return applicationRepository.validateSecretStore(applicationId, tenantSecretStore, tenantSecretName);
+            var slime = uncheck(() -> SlimeUtils.jsonToSlime(request.getData().readAllBytes()));
+            return applicationRepository.validateSecretStore(applicationId, zone.system(), slime);
         }
 
         throw new NotFoundException("Illegal POST request '" + request.getUri() + "'");
@@ -360,8 +357,8 @@ public class ApplicationHandler extends HttpHandler {
     }
 
     private static boolean isValidateSecretStoreRequest(HttpRequest request) {
-        return getBindingMatch(request).groupCount() == 8 &&
-                request.getUri().getPath().contains("/validate-secret-store/");
+        return getBindingMatch(request).groupCount() == 7 &&
+                request.getUri().getPath().endsWith("/validate-secret-store");
     }
 
     private static boolean isServiceConvergeListRequest(HttpRequest request) {
@@ -424,11 +421,6 @@ public class ApplicationHandler extends HttpHandler {
         return bm.group(8);
     }
 
-    private static String tenantSecretNameFromRequest(HttpRequest req) {
-        BindingMatch<?> bm = getBindingMatch(req);
-        return bm.group(7);
-    }
-
     private static ApplicationId getApplicationIdFromRequest(HttpRequest req) {
         // Two bindings for this: with full app id or only application name
         BindingMatch<?> bm = getBindingMatch(req);
@@ -533,14 +525,6 @@ public class ApplicationHandler extends HttpHandler {
 
     }
 
-    private TenantSecretStore tenantSecretStoreFromRequest(HttpRequest httpRequest) {
-        var data = uncheck(() -> SlimeUtils.jsonToSlime(httpRequest.getData().readAllBytes()).get());
-        var awsId = data.field("awsId").asString();
-        var name = data.field("name").asString();
-        var role = data.field("role").asString();
-        return new TenantSecretStore(name, awsId, role);
-    }
-
     private static JSONResponse createMessageResponse(String message) {
         return new JSONResponse(Response.Status.OK) { { object.setString("message", message); } };
     }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java
index fc8fbe4d7d5..5a3e0311db9 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java
@@ -205,9 +205,14 @@ public final class PrepareParams {
         }
 
         public Builder tenantSecretStores(String serialized) {
-            this.tenantSecretStores = (serialized == null)
+            List<TenantSecretStore> secretStores = (serialized == null)
                     ? List.of()
                     : TenantSecretStoreSerializer.listFromSlime(SlimeUtils.jsonToSlime(serialized).get());
+            return tenantSecretStores(secretStores);
+        }
+
+        public Builder tenantSecretStores(List<TenantSecretStore> tenantSecretStores) {
+            this.tenantSecretStores = tenantSecretStores;
             return this;
         }
 
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java
index 9b74eac5631..f1044b28049 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java
@@ -6,6 +6,7 @@ import com.yahoo.config.FileReference;
 import com.yahoo.config.application.api.ApplicationFile;
 import com.yahoo.config.application.api.ApplicationMetaData;
 import com.yahoo.config.application.api.ApplicationPackage;
+import com.yahoo.config.model.api.TenantSecretStore;
 import com.yahoo.config.provision.AllocatedHosts;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.AthenzDomain;
@@ -17,6 +18,7 @@ import com.yahoo.vespa.config.server.application.ApplicationSet;
 import com.yahoo.vespa.config.server.tenant.TenantRepository;
 
 import java.time.Instant;
+import java.util.List;
 import java.util.Optional;
 
 /**
@@ -131,6 +133,10 @@ public abstract class Session implements Comparable<Session>  {
         sessionZooKeeperClient.writeAthenzDomain(athenzDomain);
     }
 
+    public void setTenantSecretStores(List<TenantSecretStore> tenantSecretStores) {
+        sessionZooKeeperClient.writeTenantSecretStores(tenantSecretStores);
+    }
+
     /** Returns application id read from ZooKeeper. Will throw RuntimeException if not found */
     public ApplicationId getApplicationId() {
         return sessionZooKeeperClient.readApplicationId()
@@ -162,6 +168,10 @@ public abstract class Session implements Comparable<Session>  {
         return createSetStatusTransaction(Status.DEACTIVATE);
     }
 
+    public List<TenantSecretStore> getTenantSecretStores() {
+        return sessionZooKeeperClient.readTenantSecretStores();
+    }
+
     private Transaction createSetStatusTransaction(Status status) {
         return sessionZooKeeperClient.createWriteStatusTransaction(status);
     }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java
index d2260683495..f56e74c2869 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java
@@ -196,8 +196,9 @@ public class SessionRepository {
         return localSessionCache.get(sessionId);
     }
 
+    /** Returns a copy of local sessions */
     public Collection<LocalSession> getLocalSessions() {
-        return localSessionCache.values();
+        return List.copyOf(localSessionCache.values());
     }
 
     private void loadLocalSessions(ExecutorService executor) {
@@ -255,6 +256,7 @@ public class SessionRepository {
         session.setVespaVersion(existingSession.getVespaVersion());
         session.setDockerImageRepository(existingSession.getDockerImageRepository());
         session.setAthenzDomain(existingSession.getAthenzDomain());
+        session.setTenantSecretStores(existingSession.getTenantSecretStores());
         return session;
     }
 
@@ -297,8 +299,7 @@ public class SessionRepository {
     }
 
     private void deleteAllSessions() {
-        List<LocalSession> sessions = new ArrayList<>(localSessionCache.values());
-        for (LocalSession session : sessions) {
+        for (LocalSession session : getLocalSessions()) {
             deleteLocalSession(session);
         }
     }
@@ -309,6 +310,11 @@ public class SessionRepository {
         return remoteSessionCache.get(sessionId);
     }
 
+    /** Returns a copy of remote sessions */
+    public Collection<RemoteSession> getRemoteSessions() {
+        return List.copyOf(remoteSessionCache.values());
+    }
+
     public List<Long> getRemoteSessionsFromZooKeeper() {
         return getSessionList(curator.getChildren(sessionsPath));
     }
@@ -524,9 +530,7 @@ public class SessionRepository {
     private void nodeChanged() {
         zkWatcherExecutor.execute(() -> {
             Multiset<Session.Status> sessionMetrics = HashMultiset.create();
-            for (Session session : remoteSessionCache.values()) {
-                sessionMetrics.add(session.getStatus());
-            }
+            getRemoteSessions().forEach(session -> sessionMetrics.add(session.getStatus()));
             metricUpdater.setNewSessions(sessionMetrics.count(Session.Status.NEW));
             metricUpdater.setPreparedSessions(sessionMetrics.count(Session.Status.PREPARE));
             metricUpdater.setActivatedSessions(sessionMetrics.count(Session.Status.ACTIVATE));
@@ -556,7 +560,7 @@ public class SessionRepository {
         log.log(Level.FINE, () -> "Purging old sessions for tenant '" + tenantName + "'");
         Set<LocalSession> toDelete = new HashSet<>();
         try {
-            for (LocalSession candidate : List.copyOf(localSessionCache.values())) {
+            for (LocalSession candidate : getLocalSessions()) {
                 Instant createTime = candidate.getCreateTime();
                 log.log(Level.FINE, () -> "Candidate session for deletion: " + candidate.getSessionId() + ", created: " + createTime);
 
@@ -629,7 +633,7 @@ public class SessionRepository {
             sessionZKClient.createNewSession(clock.instant());
             Curator.CompletionWaiter waiter = sessionZKClient.getUploadWaiter();
             LocalSession session = new LocalSession(tenantName, sessionId, app, sessionZKClient);
-            waiter.awaitCompletion(timeoutBudget.timeLeft());
+            waiter.awaitCompletion(Duration.ofSeconds(Math.min(60, timeoutBudget.timeLeft().getSeconds())));
             addLocalSession(session);
             return session;
         } catch (Exception e) {
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/SecretStoreExternalIdRetriever.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/SecretStoreExternalIdRetriever.java
index cd2ae9d9d0c..0c254606169 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/SecretStoreExternalIdRetriever.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/SecretStoreExternalIdRetriever.java
@@ -25,7 +25,7 @@ public class SecretStoreExternalIdRetriever {
                 .collect(Collectors.toList());
     }
 
-    private static String secretName(TenantName tenant, SystemName system, String storeName) {
+    public static String secretName(TenantName tenant, SystemName system, String storeName) {
         return String.format(SECRET_NAME_FORMAT, tenantSecretGroup(system), tenant.value(), storeName);
     }
 
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantDebugger.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantDebugger.java
index d378f6e9235..334e53ee11f 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantDebugger.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantDebugger.java
@@ -1,13 +1,13 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.config.server.tenant;
 
-import java.util.logging.Level;
 import com.yahoo.vespa.curator.Curator;
 import org.apache.curator.framework.CuratorFramework;
 import org.apache.curator.framework.recipes.cache.TreeCache;
 import org.apache.curator.framework.recipes.cache.TreeCacheEvent;
 import org.apache.curator.framework.recipes.cache.TreeCacheListener;
 
+import java.util.logging.Level;
 import java.util.logging.Logger;
 
 /**
@@ -19,6 +19,7 @@ public class TenantDebugger implements TreeCacheListener {
 
     private static final Logger log = Logger.getLogger(TenantDebugger.class.getName());
 
+    @SuppressWarnings("deprecation") // TreeCache is deprecated, and recommended replacement is CuratorCache
     public TenantDebugger(Curator curator) throws Exception {
         TreeCache cache = new TreeCache(curator.framework(), "/config/v2/tenants");
         cache.getListenable().addListener(this);
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/MockSecretStoreValidator.java b/configserver/src/test/java/com/yahoo/vespa/config/server/MockSecretStoreValidator.java
index c464af404d9..a02e8a3f3a6 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/MockSecretStoreValidator.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/MockSecretStoreValidator.java
@@ -1,9 +1,12 @@
 // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.config.server;
 
+import com.yahoo.config.provision.SystemName;
 import com.yahoo.container.jdisc.HttpResponse;
 import com.yahoo.container.jdisc.SecretStoreProvider;
+import com.yahoo.restapi.SlimeJsonResponse;
 import com.yahoo.restapi.StringResponse;
+import com.yahoo.slime.Slime;
 import com.yahoo.vespa.config.server.application.Application;
 import com.yahoo.config.model.api.TenantSecretStore;
 import com.yahoo.vespa.config.server.http.SecretStoreValidator;
@@ -17,7 +20,7 @@ public class MockSecretStoreValidator extends SecretStoreValidator {
         super(new SecretStoreProvider().get());
     }
 
-    public HttpResponse validateSecretStore(Application application, TenantSecretStore tenantSecretStore, String tenantSecretName) {
-        return new StringResponse(tenantSecretStore.toString() + " - " + tenantSecretName);
+    public HttpResponse validateSecretStore(Application application, SystemName system, Slime slime) {
+        return new SlimeJsonResponse(slime);
     }
 }
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java
index 11e14192c74..f1404b30f30 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java
@@ -9,6 +9,7 @@ import com.yahoo.config.model.api.ModelContext;
 import com.yahoo.config.model.api.ModelCreateResult;
 import com.yahoo.config.model.api.ModelFactory;
 import com.yahoo.config.model.api.ServiceInfo;
+import com.yahoo.config.model.api.TenantSecretStore;
 import com.yahoo.config.model.api.ValidationParameters;
 import com.yahoo.config.model.provision.Host;
 import com.yahoo.config.model.provision.Hosts;
@@ -111,6 +112,21 @@ public class HostedDeployTest {
     }
 
     @Test
+    public void testRedeployWithTenantSecretStores() throws IOException {
+        List<TenantSecretStore> tenantSecretStores = List.of(new TenantSecretStore("foo", "123", "role"));
+        DeployTester tester = new DeployTester.Builder()
+                .modelFactory(createHostedModelFactory(Version.fromString("4.5.6"), Clock.systemUTC()))
+                .configserverConfig(createConfigserverConfig()).build();
+        tester.deployApp("src/test/apps/hosted/", Instant.now(), new PrepareParams.Builder()
+                .tenantSecretStores(tenantSecretStores));
+
+        Optional<com.yahoo.config.provision.Deployment> deployment = tester.redeployFromLocalActive(tester.applicationId());
+        assertTrue(deployment.isPresent());
+        deployment.get().activate();
+        assertEquals(tenantSecretStores, ((Deployment) deployment.get()).session().getTenantSecretStores());
+    }
+
+    @Test
     public void testDeployMultipleVersions() throws IOException {
         List<ModelFactory> modelFactories = List.of(createHostedModelFactory(Version.fromString("6.1.0")),
                                                     createHostedModelFactory(Version.fromString("6.2.0")),
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/SecretStoreValidatorTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/SecretStoreValidatorTest.java
index d308cd72b55..8e5dd16bc05 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/SecretStoreValidatorTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/SecretStoreValidatorTest.java
@@ -4,9 +4,13 @@ import com.github.tomakehurst.wiremock.junit.WireMockRule;
 import com.yahoo.config.model.api.HostInfo;
 import com.yahoo.config.model.api.Model;
 import com.yahoo.config.model.api.ServiceInfo;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.SystemName;
+import com.yahoo.config.provision.TenantName;
 import com.yahoo.container.jdisc.secretstore.SecretStore;
+import com.yahoo.slime.SlimeUtils;
 import com.yahoo.vespa.config.server.application.Application;
-import com.yahoo.config.model.api.TenantSecretStore;
+import com.yahoo.vespa.config.server.tenant.SecretStoreExternalIdRetriever;
 import org.junit.Rule;
 import org.junit.Test;
 
@@ -35,20 +39,26 @@ public class SecretStoreValidatorTest {
     @Test
     public void createsCorrectRequestData() throws IOException {
         var app = mockApplication();
-        var tenantSecretStore = new TenantSecretStore("store", "123", "role");
-        var tenantSecretName = "some-secret";
-        when(secretStore.getSecret(tenantSecretName)).thenReturn("some-secret-value");
-
+        var requestBody = SlimeUtils.jsonToSlime("{\"awsId\":\"123\"," +
+                "\"name\":\"store\"," +
+                "\"role\":\"role\"," +
+                "\"region\":\"some-region\"," +
+                "\"parameterName\":\"some-parameter\"" +
+                "}");
+        var expectedSecretName = SecretStoreExternalIdRetriever.secretName(TenantName.defaultName(), SystemName.PublicCd, "store");
+        when(secretStore.getSecret(expectedSecretName)).thenReturn("some-secret-value");
         stubFor(post(urlEqualTo("/validate-secret-store"))
-                .withRequestBody(equalToJson("{\"externalId\":\"some-secret-value\"," +
-                        "\"awsId\":\"123\"," +
+                .withRequestBody(equalToJson("{\"awsId\":\"123\"," +
                         "\"name\":\"store\"," +
-                        "\"role\":\"role\"" +
+                        "\"role\":\"role\"," +
+                        "\"region\":\"some-region\"," +
+                        "\"parameterName\":\"some-parameter\"," +
+                        "\"externalId\":\"some-secret-value\"" +
                         "}"))
                 .willReturn(aResponse()
                         .withStatus(200)
                         .withBody("is ok")));
-        var response = secretStoreValidator.validateSecretStore(app, tenantSecretStore, tenantSecretName);
+        var response = secretStoreValidator.validateSecretStore(app, SystemName.PublicCd, requestBody);
         var body = new ByteArrayOutputStream();
         response.render(body);
         assertEquals("is ok", body.toString());
@@ -60,6 +70,7 @@ public class SecretStoreValidatorTest {
         var hostList = createHostList();
         when(app.getModel()).thenReturn(model);
         when(model.getHosts()).thenReturn(hostList);
+        when(app.getId()).thenReturn(ApplicationId.defaultId());
         return app;
     }
 
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java
index ab9457c5d2a..d364785f415 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java
@@ -401,17 +401,17 @@ public class ApplicationHandlerTest {
     @Test
     public void testValidateSecretStore() throws IOException {
         applicationRepository.deploy(new File("src/test/apps/app-logserver-with-container"), prepareParams(applicationId));
-        var url = toUrlPath(applicationId, Zone.defaultZone(), true) + "/validate-secret-store/some-secret-name";
+        var url = toUrlPath(applicationId, Zone.defaultZone(), true) + "/validate-secret-store";
         var mockHandler = createApplicationHandler();
 
-        var requestData = new ByteArrayInputStream("{\"name\": \"store\", \"awsId\":\"aws-id\", \"role\":\"role\"}".getBytes(StandardCharsets.UTF_8));
+        var requestString = "{\"name\":\"store\",\"awsId\":\"aws-id\",\"role\":\"role\",\"region\":\"us-west-1\",\"parameterName\":\"some-parameter\"}";
+        var requestData = new ByteArrayInputStream(requestString.getBytes(StandardCharsets.UTF_8));
         var response = mockHandler.handle(createTestRequest(url, POST, requestData));
         assertEquals(200, response.getStatus());
 
 
-        // MockSecretStoreValidator returns response on format tenantSecretStore.toString() - tenantSecretName
-        var expectedResponse = "TenantSecretStore{name='store', awsId='aws-id', role='role'} - some-secret-name";
-        assertEquals(expectedResponse, getRenderedString(response));
+        // MockSecretStoreValidator simply returns the request body
+        assertEquals(requestString, getRenderedString(response));
     }
 
     @Test