diff options
author | Arnstein Ressem <aressem@verizonmedia.com> | 2020-09-10 13:04:38 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2020-09-10 13:04:38 +0200 |
commit | d3a5658487cb19a3046fe09915b52555b57ca586 (patch) | |
tree | d2e05fb9a222cc4febc6002569ce2572bdee45b9 | |
parent | 43db01fa344bc2fedfe25ab9477e3e5cebf1005f (diff) | |
parent | ac4a383b6edff45bba3f80a337e9885188a594dd (diff) |
Merge pull request #14363 from vespa-engine/revert-14335-revert-14330-revert-14308-hmusum/refactor-RemoteSession-take-2
Revert "Move code from RemoteSession to SessionRepository, take 3"
15 files changed, 466 insertions, 453 deletions
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 8aaf896cb72..fb07bf626f3 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -607,10 +607,9 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye try { Tenant tenant = getTenant(applicationId); if (tenant == null) throw new NotFoundException("Tenant '" + applicationId.tenant() + "' not found"); - RemoteSession session = getActiveSession(applicationId); - if (session == null) throw new NotFoundException("No active session found for '" + applicationId + "'"); - SessionRepository sessionRepository = tenant.getSessionRepository(); - return sessionRepository.ensureApplicationLoaded(session).getForVersionOrLatest(version, clock.instant()); + long sessionId = getSessionIdForApplication(tenant, applicationId); + RemoteSession session = getRemoteSession(tenant, sessionId); + return session.ensureApplicationLoaded().getForVersionOrLatest(version, clock.instant()); } catch (NotFoundException e) { log.log(Level.WARNING, "Failed getting application for '" + applicationId + "': " + e.getMessage()); throw e; @@ -949,7 +948,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye try { long currentActiveSessionId = applicationRepo.requireActiveSessionOf(appId); RemoteSession currentActiveSession = getRemoteSession(tenant, currentActiveSessionId); - currentActiveApplicationSet = Optional.ofNullable(tenant.getSessionRepository().ensureApplicationLoaded(currentActiveSession)); + currentActiveApplicationSet = Optional.ofNullable(currentActiveSession.ensureApplicationLoaded()); } catch (IllegalArgumentException e) { // Do nothing if we have no currently active session } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java index 6d2ef4028c6..7fc6b35722f 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.config.server.modelfactory; import com.google.common.collect.ImmutableSet; import com.yahoo.component.Version; import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.api.ConfigDefinitionRepo; import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.api.ModelFactory; @@ -56,6 +57,7 @@ public class ActivatedModelsBuilder extends ModelsBuilder<Application> { private final ConfigDefinitionRepo configDefinitionRepo; private final Metrics metrics; private final Curator curator; + private final DeployLogger logger; private final FlagSource flagSource; private final SecretStore secretStore; @@ -74,6 +76,7 @@ public class ActivatedModelsBuilder extends ModelsBuilder<Application> { this.configDefinitionRepo = globalComponentRegistry.getStaticConfigDefinitionRepo(); this.metrics = globalComponentRegistry.getMetrics(); this.curator = globalComponentRegistry.getCurator(); + this.logger = new SilentDeployLogger(); this.flagSource = globalComponentRegistry.getFlagSource(); this.secretStore = globalComponentRegistry.getSecretStore(); } @@ -87,14 +90,14 @@ public class ActivatedModelsBuilder extends ModelsBuilder<Application> { Optional<AllocatedHosts> ignored // Ignored since we have this in the app package for activated models ) { log.log(Level.FINE, String.format("Loading model version %s for session %s application %s", - modelFactory.version(), appGeneration, applicationId)); + modelFactory.version(), appGeneration, applicationId)); ModelContext.Properties modelContextProperties = createModelContextProperties(applicationId); Provisioned provisioned = new Provisioned(); ModelContext modelContext = new ModelContextImpl( applicationPackage, Optional.empty(), permanentApplicationPackage.applicationPackage(), - new SilentDeployLogger(), + logger, configDefinitionRepo, getForVersionOrLatest(applicationPackage.getFileRegistries(), modelFactory.version()).orElse(new MockFileRegistry()), createStaticProvisioner(applicationPackage.getAllocatedHosts(), diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java index 78a37499cef..36cac87a326 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java @@ -1,12 +1,22 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.session; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.TenantName; +import com.yahoo.lang.SettableOptional; import com.yahoo.transaction.Transaction; +import com.yahoo.vespa.config.server.GlobalComponentRegistry; import com.yahoo.vespa.config.server.application.ApplicationSet; +import com.yahoo.vespa.config.server.modelfactory.ActivatedModelsBuilder; +import com.yahoo.vespa.curator.Curator; +import org.apache.zookeeper.KeeperException; -import java.util.Objects; +import java.time.Clock; import java.util.Optional; +import java.util.Set; +import java.util.logging.Level; +import java.util.logging.Logger; /** * A RemoteSession represents a session created on another config server. This session can @@ -16,55 +26,95 @@ import java.util.Optional; */ public class RemoteSession extends Session { - private final Optional<ApplicationSet> applicationSet; + private static final Logger log = Logger.getLogger(RemoteSession.class.getName()); + private ApplicationSet applicationSet = null; + private final ActivatedModelsBuilder applicationLoader; + private final Clock clock; /** - * Creates a remote session, no application set loaded + * Creates a session. This involves loading the application, validating it and distributing it. * * @param tenant The name of the tenant creating session * @param sessionId The session id for this session. + * @param componentRegistry a registry of global components * @param zooKeeperClient a SessionZooKeeperClient instance */ - RemoteSession(TenantName tenant, long sessionId, SessionZooKeeperClient zooKeeperClient) { - this(tenant, sessionId, zooKeeperClient, Optional.empty()); + public RemoteSession(TenantName tenant, + long sessionId, + GlobalComponentRegistry componentRegistry, + SessionZooKeeperClient zooKeeperClient) { + super(tenant, sessionId, zooKeeperClient); + this.applicationLoader = new ActivatedModelsBuilder(tenant, sessionId, zooKeeperClient, componentRegistry); + this.clock = componentRegistry.getClock(); } - /** - * Creates a remote session, with application set - * - * @param tenant The name of the tenant creating session - * @param sessionId The session id for this session. - * @param zooKeeperClient a SessionZooKeeperClient instance - * @param applicationSet current application set for this session - */ - RemoteSession(TenantName tenant, - long sessionId, - SessionZooKeeperClient zooKeeperClient, - Optional<ApplicationSet> applicationSet) { - super(tenant, sessionId, zooKeeperClient); - this.applicationSet = applicationSet; + void prepare() { + Curator.CompletionWaiter waiter = sessionZooKeeperClient.getPrepareWaiter(); + ensureApplicationLoaded(); + notifyCompletion(waiter); } - Optional<ApplicationSet> applicationSet() { - return applicationSet; + private ApplicationSet loadApplication() { + ApplicationPackage applicationPackage = sessionZooKeeperClient.loadApplicationPackage(); + + // Read hosts allocated on the config server instance which created this + Optional<AllocatedHosts> allocatedHosts = applicationPackage.getAllocatedHosts(); + + return ApplicationSet.fromList(applicationLoader.buildModels(getApplicationId(), + sessionZooKeeperClient.readDockerImageRepository(), + sessionZooKeeperClient.readVespaVersion(), + applicationPackage, + new SettableOptional<>(allocatedHosts), + clock.instant())); } - public synchronized RemoteSession activated(ApplicationSet applicationSet) { - Objects.requireNonNull(applicationSet, "applicationSet cannot be null"); - return new RemoteSession(tenant, sessionId, sessionZooKeeperClient, Optional.of(applicationSet)); + public synchronized ApplicationSet ensureApplicationLoaded() { + return applicationSet == null ? applicationSet = loadApplication() : applicationSet; } - public synchronized RemoteSession deactivated() { - return new RemoteSession(tenant, sessionId, sessionZooKeeperClient, Optional.empty()); + public synchronized void deactivate() { + applicationSet = null; } public Transaction createDeleteTransaction() { return sessionZooKeeperClient.createWriteStatusTransaction(Status.DELETE); } + + void confirmUpload() { + Curator.CompletionWaiter waiter = sessionZooKeeperClient.getUploadWaiter(); + log.log(Level.FINE, "Notifying upload waiter for session " + getSessionId()); + notifyCompletion(waiter); + log.log(Level.FINE, "Done notifying upload for session " + getSessionId()); + } + + void notifyCompletion(Curator.CompletionWaiter completionWaiter) { + try { + completionWaiter.notifyCompletion(); + } catch (RuntimeException e) { + // Throw only if we get something else than NoNodeException or NodeExistsException. + // NoNodeException might happen when the session is no longer in use (e.g. the app using this session + // has been deleted) and this method has not been called yet for the previous session operation on a + // minority of the config servers. + // NodeExistsException might happen if an event for this node is delivered more than once, in that case + // this is a no-op + Set<Class<? extends KeeperException>> acceptedExceptions = Set.of(KeeperException.NoNodeException.class, + KeeperException.NodeExistsException.class); + Class<? extends Throwable> exceptionClass = e.getCause().getClass(); + if (acceptedExceptions.contains(exceptionClass)) + log.log(Level.FINE, "Not able to notify completion for session " + getSessionId() + + " (" + completionWaiter + ")," + + " node " + (exceptionClass.equals(KeeperException.NoNodeException.class) + ? "has been deleted" + : "already exists")); + else + throw e; + } + } - @Override - public String toString() { - return super.toString() + ",application set=" + applicationSet; + public void delete() { + Transaction transaction = sessionZooKeeperClient.deleteTransaction(); + transaction.commit(); + transaction.close(); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java index 85f9b575942..b3e35e955de 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java @@ -28,7 +28,7 @@ import java.util.Optional; */ public abstract class Session implements Comparable<Session> { - protected final long sessionId; + private final long sessionId; protected final TenantName tenant; protected final SessionZooKeeperClient sessionZooKeeperClient; protected final Optional<ApplicationPackage> applicationPackage; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionCache.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionCache.java index 37de857a251..b7d78f11201 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionCache.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionCache.java @@ -15,13 +15,23 @@ public class SessionCache<SESSIONTYPE extends Session> { private final HashMap<Long, SESSIONTYPE> sessions = new HashMap<>(); - public synchronized void putSession(SESSIONTYPE session) { sessions.put(session.getSessionId(), session); } + public synchronized void addSession(SESSIONTYPE session) { + sessions.putIfAbsent(session.getSessionId(), session); + } synchronized void removeSession(long id) { sessions.remove(id); } - public synchronized SESSIONTYPE getSession(long id) { return sessions.get(id); } + /** + * Gets a Session + * + * @param id session id + * @return a session belonging to the id supplied, or null if no session with the id was found + */ + public synchronized SESSIONTYPE getSession(long id) { + return sessions.get(id); + } public synchronized List<SESSIONTYPE> getSessions() { return new ArrayList<>(sessions.values()); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java index b251e2c6564..cbfa59b26e4 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java @@ -8,11 +8,9 @@ import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.application.provider.DeployData; import com.yahoo.config.model.application.provider.FilesApplicationPackage; -import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; import com.yahoo.io.IOUtils; -import com.yahoo.lang.SettableOptional; import com.yahoo.path.Path; import com.yahoo.transaction.AbstractTransaction; import com.yahoo.transaction.NestedTransaction; @@ -24,7 +22,6 @@ import com.yahoo.vespa.config.server.application.TenantApplications; import com.yahoo.vespa.config.server.configchange.ConfigChangeActions; import com.yahoo.vespa.config.server.deploy.TenantFileSystemDirs; import com.yahoo.vespa.config.server.filedistribution.FileDirectory; -import com.yahoo.vespa.config.server.modelfactory.ActivatedModelsBuilder; import com.yahoo.vespa.config.server.monitoring.MetricUpdater; import com.yahoo.vespa.config.server.monitoring.Metrics; import com.yahoo.vespa.config.server.tenant.TenantRepository; @@ -39,7 +36,6 @@ import com.yahoo.vespa.flags.Flags; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.cache.ChildData; import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; -import org.apache.zookeeper.KeeperException; import java.io.File; import java.io.FilenameFilter; @@ -55,7 +51,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.Set; import java.util.concurrent.Executor; import java.util.logging.Level; import java.util.logging.Logger; @@ -126,8 +121,10 @@ public class SessionRepository { // ---------------- Local sessions ---------------------------------------------------------------- public synchronized void addLocalSession(LocalSession session) { - localSessionCache.putSession(session); - createRemoteSession(session.getSessionId()); + localSessionCache.addSession(session); + long sessionId = session.getSessionId(); + RemoteSession remoteSession = createRemoteSession(sessionId); + addSessionStateWatcher(sessionId, remoteSession, Optional.of(session)); } public LocalSession getLocalSession(long sessionId) { @@ -265,6 +262,11 @@ public class SessionRepository { return getSessionList(curator.getChildren(sessionsPath)); } + public void addRemoteSession(RemoteSession session) { + remoteSessionCache.addSession(session); + metrics.incAddedSessions(); + } + public int deleteExpiredRemoteSessions(Clock clock, Duration expiryTime) { int deleted = 0; for (long sessionId : getRemoteSessions()) { @@ -273,26 +275,13 @@ public class SessionRepository { if (session.getStatus() == Session.Status.ACTIVATE) continue; if (sessionHasExpired(session.getCreateTime(), expiryTime, clock)) { log.log(Level.FINE, () -> "Remote session " + sessionId + " for " + tenantName + " has expired, deleting it"); - deleteRemoteSession(session); + session.delete(); deleted++; } } return deleted; } - public void deactivate(RemoteSession remoteSession) { - RemoteSession session = remoteSession.deactivated(); - remoteSessionCache.putSession(session); - updateSessionStateWatcher(session); - } - - public void deleteRemoteSession(RemoteSession session) { - SessionZooKeeperClient sessionZooKeeperClient = createSessionZooKeeperClient(session.getSessionId()); - Transaction transaction = sessionZooKeeperClient.deleteTransaction(); - transaction.commit(); - transaction.close(); - } - public int deleteExpiredLocks(Clock clock, Duration expiryTime) { int deleted = 0; for (var lock : curator.getChildren(locksPath)) { @@ -358,27 +347,36 @@ public class SessionRepository { log.log(Level.FINE, () -> "Adding remote session to SessionRepository: " + sessionId); RemoteSession remoteSession = createRemoteSession(sessionId); loadSessionIfActive(remoteSession); + addRemoteSession(remoteSession); + Optional<LocalSession> localSession = Optional.empty(); if (distributeApplicationPackage()) - createLocalSessionUsingDistributedApplicationPackage(sessionId); + localSession = createLocalSessionUsingDistributedApplicationPackage(sessionId); + addSessionStateWatcher(sessionId, remoteSession, localSession); } void activate(RemoteSession session) { long sessionId = session.getSessionId(); Curator.CompletionWaiter waiter = createSessionZooKeeperClient(sessionId).getActiveWaiter(); log.log(Level.FINE, () -> session.logPre() + "Getting session from repo: " + sessionId); - ApplicationSet app = ensureApplicationLoaded(session); + ApplicationSet app = session.ensureApplicationLoaded(); log.log(Level.FINE, () -> session.logPre() + "Reloading config for " + sessionId); applicationRepo.reloadConfig(app); log.log(Level.FINE, () -> session.logPre() + "Notifying " + waiter); - notifyCompletion(waiter, session); + session.notifyCompletion(waiter); log.log(Level.INFO, session.logPre() + "Session activated: " + sessionId); } - void deleteSession(RemoteSession remoteSession) { - LocalSession localSession = getLocalSession(remoteSession.getSessionId()); - if (localSession != null) - deleteLocalSession(localSession); - deactivate(remoteSession); + public void deactivate(RemoteSession remoteSession) { + remoteSession.deactivate(); + } + + public void delete(RemoteSession remoteSession, Optional<LocalSession> localSession) { + localSession.ifPresent(this::deleteLocalSession); + remoteSession.deactivate(); + } + + void prepare(RemoteSession session) { + session.prepare(); } boolean distributeApplicationPackage() { @@ -396,85 +394,13 @@ public class SessionRepository { for (ApplicationId applicationId : applicationRepo.activeApplications()) { if (applicationRepo.requireActiveSessionOf(applicationId) == session.getSessionId()) { log.log(Level.FINE, () -> "Found active application for session " + session.getSessionId() + " , loading it"); - applicationRepo.reloadConfig(ensureApplicationLoaded(session)); + applicationRepo.reloadConfig(session.ensureApplicationLoaded()); log.log(Level.INFO, session.logPre() + "Application activated successfully: " + applicationId + " (generation " + session.getSessionId() + ")"); return; } } } - void prepareRemoteSession(RemoteSession session) { - SessionZooKeeperClient sessionZooKeeperClient = createSessionZooKeeperClient(session.getSessionId()); - Curator.CompletionWaiter waiter = sessionZooKeeperClient.getPrepareWaiter(); - ensureApplicationLoaded(session); - notifyCompletion(waiter, session); - } - - public ApplicationSet ensureApplicationLoaded(RemoteSession session) { - try (var lock = lock(session.sessionId)) { - if (session.applicationSet().isPresent()) { - return session.applicationSet().get(); - } - - ApplicationSet applicationSet = loadApplication(session); - RemoteSession activated = session.activated(applicationSet); - remoteSessionCache.putSession(activated); - updateSessionStateWatcher(activated); - - return applicationSet; - } - } - - void confirmUpload(RemoteSession session) { - Curator.CompletionWaiter waiter = session.getSessionZooKeeperClient().getUploadWaiter(); - long sessionId = session.getSessionId(); - log.log(Level.FINE, "Notifying upload waiter for session " + sessionId); - notifyCompletion(waiter, session); - log.log(Level.FINE, "Done notifying upload for session " + sessionId); - } - - void notifyCompletion(Curator.CompletionWaiter completionWaiter, RemoteSession session) { - try { - completionWaiter.notifyCompletion(); - } catch (RuntimeException e) { - // Throw only if we get something else than NoNodeException or NodeExistsException. - // NoNodeException might happen when the session is no longer in use (e.g. the app using this session - // has been deleted) and this method has not been called yet for the previous session operation on a - // minority of the config servers. - // NodeExistsException might happen if an event for this node is delivered more than once, in that case - // this is a no-op - Set<Class<? extends KeeperException>> acceptedExceptions = Set.of(KeeperException.NoNodeException.class, - KeeperException.NodeExistsException.class); - Class<? extends Throwable> exceptionClass = e.getCause().getClass(); - if (acceptedExceptions.contains(exceptionClass)) - log.log(Level.FINE, "Not able to notify completion for session " + session.getSessionId() + - " (" + completionWaiter + ")," + - " node " + (exceptionClass.equals(KeeperException.NoNodeException.class) - ? "has been deleted" - : "already exists")); - else - throw e; - } - } - - private ApplicationSet loadApplication(RemoteSession session) { - log.log(Level.FINE, () -> "Loading application for " + session); - SessionZooKeeperClient sessionZooKeeperClient = createSessionZooKeeperClient(session.getSessionId()); - ApplicationPackage applicationPackage = sessionZooKeeperClient.loadApplicationPackage(); - ActivatedModelsBuilder builder = new ActivatedModelsBuilder(session.getTenantName(), - session.getSessionId(), - sessionZooKeeperClient, - componentRegistry); - // Read hosts allocated on the config server instance which created this - Optional<AllocatedHosts> allocatedHosts = applicationPackage.getAllocatedHosts(); - return ApplicationSet.fromList(builder.buildModels(session.getApplicationId(), - sessionZooKeeperClient.readDockerImageRepository(), - sessionZooKeeperClient.readVespaVersion(), - applicationPackage, - new SettableOptional<>(allocatedHosts), - clock.instant())); - } - private void nodeChanged() { zkWatcherExecutor.execute(() -> { Multiset<Session.Status> sessionMetrics = HashMultiset.create(); @@ -510,7 +436,7 @@ public class SessionRepository { RemoteSession session = remoteSessionCache.getSession(sessionId); if (session == null) continue; // session might have been deleted after getting session list log.log(Level.FINE, () -> session.logPre() + "Confirming upload for session " + sessionId); - confirmUpload(session); + session.confirmUpload(); } } @@ -528,16 +454,8 @@ public class SessionRepository { } public RemoteSession createRemoteSession(long sessionId) { - return createRemoteSession(sessionId, Optional.empty()); - } - - public RemoteSession createRemoteSession(long sessionId, Optional<ApplicationSet> applicationSet) { SessionZooKeeperClient sessionZKClient = createSessionZooKeeperClient(sessionId); - RemoteSession session = new RemoteSession(tenantName, sessionId, sessionZKClient, applicationSet); - remoteSessionCache.putSession(session); - updateSessionStateWatcher(session); - metrics.incAddedSessions(); - return session; + return new RemoteSession(tenantName, sessionId, componentRegistry, sessionZKClient); } private void ensureSessionPathDoesNotExist(long sessionId) { @@ -685,11 +603,10 @@ public class SessionRepository { * Returns a new local session for the given session id if it does not already exist. * Will also add the session to the local session cache if necessary */ - public void createLocalSessionUsingDistributedApplicationPackage(long sessionId) { + public Optional<LocalSession> createLocalSessionUsingDistributedApplicationPackage(long sessionId) { if (applicationRepo.hasLocalSession(sessionId)) { log.log(Level.FINE, () -> "Local session for session id " + sessionId + " already exists"); - createSessionFromId(sessionId); - return; + return Optional.of(createSessionFromId(sessionId)); } SessionZooKeeperClient sessionZKClient = createSessionZooKeeperClient(sessionId); @@ -704,15 +621,17 @@ public class SessionRepository { } catch (IllegalArgumentException e) { // We cannot be guaranteed that the file reference exists (it could be that it has not // been downloaded yet), and e.g when bootstrapping we cannot throw an exception in that case - log.log(Level.FINE, () -> "File reference for session id " + sessionId + ": " + fileReference + " not found in " + fileDirectory); - return; + log.log(Level.INFO, "File reference for session id " + sessionId + ": " + fileReference + " not found in " + fileDirectory); + return Optional.empty(); } ApplicationId applicationId = sessionZKClient.readApplicationId() .orElseThrow(() -> new RuntimeException("Could not find application id for session " + sessionId)); log.log(Level.FINE, () -> "Creating local session for tenant '" + tenantName + "' with session id " + sessionId); LocalSession localSession = createLocalSession(sessionDir, applicationId, sessionId); addLocalSession(localSession); + return Optional.of(localSession); } + return Optional.empty(); } private Optional<Long> getActiveSessionId(ApplicationId applicationId) { @@ -751,15 +670,15 @@ public class SessionRepository { return new TenantFileSystemDirs(componentRegistry.getConfigServerDB(), tenantName).getUserApplicationDir(sessionId); } - private void updateSessionStateWatcher(RemoteSession session) { - long sessionId = session.getSessionId(); - SessionStateWatcher sessionStateWatcher = sessionStateWatchers.get(sessionId); - if (sessionStateWatcher == null) { + private void addSessionStateWatcher(long sessionId, RemoteSession remoteSession, Optional<LocalSession> localSession) { + // Remote session will always be present in an existing state watcher, but local session might not + if (sessionStateWatchers.containsKey(sessionId)) { + localSession.ifPresent(session -> sessionStateWatchers.get(sessionId).addLocalSession(session)); + } else { Curator.FileCache fileCache = curator.createFileCache(getSessionStatePath(sessionId).getAbsolute(), false); fileCache.addListener(this::nodeChanged); - sessionStateWatchers.put(sessionId, new SessionStateWatcher(fileCache, session, metrics, zkWatcherExecutor, this)); - } else { - sessionStateWatcher.setSession(session); + sessionStateWatchers.put(sessionId, new SessionStateWatcher(fileCache, remoteSession, localSession, + metrics, zkWatcherExecutor, this)); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionStateWatcher.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionStateWatcher.java index 2fe0f9d6cbc..57d9f027447 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionStateWatcher.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionStateWatcher.java @@ -6,6 +6,7 @@ import com.yahoo.vespa.config.server.monitoring.MetricUpdater; import com.yahoo.vespa.curator.Curator; import org.apache.curator.framework.recipes.cache.ChildData; +import java.util.Optional; import java.util.concurrent.Executor; import java.util.logging.Level; import java.util.logging.Logger; @@ -24,18 +25,21 @@ public class SessionStateWatcher { private static final Logger log = Logger.getLogger(SessionStateWatcher.class.getName()); private final Curator.FileCache fileCache; - private RemoteSession session; + private final RemoteSession remoteSession; private final MetricUpdater metrics; private final Executor zkWatcherExecutor; private final SessionRepository sessionRepository; + private Optional<LocalSession> localSession; SessionStateWatcher(Curator.FileCache fileCache, - RemoteSession session, + RemoteSession remoteSession, + Optional<LocalSession> localSession, MetricUpdater metrics, Executor zkWatcherExecutor, SessionRepository sessionRepository) { this.fileCache = fileCache; - this.session = session; + this.remoteSession = remoteSession; + this.localSession = localSession; this.metrics = metrics; this.fileCache.addListener(this::nodeChanged); this.fileCache.start(); @@ -44,24 +48,24 @@ public class SessionStateWatcher { } private void sessionStatusChanged(Status newStatus) { - long sessionId = session.getSessionId(); + long sessionId = remoteSession.getSessionId(); switch (newStatus) { case NEW: case NONE: break; case PREPARE: createLocalSession(sessionId); - sessionRepository.prepareRemoteSession(session); + sessionRepository.prepare(remoteSession); break; case ACTIVATE: createLocalSession(sessionId); - sessionRepository.activate(session); + sessionRepository.activate(remoteSession); break; case DEACTIVATE: - sessionRepository.deactivate(session); + sessionRepository.deactivate(remoteSession); break; case DELETE: - sessionRepository.deleteSession(session); + sessionRepository.delete(remoteSession, localSession); break; default: throw new IllegalStateException("Unknown status " + newStatus); @@ -69,13 +73,13 @@ public class SessionStateWatcher { } private void createLocalSession(long sessionId) { - if (sessionRepository.distributeApplicationPackage()) { - sessionRepository.createLocalSessionUsingDistributedApplicationPackage(sessionId); + if (sessionRepository.distributeApplicationPackage() && localSession.isEmpty()) { + localSession = sessionRepository.createLocalSessionUsingDistributedApplicationPackage(sessionId); } } public long getSessionId() { - return session.getSessionId(); + return remoteSession.getSessionId(); } public void close() { @@ -93,21 +97,20 @@ public class SessionStateWatcher { ChildData node = fileCache.getCurrentData(); if (node != null) { newStatus = Status.parse(Utf8.toString(node.getData())); - final String statusName = newStatus.name(); - log.log(Level.FINE, () -> session.logPre() + "Session change: Session " - + getSessionId() + " changed status to " + statusName); + log.log(Level.FINE, remoteSession.logPre() + "Session change: Session " + + remoteSession.getSessionId() + " changed status to " + newStatus.name()); sessionStatusChanged(newStatus); } } catch (Exception e) { - log.log(Level.WARNING, session.logPre() + "Error handling session change to " + + log.log(Level.WARNING, remoteSession.logPre() + "Error handling session change to " + newStatus.name() + " for session " + getSessionId(), e); metrics.incSessionChangeErrors(); } }); } - void setSession(RemoteSession session) { - this.session = session; + void addLocalSession(LocalSession session) { + localSession = Optional.of(session); } } diff --git a/configserver/src/test/apps/app-major-version-2/deployment.xml b/configserver/src/test/apps/app-major-version-2/deployment.xml deleted file mode 100644 index 7523c104b7e..00000000000 --- a/configserver/src/test/apps/app-major-version-2/deployment.xml +++ /dev/null @@ -1 +0,0 @@ -<deployment version='1.0' major-version='2'/> diff --git a/configserver/src/test/apps/app-major-version-2/hosts.xml b/configserver/src/test/apps/app-major-version-2/hosts.xml deleted file mode 100644 index f4256c9fc81..00000000000 --- a/configserver/src/test/apps/app-major-version-2/hosts.xml +++ /dev/null @@ -1,7 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<hosts> - <host name="mytesthost"> - <alias>node1</alias> - </host> -</hosts> diff --git a/configserver/src/test/apps/app-major-version-2/searchdefinitions/music.sd b/configserver/src/test/apps/app-major-version-2/searchdefinitions/music.sd deleted file mode 100644 index 7670e78f22b..00000000000 --- a/configserver/src/test/apps/app-major-version-2/searchdefinitions/music.sd +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -# A basic search definition - called music, should be saved to music.sd -search music { - - # It contains one document type only - called music as well - document music { - - field title type string { - indexing: summary | index # How this field should be indexed - # index-to: title, default # Create two indexes - weight: 75 # Ranking importancy of this field, used by the built in nativeRank feature - } - - field artist type string { - indexing: summary | attribute | index - # index-to: artist, default - - weight: 25 - } - - field year type int { - indexing: summary | attribute - } - - # Increase query - field popularity type int { - indexing: summary | attribute - } - - field url type uri { - indexing: summary | index - } - - } - - rank-profile default inherits default { - first-phase { - expression: nativeRank(title,artist) + attribute(popularity) - } - - } - - rank-profile textmatch inherits default { - first-phase { - expression: nativeRank(title,artist) - } - - } - -} diff --git a/configserver/src/test/apps/app-major-version-2/services.xml b/configserver/src/test/apps/app-major-version-2/services.xml deleted file mode 100644 index 509d7786be0..00000000000 --- a/configserver/src/test/apps/app-major-version-2/services.xml +++ /dev/null @@ -1,38 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<services version="1.0"> - - <admin version="2.0"> - <adminserver hostalias="node1"/> - <logserver hostalias="node1" /> - </admin> - - <content version="1.0"> - <redundancy>2</redundancy> - <documents> - <document type="music" mode="index"/> - </documents> - <nodes> - <node hostalias="node1" distribution-key="0"/> - </nodes> - - </content> - - <container version="1.0"> - <document-processing compressdocuments="true"> - <chain id="ContainerWrapperTest"> - <documentprocessor id="com.yahoo.vespa.config.AppleDocProc"/> - </chain> - </document-processing> - - <config name="project.specific"> - <value>someval</value> - </config> - - <nodes> - <node hostalias="node1" /> - </nodes> - - </container> - -</services> diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java index b7310917449..d84b81bd8e7 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java @@ -226,7 +226,7 @@ public class ApplicationHandlerTest { ApplicationId unknown = new ApplicationId.Builder().applicationName("unknown").tenant("default").build(); HttpResponse responseForUnknown = fileDistributionStatus(unknown, zone); assertEquals(404, responseForUnknown.getStatus()); - assertEquals("{\"error-code\":\"NOT_FOUND\",\"message\":\"No active session found for 'default.unknown'\"}", + assertEquals("{\"error-code\":\"NOT_FOUND\",\"message\":\"Unknown application id 'default.unknown'\"}", getRenderedString(responseForUnknown)); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java index d8d20d37551..ae6bd5feeab 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java @@ -64,8 +64,7 @@ public class RpcServerTest { ApplicationRepository applicationRepository = tester.applicationRepository(); applicationRepository.deploy(testApp, new PrepareParams.Builder().applicationId(applicationId).build()); TenantApplications applicationRepo = tester.tenant().getApplicationRepo(); - ApplicationSet applicationSet = tester.tenant().getSessionRepository().ensureApplicationLoaded(applicationRepository.getActiveSession(applicationId)); - applicationRepo.reloadConfig(applicationSet); + applicationRepo.reloadConfig(applicationRepository.getActiveSession(applicationId).ensureApplicationLoaded()); testPrintStatistics(tester); testGetConfig(tester); testEnabled(tester); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/RemoteSessionTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/RemoteSessionTest.java new file mode 100644 index 00000000000..cda3e09d3a8 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/RemoteSessionTest.java @@ -0,0 +1,299 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.session; + +import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.component.Version; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.NullConfigModelRegistry; +import com.yahoo.config.model.api.Model; +import com.yahoo.config.model.api.ModelContext; +import com.yahoo.config.model.api.ModelCreateResult; +import com.yahoo.config.model.api.ModelFactory; +import com.yahoo.config.model.api.ValidationParameters; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.config.server.TestComponentRegistry; +import com.yahoo.vespa.config.server.application.ApplicationSet; +import com.yahoo.vespa.config.server.application.PermanentApplicationPackage; +import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry; +import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.VespaModelFactory; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.IOException; +import java.time.Clock; +import java.time.Instant; +import java.time.LocalDate; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Optional; + +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +/** + * @author Ulf Lilleengen + * @author bratseth + */ +public class RemoteSessionTest { + + private static final TenantName tenantName = TenantName.from("default"); + + private Curator curator; + + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + @Before + public void setupTest() { + curator = new MockCurator(); + } + + @Test + public void require_that_session_is_initialized() { + Clock clock = Clock.systemUTC(); + Session session = createSession(2, clock); + assertThat(session.getSessionId(), is(2L)); + session = createSession(Long.MAX_VALUE, clock); + assertThat(session.getSessionId(), is(Long.MAX_VALUE)); + } + + @Test + public void require_that_applications_are_loaded() { + RemoteSession session = createSession(3, Arrays.asList(new MockModelFactory(), new VespaModelFactory(new NullConfigModelRegistry()))); + session.prepare(); + ApplicationSet applicationSet = session.ensureApplicationLoaded(); + assertNotNull(applicationSet); + assertThat(applicationSet.getApplicationGeneration(), is(3L)); + assertThat(applicationSet.getForVersionOrLatest(Optional.empty(), Instant.now()).getId().application().value(), is("foo")); + assertNotNull(applicationSet.getForVersionOrLatest(Optional.empty(), Instant.now()).getModel()); + session.deactivate(); + + applicationSet = session.ensureApplicationLoaded(); + assertNotNull(applicationSet); + assertThat(applicationSet.getApplicationGeneration(), is(3L)); + assertThat(applicationSet.getForVersionOrLatest(Optional.empty(), Instant.now()).getId().application().value(), is("foo")); + assertNotNull(applicationSet.getForVersionOrLatest(Optional.empty(), Instant.now()).getModel()); + } + + @Test(expected = IllegalArgumentException.class) + public void require_that_new_invalid_application_throws_exception() { + MockModelFactory failingFactory = new MockModelFactory(); + failingFactory.vespaVersion = new Version(1, 2, 0); + failingFactory.throwOnLoad = true; + + MockModelFactory okFactory = new MockModelFactory(); + okFactory.vespaVersion = new Version(1, 1, 0); + okFactory.throwOnLoad = false; + + RemoteSession session = createSession(3, Arrays.asList(okFactory, failingFactory)); + session.prepare(); + } + + @Test + public void require_that_old_invalid_application_does_not_throw_exception_if_skipped_also_across_major_versions() { + MockModelFactory failingFactory = new MockModelFactory(); + failingFactory.vespaVersion = new Version(1, 0, 0); + failingFactory.throwOnLoad = true; + + MockModelFactory okFactory = + new MockModelFactory("<validation-overrides><allow until='2000-01-30'>skip-old-config-models</allow></validation-overrides>"); + okFactory.vespaVersion = new Version(2, 0, 0); + okFactory.throwOnLoad = false; + + RemoteSession session = createSession(3, Arrays.asList(okFactory, failingFactory), failingFactory.clock()); + session.prepare(); + } + + @Test + public void require_that_an_application_package_can_limit_to_one_major_version() { + ApplicationPackage application = + new MockApplicationPackage.Builder().withServices("<services version='1.0'/>") + .withDeploymentSpec("<deployment version='1.0' major-version='2'/>") + .build(); + assertTrue(application.getMajorVersion().isPresent()); + assertEquals(2, (int)application.getMajorVersion().get()); + + MockModelFactory failingFactory = new MockModelFactory(); + failingFactory.vespaVersion = new Version(3, 0, 0); + failingFactory.throwErrorOnLoad = true; + + MockModelFactory okFactory = new MockModelFactory(); + okFactory.vespaVersion = new Version(2, 0, 0); + okFactory.throwErrorOnLoad = false; + + SessionZooKeeperClient zkc = new MockSessionZKClient(curator, tenantName, 3, application); + RemoteSession session = createSession(3, zkc, Arrays.asList(okFactory, failingFactory)); + session.prepare(); + + // Does not cause an error because model version 3 is skipped + } + + @Test + public void require_that_an_application_package_can_limit_to_one_higher_major_version() { + ApplicationPackage application = + new MockApplicationPackage.Builder().withServices("<services version='1.0'/>") + .withDeploymentSpec("<deployment version='1.0' major-version='3'/>") + .build(); + assertTrue(application.getMajorVersion().isPresent()); + assertEquals(3, (int)application.getMajorVersion().get()); + + MockModelFactory failingFactory = new MockModelFactory(); + failingFactory.vespaVersion = new Version(4, 0, 0); + failingFactory.throwErrorOnLoad = true; + + MockModelFactory okFactory = new MockModelFactory(); + okFactory.vespaVersion = new Version(2, 0, 0); + okFactory.throwErrorOnLoad = false; + + SessionZooKeeperClient zkc = new MockSessionZKClient(curator, tenantName, 3, application); + RemoteSession session = createSession(4, zkc, Arrays.asList(okFactory, failingFactory)); + session.prepare(); + + // Does not cause an error because model version 4 is skipped + } + + @Test + public void require_that_session_status_is_updated() { + SessionZooKeeperClient zkc = new MockSessionZKClient(curator, tenantName, 3); + RemoteSession session = createSession(3, zkc, Clock.systemUTC()); + assertThat(session.getStatus(), is(Session.Status.NEW)); + zkc.writeStatus(Session.Status.PREPARE); + assertThat(session.getStatus(), is(Session.Status.PREPARE)); + } + + @Test + public void require_that_permanent_app_is_used() throws IOException { + Optional<PermanentApplicationPackage> permanentApp = Optional.of(new PermanentApplicationPackage( + new ConfigserverConfig(new ConfigserverConfig.Builder() + .applicationDirectory(temporaryFolder.newFolder("appdir").getAbsolutePath())))); + MockModelFactory mockModelFactory = new MockModelFactory(); + try { + int sessionId = 3; + SessionZooKeeperClient zkc = new MockSessionZKClient(curator, tenantName, sessionId); + createSession(sessionId, zkc, Collections.singletonList(mockModelFactory), permanentApp, Clock.systemUTC()).ensureApplicationLoaded(); + } catch (Exception e) { + e.printStackTrace(); + // ignore, we're not interested in deploy errors as long as the below state is OK. + } + assertNotNull(mockModelFactory.modelContext); + assertTrue(mockModelFactory.modelContext.permanentApplicationPackage().isPresent()); + } + + private RemoteSession createSession(long sessionId, Clock clock) { + return createSession(sessionId, Collections.singletonList(new VespaModelFactory(new NullConfigModelRegistry())), clock); + } + + private RemoteSession createSession(long sessionId, SessionZooKeeperClient zkc, Clock clock) { + return createSession(sessionId, zkc, Collections.singletonList(new VespaModelFactory(new NullConfigModelRegistry())), clock); + } + + private RemoteSession createSession(long sessionId, List<ModelFactory> modelFactories) { + SessionZooKeeperClient zkc = new MockSessionZKClient(curator, tenantName, sessionId); + return createSession(sessionId, zkc, modelFactories, Clock.systemUTC()); + } + + private RemoteSession createSession(long sessionId, List<ModelFactory> modelFactories, Clock clock) { + SessionZooKeeperClient zkc = new MockSessionZKClient(curator, tenantName, sessionId); + return createSession(sessionId, zkc, modelFactories, clock); + } + + private RemoteSession createSession(long sessionId, SessionZooKeeperClient zkc, List<ModelFactory> modelFactories) { + return createSession(sessionId, zkc, modelFactories, Optional.empty(), Clock.systemUTC()); + } + + private RemoteSession createSession(long sessionId, SessionZooKeeperClient zkc, List<ModelFactory> modelFactories, Clock clock) { + return createSession(sessionId, zkc, modelFactories, Optional.empty(), clock); + } + + private RemoteSession createSession(long sessionId, SessionZooKeeperClient zkc, + List<ModelFactory> modelFactories, + Optional<PermanentApplicationPackage> permanentApplicationPackage, + Clock clock) { + zkc.writeStatus(Session.Status.NEW); + zkc.writeApplicationId(new ApplicationId.Builder().applicationName("foo").instanceName("bim").build()); + TestComponentRegistry.Builder registryBuilder = new TestComponentRegistry.Builder() + .curator(curator) + .clock(clock) + .modelFactoryRegistry(new ModelFactoryRegistry(modelFactories)); + permanentApplicationPackage.ifPresent(registryBuilder::permanentApplicationPackage); + + return new RemoteSession(tenantName, sessionId, registryBuilder.build(), zkc); + } + + private static class MockModelFactory implements ModelFactory { + + /** Throw a RuntimeException on load - this is handled gracefully during model building */ + boolean throwOnLoad = false; + + /** Throw an Error on load - this is useful to propagate this condition all the way to the test */ + boolean throwErrorOnLoad = false; + + ModelContext modelContext; + public Version vespaVersion = new Version(1, 2, 3); + + /** The validation overrides of this, or null if none */ + private final String validationOverrides; + + private final Clock clock = Clock.fixed(LocalDate.parse("2000-01-01", DateTimeFormatter.ISO_DATE).atStartOfDay().atZone(ZoneOffset.UTC).toInstant(), ZoneOffset.UTC); + + MockModelFactory() { this(null); } + + MockModelFactory(String validationOverrides) { + this.validationOverrides = validationOverrides; + } + + @Override + public Version version() { + return vespaVersion; + } + + /** Returns the clock used by this, which is fixed at the instant 2000-01-01T00:00:00 */ + public Clock clock() { return clock; } + + @Override + public Model createModel(ModelContext modelContext) { + if (throwErrorOnLoad) + throw new Error("Foo"); + if (throwOnLoad) + throw new IllegalArgumentException("Foo"); + this.modelContext = modelContext; + return loadModel(); + } + + Model loadModel() { + try { + ApplicationPackage application = new MockApplicationPackage.Builder().withEmptyHosts().withEmptyServices().withValidationOverrides(validationOverrides).build(); + DeployState deployState = new DeployState.Builder().applicationPackage(application).now(clock.instant()).build(); + return new VespaModel(deployState); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + public ModelCreateResult createAndValidateModel(ModelContext modelContext, ValidationParameters validationParameters) { + if (throwErrorOnLoad) + throw new Error("Foo"); + if (throwOnLoad) + throw new IllegalArgumentException("Foo"); + this.modelContext = modelContext; + return new ModelCreateResult(loadModel(), new ArrayList<>()); + } + } + +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionRepositoryTest.java index d8adbd398d1..b89b63aed46 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionRepositoryTest.java @@ -2,26 +2,14 @@ package com.yahoo.vespa.config.server.session; import com.yahoo.cloud.config.ConfigserverConfig; -import com.yahoo.component.Version; -import com.yahoo.config.application.api.ApplicationPackage; -import com.yahoo.config.model.api.Model; -import com.yahoo.config.model.api.ModelContext; -import com.yahoo.config.model.api.ModelCreateResult; -import com.yahoo.config.model.api.ModelFactory; -import com.yahoo.config.model.api.ValidationParameters; -import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; import com.yahoo.text.Utf8; import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.GlobalComponentRegistry; import com.yahoo.vespa.config.server.TestComponentRegistry; -import com.yahoo.vespa.config.server.application.ApplicationSet; import com.yahoo.vespa.config.server.application.OrchestratorMock; -import com.yahoo.vespa.config.server.http.InvalidApplicationException; import com.yahoo.vespa.config.server.http.SessionHandlerTest; -import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry; import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; import com.yahoo.vespa.config.util.ConfigUtils; @@ -29,25 +17,16 @@ import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.flags.InMemoryFlagSource; -import com.yahoo.vespa.model.VespaModel; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; import java.io.File; -import java.time.Clock; import java.time.Duration; import java.time.Instant; -import java.time.LocalDate; -import java.time.ZoneOffset; -import java.time.format.DateTimeFormatter; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; import java.util.function.LongPredicate; import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertThat; @@ -74,13 +53,9 @@ public class SessionRepositoryTest { } private void setup(FlagSource flagSource) throws Exception { - setup(flagSource, new TestComponentRegistry.Builder()); - } - - private void setup(FlagSource flagSource, TestComponentRegistry.Builder componentRegistryBuilder) throws Exception { curator = new MockCurator(); File configserverDbDir = temporaryFolder.newFolder().getAbsoluteFile(); - GlobalComponentRegistry globalComponentRegistry = componentRegistryBuilder + GlobalComponentRegistry globalComponentRegistry = new TestComponentRegistry.Builder() .curator(curator) .configServerConfig(new ConfigserverConfig.Builder() .configServerDBDir(configserverDbDir.getAbsolutePath()) @@ -100,7 +75,6 @@ public class SessionRepositoryTest { sessionRepository = tenantRepository.getTenant(tenantName).getSessionRepository(); } - @Test public void require_that_local_sessions_are_created_and_deleted() throws Exception { setup(); @@ -110,12 +84,6 @@ public class SessionRepositoryTest { assertNotNull(sessionRepository.getLocalSession(secondSessionId)); assertNull(sessionRepository.getLocalSession(secondSessionId + 1)); - ApplicationSet applicationSet = sessionRepository.ensureApplicationLoaded(sessionRepository.getRemoteSession(firstSessionId)); - assertNotNull(applicationSet); - assertEquals(2, applicationSet.getApplicationGeneration()); - assertEquals(applicationId.application(), applicationSet.getForVersionOrLatest(Optional.empty(), Instant.now()).getId().application()); - assertNotNull(applicationSet.getForVersionOrLatest(Optional.empty(), Instant.now()).getModel()); - sessionRepository.close(); // All created sessions are deleted assertNull(sessionRepository.getLocalSession(firstSessionId)); @@ -182,81 +150,6 @@ public class SessionRepositoryTest { assertThat(sessionRepository.getRemoteSessions().size(), is(1)); } - @Test(expected = InvalidApplicationException.class) - public void require_that_new_invalid_application_throws_exception() throws Exception { - MockModelFactory failingFactory = new MockModelFactory(); - failingFactory.vespaVersion = new Version(1, 2, 0); - failingFactory.throwOnLoad = true; - - MockModelFactory okFactory = new MockModelFactory(); - okFactory.vespaVersion = new Version(1, 1, 0); - okFactory.throwOnLoad = false; - - TestComponentRegistry.Builder registryBuilder = new TestComponentRegistry.Builder() - .modelFactoryRegistry(new ModelFactoryRegistry(List.of(okFactory, failingFactory))); - setup(new InMemoryFlagSource(), registryBuilder); - - deploy(); - } - - @Test - public void require_that_old_invalid_application_does_not_throw_exception_if_skipped_also_across_major_versions() throws Exception { - MockModelFactory failingFactory = new MockModelFactory(); - failingFactory.vespaVersion = new Version(1, 0, 0); - failingFactory.throwOnLoad = true; - - MockModelFactory okFactory = - new MockModelFactory("<validation-overrides><allow until='2000-01-30'>skip-old-config-models</allow></validation-overrides>"); - okFactory.vespaVersion = new Version(2, 0, 0); - okFactory.throwOnLoad = false; - - TestComponentRegistry.Builder registryBuilder = new TestComponentRegistry.Builder() - .modelFactoryRegistry(new ModelFactoryRegistry(List.of(okFactory, failingFactory))); - setup(new InMemoryFlagSource(), registryBuilder); - - deploy(); - } - - @Test - public void require_that_an_application_package_can_limit_to_one_major_version() throws Exception { - MockModelFactory failingFactory = new MockModelFactory(); - failingFactory.vespaVersion = new Version(3, 0, 0); - failingFactory.throwErrorOnLoad = true; - - MockModelFactory okFactory = new MockModelFactory(); - okFactory.vespaVersion = new Version(2, 0, 0); - okFactory.throwErrorOnLoad = false; - - TestComponentRegistry.Builder registryBuilder = new TestComponentRegistry.Builder() - .modelFactoryRegistry(new ModelFactoryRegistry(List.of(okFactory, failingFactory))); - setup(new InMemoryFlagSource(), registryBuilder); - - File testApp = new File("src/test/apps/app-major-version-2"); - deploy(applicationId, testApp); - - // Does not cause an error because model version 3 is skipped - } - - @Test - public void require_that_an_application_package_can_limit_to_one_higher_major_version() throws Exception { - MockModelFactory failingFactory = new MockModelFactory(); - failingFactory.vespaVersion = new Version(3, 0, 0); - failingFactory.throwErrorOnLoad = true; - - MockModelFactory okFactory = new MockModelFactory(); - okFactory.vespaVersion = new Version(1, 0, 0); - okFactory.throwErrorOnLoad = false; - - TestComponentRegistry.Builder registryBuilder = new TestComponentRegistry.Builder() - .modelFactoryRegistry(new ModelFactoryRegistry(List.of(okFactory, failingFactory))); - setup(new InMemoryFlagSource(), registryBuilder); - - File testApp = new File("src/test/apps/app-major-version-2"); - deploy(applicationId, testApp); - - // Does not cause an error because model version 3 is skipped - } - private void createSession(long sessionId, boolean wait) { SessionZooKeeperClient zkc = new SessionZooKeeperClient(curator, ConfigCurator.create(curator), @@ -311,74 +204,8 @@ public class SessionRepositoryTest { } private long deploy(ApplicationId applicationId) { - return deploy(applicationId, testApp); - } - - private long deploy(ApplicationId applicationId, File testApp) { applicationRepository.deploy(testApp, new PrepareParams.Builder().applicationId(applicationId).build()); return applicationRepository.getActiveSession(applicationId).getSessionId(); } - private static class MockModelFactory implements ModelFactory { - - /** Throw a RuntimeException on load - this is handled gracefully during model building */ - boolean throwOnLoad = false; - - /** Throw an Error on load - this is useful to propagate this condition all the way to the test */ - boolean throwErrorOnLoad = false; - - ModelContext modelContext; - public Version vespaVersion = new Version(1, 2, 3); - - /** The validation overrides of this, or null if none */ - private final String validationOverrides; - - private final Clock clock = Clock.fixed(LocalDate.parse("2000-01-01", DateTimeFormatter.ISO_DATE).atStartOfDay().atZone(ZoneOffset.UTC).toInstant(), ZoneOffset.UTC); - - MockModelFactory() { this(null); } - - MockModelFactory(String validationOverrides) { - this.validationOverrides = validationOverrides; - } - - @Override - public Version version() { - return vespaVersion; - } - - /** Returns the clock used by this, which is fixed at the instant 2000-01-01T00:00:00 */ - public Clock clock() { return clock; } - - @Override - public Model createModel(ModelContext modelContext) { - if (throwErrorOnLoad) - throw new Error("error on load"); - if (throwOnLoad) - throw new IllegalArgumentException("exception on load"); - this.modelContext = modelContext; - return loadModel(); - } - - Model loadModel() { - try { - ApplicationPackage application = new MockApplicationPackage.Builder().withEmptyHosts().withEmptyServices().withValidationOverrides(validationOverrides).build(); - DeployState deployState = new DeployState.Builder().applicationPackage(application).now(clock.instant()).build(); - return new VespaModel(deployState); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - @Override - public ModelCreateResult createAndValidateModel(ModelContext modelContext, ValidationParameters validationParameters) { - if (throwErrorOnLoad) - throw new Error("error on load"); - if (throwOnLoad) - throw new IllegalArgumentException("exception on load"); - this.modelContext = modelContext; - return new ModelCreateResult(loadModel(), new ArrayList<>()); - } - } - - } |