diff options
Diffstat (limited to 'configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java')
| -rw-r--r-- | configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java | 253 |
1 files changed, 239 insertions, 14 deletions
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java index 9fe725453f4..c0d09cfba30 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java @@ -3,19 +3,31 @@ package com.yahoo.vespa.config.server.session; import com.google.common.collect.HashMultiset; import com.google.common.collect.Multiset; +import com.yahoo.config.FileReference; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.application.provider.DeployData; +import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.TenantName; +import com.yahoo.io.IOUtils; import com.yahoo.path.Path; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.config.server.GlobalComponentRegistry; import com.yahoo.vespa.config.server.ReloadHandler; +import com.yahoo.vespa.config.server.TimeoutBudget; import com.yahoo.vespa.config.server.application.TenantApplications; import com.yahoo.vespa.config.server.deploy.TenantFileSystemDirs; +import com.yahoo.vespa.config.server.filedistribution.FileDirectory; +import com.yahoo.vespa.config.server.host.HostValidator; import com.yahoo.vespa.config.server.monitoring.MetricUpdater; import com.yahoo.vespa.config.server.monitoring.Metrics; import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; +import com.yahoo.vespa.config.server.zookeeper.SessionCounter; import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.flags.BooleanFlag; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.flags.Flags; @@ -25,6 +37,7 @@ import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; import java.io.File; import java.io.FilenameFilter; +import java.io.IOException; import java.time.Clock; import java.time.Duration; import java.time.Instant; @@ -33,6 +46,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.concurrent.Executor; import java.util.logging.Level; import java.util.logging.Logger; @@ -51,9 +65,9 @@ public class SessionRepository { private static final Logger log = Logger.getLogger(SessionRepository.class.getName()); private static final FilenameFilter sessionApplicationsFilter = (dir, name) -> name.matches("\\d+"); + private static final long nonExistingActiveSession = 0; private final SessionCache<LocalSession> localSessionCache; - private final SessionCache<RemoteSession> remoteSessionCache = new SessionCache<>(); private final Map<Long, LocalSessionStateWatcher> localSessionStateWatchers = new HashMap<>(); private final Map<Long, RemoteSessionStateWatcher> remoteSessionStateWatchers = new HashMap<>(); @@ -67,28 +81,36 @@ public class SessionRepository { private final MetricUpdater metrics; private final Curator.DirectoryCache directoryCache; private final TenantApplications applicationRepo; + private final HostValidator<ApplicationId> hostRegistry; + private final SessionPreparer sessionPreparer; private final Path sessionsPath; - private final SessionFactory sessionFactory; private final TenantName tenantName; - - public SessionRepository(TenantName tenantName, GlobalComponentRegistry componentRegistry, - SessionFactory sessionFactory, TenantApplications applicationRepo, - ReloadHandler reloadHandler, FlagSource flagSource) { + private final GlobalComponentRegistry componentRegistry; + + public SessionRepository(TenantName tenantName, + GlobalComponentRegistry componentRegistry, + TenantApplications applicationRepo, + ReloadHandler reloadHandler, + FlagSource flagSource, + HostValidator<ApplicationId> hostRegistry, + SessionPreparer sessionPreparer) { this.tenantName = tenantName; + this.componentRegistry = componentRegistry; + this.sessionsPath = TenantRepository.getSessionsPath(tenantName); localSessionCache = new SessionCache<>(); this.clock = componentRegistry.getClock(); this.curator = componentRegistry.getCurator(); this.sessionLifetime = Duration.ofSeconds(componentRegistry.getConfigserverConfig().sessionLifetime()); this.zkWatcherExecutor = command -> componentRegistry.getZkWatcherExecutor().execute(tenantName, command); this.tenantFileSystemDirs = new TenantFileSystemDirs(componentRegistry.getConfigServerDB(), tenantName); - this.sessionFactory = sessionFactory; this.applicationRepo = applicationRepo; - loadLocalSessions(sessionFactory); - + this.hostRegistry = hostRegistry; + this.sessionPreparer = sessionPreparer; this.distributeApplicationPackage = Flags.CONFIGSERVER_DISTRIBUTE_APPLICATION_PACKAGE.bindTo(flagSource); this.reloadHandler = reloadHandler; - this.sessionsPath = TenantRepository.getSessionsPath(tenantName); this.metrics = componentRegistry.getMetrics().getOrCreateMetricUpdater(Metrics.createDimensions(tenantName)); + + loadLocalSessions(); initializeRemoteSessions(); this.directoryCache = curator.createDirectoryCache(sessionsPath.getAbsolute(), false, false, componentRegistry.getZkCacheExecutor()); this.directoryCache.addListener(this::childEvent); @@ -113,14 +135,14 @@ public class SessionRepository { return localSessionCache.getSessions(); } - private void loadLocalSessions(SessionFactory sessionFactory) { + private void loadLocalSessions() { File[] sessions = tenantFileSystemDirs.sessionsPath().listFiles(sessionApplicationsFilter); if (sessions == null) { return; } for (File session : sessions) { try { - addSession(sessionFactory.createSessionFromId(Long.parseLong(session.getName()))); + addSession(createSessionFromId(Long.parseLong(session.getName()))); } catch (IllegalArgumentException e) { log.log(Level.WARNING, "Could not load session '" + session.getAbsolutePath() + "':" + e.getMessage() + ", skipping it."); @@ -269,7 +291,7 @@ public class SessionRepository { */ private void sessionAdded(long sessionId) { log.log(Level.FINE, () -> "Adding session to SessionRepository: " + sessionId); - RemoteSession session = sessionFactory.createRemoteSession(sessionId); + RemoteSession session = createRemoteSession(sessionId); Path sessionPath = sessionsPath.append(String.valueOf(sessionId)); Curator.FileCache fileCache = curator.createFileCache(sessionPath.append(ConfigCurator.SESSIONSTATE_ZK_SUBPATH).getAbsolute(), false); fileCache.addListener(this::nodeChanged); @@ -277,7 +299,7 @@ public class SessionRepository { addRemoteSession(session); remoteSessionStateWatchers.put(sessionId, new RemoteSessionStateWatcher(fileCache, reloadHandler, session, metrics, zkWatcherExecutor)); if (distributeApplicationPackage.value()) - sessionFactory.createLocalSessionUsingDistributedApplicationPackage(sessionId); + createLocalSessionUsingDistributedApplicationPackage(sessionId); } private void sessionRemoved(long sessionId) { @@ -339,6 +361,209 @@ public class SessionRepository { } } + /** + * Creates a new deployment session from an application package. + * + * @param applicationDirectory a File pointing to an application. + * @param applicationId application id for this new session. + * @param timeoutBudget Timeout for creating session and waiting for other servers. + * @return a new session + */ + public LocalSession createSession(File applicationDirectory, ApplicationId applicationId, + TimeoutBudget timeoutBudget, Optional<Long> activeSessionId) { + return create(applicationDirectory, applicationId, activeSessionId.orElse(nonExistingActiveSession), false, timeoutBudget); + } + + public RemoteSession createRemoteSession(long sessionId) { + SessionZooKeeperClient sessionZKClient = createSessionZooKeeperClient(getSessionPath(sessionId)); + return new RemoteSession(tenantName, sessionId, componentRegistry, sessionZKClient); + } + + private void ensureSessionPathDoesNotExist(long sessionId) { + Path sessionPath = getSessionPath(sessionId); + if (componentRegistry.getConfigCurator().exists(sessionPath.getAbsolute())) { + throw new IllegalArgumentException("Path " + sessionPath.getAbsolute() + " already exists in ZooKeeper"); + } + } + + private ApplicationPackage createApplication(File userDir, + File configApplicationDir, + ApplicationId applicationId, + long sessionId, + long currentlyActiveSessionId, + boolean internalRedeploy) { + long deployTimestamp = System.currentTimeMillis(); + String user = System.getenv("USER"); + if (user == null) { + user = "unknown"; + } + DeployData deployData = new DeployData(user, userDir.getAbsolutePath(), applicationId, deployTimestamp, internalRedeploy, sessionId, currentlyActiveSessionId); + return FilesApplicationPackage.fromFileWithDeployData(configApplicationDir, deployData); + } + + private LocalSession createSessionFromApplication(ApplicationPackage applicationPackage, + long sessionId, + TimeoutBudget timeoutBudget, + Clock clock) { + log.log(Level.FINE, TenantRepository.logPre(tenantName) + "Creating session " + sessionId + " in ZooKeeper"); + SessionZooKeeperClient sessionZKClient = createSessionZooKeeperClient(getSessionPath(sessionId)); + sessionZKClient.createNewSession(clock.instant()); + Curator.CompletionWaiter waiter = sessionZKClient.getUploadWaiter(); + LocalSession session = new LocalSession(tenantName, sessionId, sessionPreparer, applicationPackage, sessionZKClient, + getSessionAppDir(sessionId), applicationRepo, hostRegistry); + waiter.awaitCompletion(timeoutBudget.timeLeft()); + return session; + } + + /** + * Creates a new deployment session from an already existing session. + * + * @param existingSession the session to use as base + * @param logger a deploy logger where the deploy log will be written. + * @param internalRedeploy whether this session is for a system internal redeploy — not an application package change + * @param timeoutBudget timeout for creating session and waiting for other servers. + * @return a new session + */ + public LocalSession createSessionFromExisting(Session existingSession, + DeployLogger logger, + boolean internalRedeploy, + TimeoutBudget timeoutBudget) { + File existingApp = getSessionAppDir(existingSession.getSessionId()); + ApplicationId existingApplicationId = existingSession.getApplicationId(); + + long activeSessionId = getActiveSessionId(existingApplicationId); + logger.log(Level.FINE, "Create new session for application id '" + existingApplicationId + "' from existing active session " + activeSessionId); + LocalSession session = create(existingApp, existingApplicationId, activeSessionId, internalRedeploy, timeoutBudget); + // Note: Needs to be kept in sync with calls in SessionPreparer.writeStateToZooKeeper() + session.setApplicationId(existingApplicationId); + if (distributeApplicationPackage.value() && existingSession.getApplicationPackageReference() != null) { + session.setApplicationPackageReference(existingSession.getApplicationPackageReference()); + } + session.setVespaVersion(existingSession.getVespaVersion()); + session.setDockerImageRepository(existingSession.getDockerImageRepository()); + session.setAthenzDomain(existingSession.getAthenzDomain()); + return session; + } + + private LocalSession create(File applicationFile, ApplicationId applicationId, long currentlyActiveSessionId, + boolean internalRedeploy, TimeoutBudget timeoutBudget) { + long sessionId = getNextSessionId(); + try { + ensureSessionPathDoesNotExist(sessionId); + ApplicationPackage app = createApplicationPackage(applicationFile, applicationId, + sessionId, currentlyActiveSessionId, internalRedeploy); + return createSessionFromApplication(app, sessionId, timeoutBudget, clock); + } catch (Exception e) { + throw new RuntimeException("Error creating session " + sessionId, e); + } + } + + /** + * This method is used when creating a session based on a remote session and the distributed application package + * It does not wait for session being created on other servers + */ + private LocalSession createLocalSession(File applicationFile, ApplicationId applicationId, + long sessionId, long currentlyActiveSessionId) { + try { + ApplicationPackage applicationPackage = createApplicationPackage(applicationFile, applicationId, + sessionId, currentlyActiveSessionId, false); + SessionZooKeeperClient sessionZooKeeperClient = createSessionZooKeeperClient(getSessionPath(sessionId)); + return new LocalSession(tenantName, sessionId, sessionPreparer, applicationPackage, sessionZooKeeperClient, + getSessionAppDir(sessionId), applicationRepo, hostRegistry); + } catch (Exception e) { + throw new RuntimeException("Error creating session " + sessionId, e); + } + } + + private ApplicationPackage createApplicationPackage(File applicationFile, ApplicationId applicationId, + long sessionId, long currentlyActiveSessionId, + boolean internalRedeploy) throws IOException { + File userApplicationDir = getSessionAppDir(sessionId); + IOUtils.copyDirectory(applicationFile, userApplicationDir); + ApplicationPackage applicationPackage = createApplication(applicationFile, + userApplicationDir, + applicationId, + sessionId, + currentlyActiveSessionId, + internalRedeploy); + applicationPackage.writeMetaData(); + return applicationPackage; + } + + /** + * Returns a new session instance for the given session id. + */ + LocalSession createSessionFromId(long sessionId) { + File sessionDir = getAndValidateExistingSessionAppDir(sessionId); + ApplicationPackage applicationPackage = FilesApplicationPackage.fromFile(sessionDir); + Path sessionIdPath = sessionsPath.append(String.valueOf(sessionId)); + SessionZooKeeperClient sessionZKClient = createSessionZooKeeperClient(sessionIdPath); + return new LocalSession(tenantName, sessionId, sessionPreparer, applicationPackage, sessionZKClient, + getSessionAppDir(sessionId), applicationRepo, hostRegistry); + } + + /** + * Returns a new session instance for the given session id. + */ + LocalSession createLocalSessionUsingDistributedApplicationPackage(long sessionId) { + if (applicationRepo.hasLocalSession(sessionId)) { + log.log(Level.FINE, "Local session for session id " + sessionId + " already exists"); + return createSessionFromId(sessionId); + } + + log.log(Level.INFO, "Creating local session for session id " + sessionId); + SessionZooKeeperClient sessionZKClient = createSessionZooKeeperClient(getSessionPath(sessionId)); + FileReference fileReference = sessionZKClient.readApplicationPackageReference(); + log.log(Level.FINE, "File reference for session id " + sessionId + ": " + fileReference); + if (fileReference != null) { + File rootDir = new File(Defaults.getDefaults().underVespaHome(componentRegistry.getConfigserverConfig().fileReferencesDir())); + File sessionDir = new FileDirectory(rootDir).getFile(fileReference); + if (!sessionDir.exists()) + throw new RuntimeException("File reference for session " + sessionId + " not found (" + sessionDir.getAbsolutePath() + ")"); + ApplicationId applicationId = sessionZKClient.readApplicationId(); + return createLocalSession(sessionDir, + applicationId, + sessionId, + applicationRepo.activeSessionOf(applicationId).orElse(nonExistingActiveSession)); + } + return null; + } + + // Return Optional instead of faking it with nonExistingActiveSession + private long getActiveSessionId(ApplicationId applicationId) { + List<ApplicationId> applicationIds = applicationRepo.activeApplications(); + if (applicationIds.contains(applicationId)) { + return applicationRepo.requireActiveSessionOf(applicationId); + } + return nonExistingActiveSession; + } + + private long getNextSessionId() { + return new SessionCounter(componentRegistry.getConfigCurator(), tenantName).nextSessionId(); + } + + private Path getSessionPath(long sessionId) { + return sessionsPath.append(String.valueOf(sessionId)); + } + + private SessionZooKeeperClient createSessionZooKeeperClient(Path sessionPath) { + String serverId = componentRegistry.getConfigserverConfig().serverId(); + Optional<NodeFlavors> nodeFlavors = componentRegistry.getZone().nodeFlavors(); + return new SessionZooKeeperClient(curator, componentRegistry.getConfigCurator(), sessionPath, serverId, nodeFlavors); + } + + private File getAndValidateExistingSessionAppDir(long sessionId) { + File appDir = getSessionAppDir(sessionId); + if (!appDir.exists() || !appDir.isDirectory()) { + throw new IllegalArgumentException("Unable to find correct application directory for session " + sessionId); + } + return appDir; + } + + private File getSessionAppDir(long sessionId) { + return new TenantFileSystemDirs(componentRegistry.getConfigServerDB(), tenantName).getUserApplicationDir(sessionId); + } + @Override public String toString() { return getSessions().toString(); |