diff options
Diffstat (limited to 'configserver/src/main/java')
15 files changed, 241 insertions, 205 deletions
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index ea7b6a88a9c..481f071a32e 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -53,6 +53,7 @@ import com.yahoo.vespa.config.server.session.SilentDeployLogger; import com.yahoo.vespa.config.server.tenant.Rotations; import com.yahoo.vespa.config.server.tenant.Tenant; import com.yahoo.vespa.config.server.tenant.TenantRepository; +import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.orchestrator.Orchestrator; import java.io.File; @@ -69,6 +70,7 @@ import java.util.Collection; import java.util.HashSet; import java.util.List; import java.util.Optional; +import java.util.OptionalLong; import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; @@ -262,8 +264,10 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye @Override public Optional<Instant> lastDeployTime(ApplicationId application) { Tenant tenant = tenantRepository.getTenant(application.tenant()); - if (tenant == null) return Optional.empty(); - LocalSession activeSession = getActiveSession(tenant, application); + if (tenant == null || ! tenant.getApplicationRepo().exists(application)) return Optional.empty(); + OptionalLong activeSessionId = tenant.getApplicationRepo().activeSessionOf(application); + if ( ! activeSessionId.isPresent()) return Optional.empty(); + LocalSession activeSession = tenant.getLocalSessionRepo().getSession(activeSessionId.getAsLong()); if (activeSession == null) return Optional.empty(); return Optional.of(Instant.ofEpochSecond(activeSession.getCreateTime())); } @@ -296,34 +300,35 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye if (tenant == null) return false; TenantApplications tenantApplications = tenant.getApplicationRepo(); - if (!tenantApplications.listApplications().contains(applicationId)) return false; - - // Deleting an application is done by deleting the remote session and waiting - // until the config server where the deployment happened picks it up and deletes - // the local session - long sessionId = tenantApplications.getSessionIdForApplication(applicationId); - RemoteSession remoteSession = getRemoteSession(tenant, sessionId); - remoteSession.createDeleteTransaction().commit(); - - log.log(LogLevel.INFO, TenantRepository.logPre(applicationId) + "Waiting for session " + sessionId + " to be deleted"); - // TODO: Add support for timeout in request - Duration waitTime = Duration.ofSeconds(60); - if (localSessionHasBeenDeleted(applicationId, sessionId, waitTime)) { - log.log(LogLevel.INFO, TenantRepository.logPre(applicationId) + "Session " + sessionId + " deleted"); - } else { - log.log(LogLevel.ERROR, TenantRepository.logPre(applicationId) + "Session " + sessionId + " was not deleted (waited " + waitTime + ")"); - return false; - } + try (Lock lock = tenantApplications.lock(applicationId)) { + if ( ! tenantApplications.exists(applicationId)) return false; + // Deleting an application is done by deleting the remote session and waiting + // until the config server where the deployment happened picks it up and deletes + // the local session + long sessionId = tenantApplications.requireActiveSessionOf(applicationId); + RemoteSession remoteSession = getRemoteSession(tenant, sessionId); + remoteSession.createDeleteTransaction().commit(); + + log.log(LogLevel.INFO, TenantRepository.logPre(applicationId) + "Waiting for session " + sessionId + " to be deleted"); + // TODO: Add support for timeout in request + Duration waitTime = Duration.ofSeconds(60); + if (localSessionHasBeenDeleted(applicationId, sessionId, waitTime)) { + log.log(LogLevel.INFO, TenantRepository.logPre(applicationId) + "Session " + sessionId + " deleted"); + } else { + log.log(LogLevel.ERROR, TenantRepository.logPre(applicationId) + "Session " + sessionId + " was not deleted (waited " + waitTime + ")"); + return false; + } - NestedTransaction transaction = new NestedTransaction(); - transaction.add(new Rotations(tenant.getCurator(), tenant.getPath()).delete(applicationId)); // TODO: Not unit tested - // (When rotations are updated in zk, we need to redeploy the zone app, on the right config server - // this is done asynchronously in application maintenance by the node repository) - transaction.add(tenantApplications.deleteApplication(applicationId)); + NestedTransaction transaction = new NestedTransaction(); + transaction.add(new Rotations(tenant.getCurator(), tenant.getPath()).delete(applicationId)); // TODO: Not unit tested + // (When rotations are updated in zk, we need to redeploy the zone app, on the right config server + // this is done asynchronously in application maintenance by the node repository) + transaction.add(tenantApplications.createDeleteTransaction(applicationId)); - hostProvisioner.ifPresent(provisioner -> provisioner.remove(transaction, applicationId)); - transaction.onCommitted(() -> log.log(LogLevel.INFO, "Deleted " + applicationId)); - transaction.commit(); + hostProvisioner.ifPresent(provisioner -> provisioner.remove(transaction, applicationId)); + transaction.onCommitted(() -> log.log(LogLevel.INFO, "Deleted " + applicationId)); + transaction.commit(); + } return true; } @@ -419,7 +424,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye Set<ApplicationId> listApplications() { return tenantRepository.getAllTenants().stream() - .flatMap(tenant -> tenant.getApplicationRepo().listApplications().stream()) + .flatMap(tenant -> tenant.getApplicationRepo().activeApplications().stream()) .collect(Collectors.toSet()); } @@ -478,7 +483,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye if (applicationRepo == null) throw new IllegalArgumentException("Application repo for tenant '" + tenant.getName() + "' not found"); - return applicationRepo.getSessionIdForApplication(applicationId); + return applicationRepo.requireActiveSessionOf(applicationId); } public void validateThatRemoteSessionIsNotActive(Tenant tenant, long sessionId) { @@ -520,6 +525,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye public long createSession(ApplicationId applicationId, TimeoutBudget timeoutBudget, File applicationDirectory) { Tenant tenant = tenantRepository.getTenant(applicationId.tenant()); + tenant.getApplicationRepo().createApplication(applicationId); LocalSessionRepo localSessionRepo = tenant.getLocalSessionRepo(); SessionFactory sessionFactory = tenant.getSessionFactory(); LocalSession session = sessionFactory.createSession(applicationDirectory, applicationId, timeoutBudget); @@ -560,7 +566,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye } private List<ApplicationId> activeApplications(TenantName tenantName) { - return tenantRepository.getTenant(tenantName).getApplicationRepo().listApplications(); + return tenantRepository.getTenant(tenantName).getApplicationRepo().activeApplications(); } // ---------------- Misc operations ---------------------------------------------------------------- @@ -611,7 +617,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye Optional<ApplicationSet> currentActiveApplicationSet = Optional.empty(); TenantApplications applicationRepo = tenant.getApplicationRepo(); try { - long currentActiveSessionId = applicationRepo.getSessionIdForApplication(appId); + long currentActiveSessionId = applicationRepo.requireActiveSessionOf(appId); RemoteSession currentActiveSession = getRemoteSession(tenant, currentActiveSessionId); if (currentActiveSession != null) { currentActiveApplicationSet = Optional.ofNullable(currentActiveSession.ensureApplicationLoaded()); @@ -641,7 +647,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye private List<ApplicationId> listApplicationIds(Tenant tenant) { TenantApplications applicationRepo = tenant.getApplicationRepo(); - return applicationRepo.listApplications(); + return applicationRepo.activeApplications(); } private void cleanupTempDirectory(File tempDir) { @@ -653,13 +659,13 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye private LocalSession getExistingSession(Tenant tenant, ApplicationId applicationId) { TenantApplications applicationRepo = tenant.getApplicationRepo(); - return getLocalSession(tenant, applicationRepo.getSessionIdForApplication(applicationId)); + return getLocalSession(tenant, applicationRepo.requireActiveSessionOf(applicationId)); } private LocalSession getActiveSession(Tenant tenant, ApplicationId applicationId) { TenantApplications applicationRepo = tenant.getApplicationRepo(); - if (applicationRepo.listApplications().contains(applicationId)) { - return tenant.getLocalSessionRepo().getSession(applicationRepo.getSessionIdForApplication(applicationId)); + if (applicationRepo.activeApplications().contains(applicationId)) { + return tenant.getLocalSessionRepo().getSession(applicationRepo.requireActiveSessionOf(applicationId)); } return null; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java index 2c4d3d99408..81eb4107089 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.application; -import com.google.common.collect.ImmutableSet; import com.yahoo.concurrent.ThreadFactoryFactory; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; @@ -12,25 +11,32 @@ import com.yahoo.transaction.Transaction; import com.yahoo.vespa.config.server.ReloadHandler; import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.curator.transaction.CuratorOperations; import com.yahoo.vespa.curator.transaction.CuratorTransaction; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; -import java.util.ArrayList; +import java.time.Duration; import java.util.List; -import java.util.Optional; +import java.util.Map; +import java.util.OptionalLong; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.logging.Logger; +import java.util.stream.Collectors; /** * The applications of a tenant, backed by ZooKeeper. * - * Each application is stored as a single node under /config/v2/tenants/<tenant>/applications/<applications>, - * named the same as the application id and containing the id of the session storing the content of the application. + * Each application is stored under /config/v2/tenants/<tenant>/applications/<application>, + * the root contains the currently active session, if any. Locks for synchronising writes to these paths, and changes + * to the config of this application, are found under /config/v2/tenants/<tenant>/locks/<application>. * * @author Ulf Lilleengen + * @author jonmv */ public class TenantApplications { @@ -38,17 +44,20 @@ public class TenantApplications { private final Curator curator; private final Path applicationsPath; + private final Path locksPath; // One thread pool for all instances of this class private static final ExecutorService pathChildrenExecutor = Executors.newCachedThreadPool(ThreadFactoryFactory.getDaemonThreadFactory(TenantApplications.class.getName())); private final Curator.DirectoryCache directoryCache; private final ReloadHandler reloadHandler; private final TenantName tenant; + private final Map<ApplicationId, Lock> locks; - private TenantApplications(Curator curator, Path applicationsPath, ReloadHandler reloadHandler, TenantName tenant) { + private TenantApplications(Curator curator, ReloadHandler reloadHandler, TenantName tenant) { this.curator = curator; - this.applicationsPath = applicationsPath; - curator.create(applicationsPath); + this.applicationsPath = TenantRepository.getApplicationsPath(tenant); + this.locksPath = TenantRepository.getLocksPath(tenant); + this.locks = new ConcurrentHashMap<>(2); this.reloadHandler = reloadHandler; this.tenant = tenant; this.directoryCache = curator.createDirectoryCache(applicationsPath.getAbsolute(), false, false, pathChildrenExecutor); @@ -57,11 +66,7 @@ public class TenantApplications { } public static TenantApplications create(Curator curator, ReloadHandler reloadHandler, TenantName tenant) { - try { - return new TenantApplications(curator, TenantRepository.getApplicationsPath(tenant), reloadHandler, tenant); - } catch (Exception e) { - throw new RuntimeException(TenantRepository.logPre(tenant) + "Error creating application repo", e); - } + return new TenantApplications(curator, reloadHandler, tenant); } /** @@ -69,75 +74,103 @@ public class TenantApplications { * * @return a list of {@link ApplicationId}s that are active. */ - public List<ApplicationId> listApplications() { + public List<ApplicationId> activeApplications() { + return curator.getChildren(applicationsPath).stream() + .filter(this::isValid) + .sorted() + .map(ApplicationId::fromSerializedForm) + .filter(id -> activeSessionOf(id).isPresent()) + .collect(Collectors.toUnmodifiableList()); + } + + private boolean isValid(String appNode) { // TODO jvenstad: Remove after it has run once everywhere. try { - List<String> appNodes = curator.framework().getChildren().forPath(applicationsPath.getAbsolute()); - List<ApplicationId> applicationIds = new ArrayList<>(); - for (String appNode : appNodes) { - parseApplication(appNode).ifPresent(applicationIds::add); + ApplicationId.fromSerializedForm(appNode); + return true; + } catch (IllegalArgumentException __) { + log.log(LogLevel.INFO, TenantRepository.logPre(tenant) + "Unable to parse application id from '" + + appNode + "'; deleting it as it shouldn't be here."); + try { + curator.delete(applicationsPath.append(appNode)); } - return applicationIds; - } catch (Exception e) { - throw new RuntimeException(TenantRepository.logPre(tenant)+"Unable to list applications", e); + catch (RuntimeException e) { + log.log(LogLevel.WARNING, TenantRepository.logPre(tenant) + "Failed to clean up stray node '" + appNode + "'!", e); + } + return false; } } - private Optional<ApplicationId> parseApplication(String appNode) { - try { - return Optional.of(ApplicationId.fromSerializedForm(appNode)); - } catch (IllegalArgumentException e) { - log.log(LogLevel.INFO, TenantRepository.logPre(tenant)+"Unable to parse application with id '" + appNode + "', ignoring."); - return Optional.empty(); - } + public boolean exists(ApplicationId id) { + return curator.exists(applicationPath(id)); + } + + /** Returns the id of the currently active session for the given application, if any. Throws on unknown applications. */ + public OptionalLong activeSessionOf(ApplicationId id) { + String data = curator.getData(applicationPath(id)).map(Utf8::toString) + .orElseThrow(() -> new IllegalArgumentException("Unknown application '" + id + "'.")); + return data.isEmpty() ? OptionalLong.empty() : OptionalLong.of(Long.parseLong(data)); } /** - * Register active application and adds it to the repo. If it already exists it is overwritten. + * Returns a transaction which writes the given session id as the currently active for the given application. * * @param applicationId An {@link ApplicationId} that represents an active application. * @param sessionId Id of the session containing the application package for this id. */ - public Transaction createPutApplicationTransaction(ApplicationId applicationId, long sessionId) { - if (listApplications().contains(applicationId)) { - return new CuratorTransaction(curator).add(CuratorOperations.setData(applicationsPath.append(applicationId.serializedForm()).getAbsolute(), Utf8.toAsciiBytes(sessionId))); - } else { - return new CuratorTransaction(curator).add(CuratorOperations.create(applicationsPath.append(applicationId.serializedForm()).getAbsolute(), Utf8.toAsciiBytes(sessionId))); + public Transaction createPutTransaction(ApplicationId applicationId, long sessionId) { + return new CuratorTransaction(curator).add(CuratorOperations.setData(applicationPath(applicationId).getAbsolute(), Utf8.toAsciiBytes(sessionId))); + } + + /** + * Creates a node for the given application, marking its existence. + */ + public void createApplication(ApplicationId id) { + try (Lock lock = lock(id)) { + curator.create(applicationPath(id)); } } /** - * Return the stored session id for a given application. + * Return the active session id for a given application. * * @param applicationId an {@link ApplicationId} * @return session id of given application id. - * @throws IllegalArgumentException if the application does not exist + * @throws IllegalArgumentException if the application has no active session */ - public long getSessionIdForApplication(ApplicationId applicationId) { - String path = applicationsPath.append(applicationId.serializedForm()).getAbsolute(); - try { - return Long.parseLong(Utf8.toString(curator.framework().getData().forPath(path))); - } catch (Exception e) { - throw new IllegalArgumentException(TenantRepository.logPre(applicationId) + "Unable to read the session id from '" + path + "'", e); - } + public long requireActiveSessionOf(ApplicationId applicationId) { + return activeSessionOf(applicationId) + .orElseThrow(() -> new IllegalArgumentException("Application '" + applicationId + "' has no active session.")); } /** - * Returns a transaction which deletes this application - * - * @param applicationId an {@link ApplicationId} to delete. + * Returns a transaction which deletes this application. + */ + public CuratorTransaction createDeleteTransaction(ApplicationId applicationId) { + return CuratorTransaction.from(CuratorOperations.deleteAll(applicationPath(applicationId).getAbsolute(), curator), curator); + } + + /** + * Removes all applications not known to this from the config server state. */ - public CuratorTransaction deleteApplication(ApplicationId applicationId) { - Path path = applicationsPath.append(applicationId.serializedForm()); - return CuratorTransaction.from(CuratorOperations.delete(path.getAbsolute()), curator); + public void removeUnusedApplications() { + reloadHandler.removeApplicationsExcept(Set.copyOf(activeApplications())); } /** - * Closes the application repo. Once a repo has been closed, it should not be used again. - */ + * Closes the application repo. Once a repo has been closed, it should not be used again. + */ public void close() { directoryCache.close(); } + /** Returns the lock for changing the session status of the given application. */ + public Lock lock(ApplicationId id) { + curator.create(lockPath(id)); + Lock lock = locks.computeIfAbsent(id, __ -> new Lock(lockPath(id).getAbsolute(), curator)); + lock.acquire(Duration.ofMinutes(1)); // These locks shouldn't be held for very long. + return lock; + } + private void childEvent(CuratorFramework client, PathChildrenCacheEvent event) { switch (event.getType()) { case CHILD_ADDED: @@ -167,13 +200,12 @@ public class TenantApplications { log.log(LogLevel.DEBUG, TenantRepository.logPre(applicationId) + "Application added: " + applicationId); } - /** - * Removes unused applications - * - */ - public void removeUnusedApplications() { - ImmutableSet<ApplicationId> activeApplications = ImmutableSet.copyOf(listApplications()); - reloadHandler.removeApplicationsExcept(activeApplications); + private Path applicationPath(ApplicationId id) { + return applicationsPath.append(id.serializedForm()); + } + + private Path lockPath(ApplicationId id) { + return locksPath.append(id.serializedForm()); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java index 21716730825..6d875c529a3 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java @@ -89,9 +89,8 @@ public class Deployment implements com.yahoo.config.provision.Deployment { timeout, clock, true, true, session.getVespaVersion(), isBootstrap); } - public Deployment setIgnoreSessionStaleFailure(boolean ignoreSessionStaleFailure) { + public void setIgnoreSessionStaleFailure(boolean ignoreSessionStaleFailure) { this.ignoreSessionStaleFailure = ignoreSessionStaleFailure; - return this; } /** Prepares this. This does nothing if this is already prepared */ @@ -116,13 +115,16 @@ public class Deployment implements com.yahoo.config.provision.Deployment { /** Activates this. If it is not already prepared, this will call prepare first. */ @Override public void activate() { - if (! prepared) + if ( ! prepared) prepare(); TimeoutBudget timeoutBudget = new TimeoutBudget(clock, timeout); - long sessionId = session.getSessionId(); - validateSessionStatus(session); - try (Lock lock = tenant.getSessionLock(timeout)) { + + try (Lock lock = tenant.getApplicationRepo().lock(session.getApplicationId())) { + if ( ! tenant.getApplicationRepo().exists(session.getApplicationId())) + return; // Application was deleted. + + validateSessionStatus(session); NestedTransaction transaction = new NestedTransaction(); transaction.add(deactivateCurrentActivateNew(applicationRepository.getActiveSession(session.getApplicationId()), session, ignoreSessionStaleFailure)); @@ -130,13 +132,15 @@ public class Deployment implements com.yahoo.config.provision.Deployment { hostProvisioner.get().activate(transaction, session.getApplicationId(), session.getAllocatedHosts().getHosts()); } transaction.commit(); - session.waitUntilActivated(timeoutBudget); } catch (RuntimeException e) { throw e; } catch (Exception e) { throw new InternalServerException("Error activating application", e); } - log.log(LogLevel.INFO, session.logPre() + "Session " + sessionId + + + session.waitUntilActivated(timeoutBudget); + + log.log(LogLevel.INFO, session.logPre() + "Session " + session.getSessionId() + " activated successfully using " + ( hostProvisioner.isPresent() ? hostProvisioner.get() : "no host provisioner" ) + ". Config generation " + session.getMetaData().getGeneration()); @@ -154,7 +158,7 @@ public class Deployment implements com.yahoo.config.provision.Deployment { /** Exposes the session of this for testing only */ public LocalSession session() { return session; } - + private long validateSessionStatus(LocalSession localSession) { long sessionId = localSession.getSessionId(); if (Session.Status.NEW.equals(localSession.getStatus())) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/host/HostRegistry.java b/configserver/src/main/java/com/yahoo/vespa/config/server/host/HostRegistry.java index 77572856ff5..a7f8b8164a5 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/host/HostRegistry.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/host/HostRegistry.java @@ -36,6 +36,7 @@ public class HostRegistry<T> implements HostValidator<T> { addHosts(key, newHosts); } + @Override public synchronized void verifyHosts(T key, Collection<String> newHosts) { for (String host : newHosts) { if (hostAlreadyTaken(host, key)) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsHandler.java index 6345532d4ff..cdf995b80bb 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsHandler.java @@ -56,7 +56,7 @@ public class ListApplicationsHandler extends HttpHandler { Utils.checkThatTenantExists(tenantRepository, tenantName); Tenant tenant = tenantRepository.getTenant(tenantName); TenantApplications applicationRepo = tenant.getApplicationRepo(); - return applicationRepo.listApplications(); + return applicationRepo.activeApplications(); } private static String createUrlStringFromId(String urlBase, ApplicationId id, Zone zone) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java index 0f9f8b72de1..af8956803ab 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java @@ -83,12 +83,6 @@ public class LocalSession extends Session implements Comparable<LocalSession> { setStatus(Session.Status.PREPARE); } - private Transaction setActive() { - Transaction transaction = createSetStatusTransaction(Status.ACTIVATE); - transaction.add(applicationRepo.createPutApplicationTransaction(zooKeeperClient.readApplicationId(), getSessionId()).operations()); - return transaction; - } - private Transaction createSetStatusTransaction(Status status) { return zooKeeperClient.createWriteStatusTransaction(status); } @@ -99,8 +93,10 @@ public class LocalSession extends Session implements Comparable<LocalSession> { public Transaction createActivateTransaction() { zooKeeperClient.createActiveWaiter(); - superModelGenerationCounter.increment(); - return setActive(); + superModelGenerationCounter.increment(); // TODO jvenstad: I hope this counter isn't used for serious things, as it's updated way ahead of activation. + Transaction transaction = createSetStatusTransaction(Status.ACTIVATE); + transaction.add(applicationRepo.createPutTransaction(zooKeeperClient.readApplicationId(), getSessionId()).operations()); + return transaction; } public Transaction createDeactivateTransaction() { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java index 172d9c025d5..3f1619882cd 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java @@ -5,8 +5,8 @@ import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.TenantName; import com.yahoo.lang.SettableOptional; -import com.yahoo.transaction.Transaction; import com.yahoo.log.LogLevel; +import com.yahoo.transaction.Transaction; import com.yahoo.vespa.config.server.GlobalComponentRegistry; import com.yahoo.vespa.config.server.ReloadHandler; import com.yahoo.vespa.config.server.application.ApplicationSet; @@ -29,7 +29,7 @@ import java.util.logging.Logger; public class RemoteSession extends Session { private static final Logger log = Logger.getLogger(RemoteSession.class.getName()); - private volatile ApplicationSet applicationSet = null; + private ApplicationSet applicationSet = null; private final ActivatedModelsBuilder applicationLoader; private final Clock clock; @@ -69,18 +69,15 @@ public class RemoteSession extends Session { clock.instant())); } - public ApplicationSet ensureApplicationLoaded() { - if (applicationSet == null) { - applicationSet = loadApplication(); - } - return applicationSet; + public synchronized ApplicationSet ensureApplicationLoaded() { + return applicationSet == null ? applicationSet = loadApplication() : applicationSet; } public Session.Status getStatus() { return zooKeeperClient.readStatus(); } - public void deactivate() { + public synchronized void deactivate() { applicationSet = null; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionRepo.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionRepo.java index 15182813a22..bbb06515bef 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionRepo.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionRepo.java @@ -1,33 +1,36 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.session; -import java.time.Duration; -import java.time.Instant; -import java.util.*; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.stream.Collectors; - import com.google.common.collect.HashMultiset; import com.google.common.collect.Multiset; import com.yahoo.concurrent.ThreadFactoryFactory; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; import com.yahoo.log.LogLevel; import com.yahoo.path.Path; -import com.yahoo.vespa.config.server.application.ApplicationSet; +import com.yahoo.vespa.config.server.ReloadHandler; import com.yahoo.vespa.config.server.application.TenantApplications; +import com.yahoo.vespa.config.server.monitoring.MetricUpdater; import com.yahoo.vespa.config.server.tenant.TenantRepository; +import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; import com.yahoo.vespa.curator.Curator; import com.yahoo.yolean.Exceptions; -import com.yahoo.vespa.config.server.ReloadHandler; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.vespa.config.server.monitoring.MetricUpdater; -import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; - import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.recipes.cache.*; +import org.apache.curator.framework.recipes.cache.ChildData; +import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; + +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.stream.Collectors; /** * Will watch/prepare sessions (applications) based on watched nodes in ZooKeeper, set for example @@ -115,19 +118,6 @@ public class RemoteSessionRepo extends SessionRepo<RemoteSession> { return (created.plus(expiryTime).isBefore(Instant.now())); } - private void loadActiveSession(RemoteSession session) { - tryReload(session.ensureApplicationLoaded(), session.logPre()); - } - - private void tryReload(ApplicationSet applicationSet, String logPre) { - try { - reloadHandler.reloadConfig(applicationSet); - log.log(LogLevel.INFO, logPre + "Application activated successfully: " + applicationSet.getId()); - } catch (Exception e) { - log.log(LogLevel.WARNING, logPre + "Skipping loading of application '" + applicationSet.getId() + "': " + Exceptions.toMessageString(e)); - } - } - private List<Long> getSessionListFromDirectoryCache(List<ChildData> children) { return getSessionList(children.stream() .map(child -> Path.fromString(child.getPath()).getName()) @@ -190,15 +180,16 @@ public class RemoteSessionRepo extends SessionRepo<RemoteSession> { } private void loadSessionIfActive(RemoteSession session) { - for (ApplicationId applicationId : applicationRepo.listApplications()) { + for (ApplicationId applicationId : applicationRepo.activeApplications()) { try { - if (applicationRepo.getSessionIdForApplication(applicationId) == session.getSessionId()) { + if (applicationRepo.requireActiveSessionOf(applicationId) == session.getSessionId()) { log.log(LogLevel.DEBUG, "Found active application for session " + session.getSessionId() + " , loading it"); - loadActiveSession(session); - break; + reloadHandler.reloadConfig(session.ensureApplicationLoaded()); + log.log(LogLevel.INFO, session.logPre() + "Application activated successfully: " + applicationId); + return; } } catch (Exception e) { - log.log(LogLevel.WARNING, session.logPre() + " error reading session id for " + applicationId, e); + log.log(LogLevel.WARNING, session.logPre() + "Skipping loading of application '" + applicationId + "': " + Exceptions.toMessageString(e)); } } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactory.java index a3dea83d50c..5527d3060f7 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactory.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactory.java @@ -17,8 +17,6 @@ public interface SessionFactory { /** * Creates a new deployment session from an application package. * - * - * * @param applicationDirectory a File pointing to an application. * @param applicationId application id for this new session. * @param timeoutBudget Timeout for creating session and waiting for other servers. @@ -29,10 +27,10 @@ public interface SessionFactory { /** * Creates a new deployment session from an already existing session. * - * @param existingSession The session to use as base + * @param existingSession the session to use as base * @param logger a deploy logger where the deploy log will be written. - * @param internalRedeploy if this session is for a system internal redeploy not an application package change - * @param timeoutBudget Timeout for creating session and waiting for other servers. + * @param internalRedeploy whether this session is for a system internal redeploy — not an application package change + * @param timeoutBudget timeout for creating session and waiting for other servers. * @return a new session */ LocalSession createSessionFromExisting(LocalSession existingSession, DeployLogger logger, diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java index b79ea720aea..d8ba5890545 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java @@ -188,10 +188,11 @@ public class SessionFactoryImpl implements SessionFactory, LocalSessionLoader { } private long getActiveSessionId(ApplicationId applicationId) { - List<ApplicationId> applicationIds = applicationRepo.listApplications(); + List<ApplicationId> applicationIds = applicationRepo.activeApplications(); if (applicationIds.contains(applicationId)) { - return applicationRepo.getSessionIdForApplication(applicationId); + return applicationRepo.requireActiveSessionOf(applicationId); } return nonExistingActiveSession; } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java index 43a5ff6d0c2..f0ceeb186fe 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java @@ -113,9 +113,8 @@ public class SessionPreparer { log.log(LogLevel.DEBUG, () -> "time used " + params.getTimeoutBudget().timesUsed() + " : " + params.getApplicationId()); return preparation.result(); - } catch (OutOfCapacityException e) { - throw e; - } catch (IllegalArgumentException e) { + } + catch (IllegalArgumentException e) { throw new InvalidApplicationException("Invalid application package", e); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenant.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenant.java index a68f4a396cd..88e71d7ddd1 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenant.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenant.java @@ -11,10 +11,8 @@ import com.yahoo.vespa.config.server.session.LocalSessionRepo; import com.yahoo.vespa.config.server.session.RemoteSessionRepo; import com.yahoo.vespa.config.server.session.SessionFactory; import com.yahoo.vespa.curator.Curator; -import com.yahoo.vespa.curator.Lock; import org.apache.zookeeper.data.Stat; -import java.time.Duration; import java.time.Instant; import java.util.Optional; @@ -30,7 +28,7 @@ public class Tenant implements TenantHandlerProvider { static final String SESSIONS = "sessions"; static final String APPLICATIONS = "applications"; - static final String SESSION_LOCK_PATH = "activateLock"; + static final String LOCKS = "locks"; private final TenantName name; private final RemoteSessionRepo remoteSessionRepo; @@ -38,7 +36,6 @@ public class Tenant implements TenantHandlerProvider { private final SessionFactory sessionFactory; private final LocalSessionRepo localSessionRepo; private final TenantApplications applicationRepo; - private final Lock sessionLock; private final RequestHandler requestHandler; private final ReloadHandler reloadHandler; private final TenantFileSystemDirs tenantFileSystemDirs; @@ -61,7 +58,6 @@ public class Tenant implements TenantHandlerProvider { this.remoteSessionRepo = remoteSessionRepo; this.sessionFactory = sessionFactory; this.localSessionRepo = localSessionRepo; - this.sessionLock = createLock(curator, path); this.applicationRepo = applicationRepo; this.tenantFileSystemDirs = tenantFileSystemDirs; this.curator = curator; @@ -110,14 +106,6 @@ public class Tenant implements TenantHandlerProvider { return localSessionRepo; } - /** - * This lock allows activation and deactivation of sessions under this tenant. - */ - public Lock getSessionLock(Duration timeout) { - sessionLock.acquire(timeout); - return sessionLock; - } - @Override public String toString() { return getName().value(); @@ -165,10 +153,4 @@ public class Tenant implements TenantHandlerProvider { curator.delete(path); } - private static Lock createLock(Curator curator, Path tenantPath) { - Path lockPath = tenantPath.append(SESSION_LOCK_PATH); - curator.create(lockPath); - return new Lock(lockPath.getAbsolute(), curator); - } - } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantBuilder.java index 078b6e861a9..943ae6248fc 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantBuilder.java @@ -31,7 +31,7 @@ public class TenantBuilder { private SessionFactory sessionFactory; private LocalSessionLoader localSessionLoader; private TenantApplications applicationRepo; - private ReloadHandler reloadHandler; + private TenantRequestHandler reloadHandler; private RequestHandler requestHandler; private RemoteSessionFactory remoteSessionFactory; private TenantFileSystemDirs tenantFileSystemDirs; @@ -120,7 +120,7 @@ public class TenantBuilder { private void createApplicationRepo() { if (applicationRepo == null) { - applicationRepo = TenantApplications.create(componentRegistry.getCurator(), reloadHandler, tenant); + applicationRepo = reloadHandler.applications(); } } @@ -130,7 +130,8 @@ public class TenantBuilder { tenant, Collections.singletonList(componentRegistry.getReloadListener()), ConfigResponseFactory.create(componentRegistry.getConfigserverConfig()), - componentRegistry.getHostRegistries()); + componentRegistry.getHostRegistries(), + componentRegistry.getCurator()); if (hostValidator == null) { this.hostValidator = impl; } @@ -164,9 +165,5 @@ public class TenantBuilder { } } - public TenantApplications getApplicationRepo() { - return applicationRepo; - } - public TenantName getTenantName() { return tenant; } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java index 9c74c9c1e67..c37db0e0df5 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java @@ -262,7 +262,10 @@ public class TenantRepository { */ private synchronized void writeTenantPath(TenantName name) { Path tenantPath = getTenantPath(name); - curator.createAtomically(tenantPath, tenantPath.append(Tenant.SESSIONS), tenantPath.append(Tenant.APPLICATIONS)); + curator.createAtomically(tenantPath, + tenantPath.append(Tenant.SESSIONS), + tenantPath.append(Tenant.APPLICATIONS), + tenantPath.append(Tenant.LOCKS)); } /** @@ -406,4 +409,11 @@ public class TenantRepository { return getTenantPath(tenantName).append(Tenant.APPLICATIONS); } + /** + * Gets zookeeper path for locks for a tenant's applications + */ + public static Path getLocksPath(TenantName tenantName) { + return getTenantPath(tenantName).append(Tenant.LOCKS); + } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandler.java index a341061bd9a..3f2b8c76e44 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandler.java @@ -6,6 +6,7 @@ import java.util.Collection; import java.util.LinkedHashSet; import java.util.List; import java.util.Optional; +import java.util.OptionalLong; import java.util.Set; import com.yahoo.component.Version; @@ -16,6 +17,7 @@ import com.yahoo.vespa.config.protocol.ConfigResponse; import com.yahoo.vespa.config.server.NotFoundException; import com.yahoo.vespa.config.server.application.ApplicationMapper; import com.yahoo.vespa.config.server.application.ApplicationSet; +import com.yahoo.vespa.config.server.application.TenantApplications; import com.yahoo.vespa.config.server.rpc.ConfigResponseFactory; import com.yahoo.vespa.config.server.host.HostRegistries; import com.yahoo.vespa.config.server.host.HostRegistry; @@ -29,6 +31,8 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; import com.yahoo.vespa.config.server.monitoring.MetricUpdater; import com.yahoo.vespa.config.server.monitoring.Metrics; +import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.curator.Lock; /** * A per tenant request handler, for handling reload (activate application) and getConfig requests for @@ -44,23 +48,25 @@ public class TenantRequestHandler implements RequestHandler, ReloadHandler, Host private final TenantName tenant; private final List<ReloadListener> reloadListeners; private final ConfigResponseFactory responseFactory; - private final HostRegistry<ApplicationId> hostRegistry; private final ApplicationMapper applicationMapper = new ApplicationMapper(); private final MetricUpdater tenantMetricUpdater; private final Clock clock = Clock.systemUTC(); + private final TenantApplications applications; public TenantRequestHandler(Metrics metrics, TenantName tenant, List<ReloadListener> reloadListeners, ConfigResponseFactory responseFactory, - HostRegistries hostRegistries) { + HostRegistries hostRegistries, + Curator curator) { // TODO jvenstad: Merge this class with TenantApplications, and straighten this out. this.metrics = metrics; this.tenant = tenant; - this.reloadListeners = reloadListeners; + this.reloadListeners = List.copyOf(reloadListeners); this.responseFactory = responseFactory; - tenantMetricUpdater = metrics.getOrCreateMetricUpdater(Metrics.createDimensions(tenant)); - hostRegistry = hostRegistries.createApplicationHostRegistry(tenant); + this.tenantMetricUpdater = metrics.getOrCreateMetricUpdater(Metrics.createDimensions(tenant)); + this.hostRegistry = hostRegistries.createApplicationHostRegistry(tenant); + this.applications = TenantApplications.create(curator, this, tenant); } /** @@ -93,26 +99,40 @@ public class TenantRequestHandler implements RequestHandler, ReloadHandler, Host * * @param applicationSet the {@link ApplicationSet} to be reloaded */ + @Override public void reloadConfig(ApplicationSet applicationSet) { - setLiveApp(applicationSet); - notifyReloadListeners(applicationSet); + ApplicationId id = applicationSet.getId(); + try (Lock lock = applications.lock(id)) { + if ( ! applications.exists(id)) + return; // Application was deleted before activation. + if (applicationSet.getApplicationGeneration() != applications.requireActiveSessionOf(id)) + return; // Application activated a new session before we got here. + + setLiveApp(applicationSet); + notifyReloadListeners(applicationSet); + } } @Override public void removeApplication(ApplicationId applicationId) { - if (applicationMapper.hasApplication(applicationId, clock.instant())) { - applicationMapper.remove(applicationId); - hostRegistry.removeHostsForKey(applicationId); - reloadListenersOnRemove(applicationId); - tenantMetricUpdater.setApplications(applicationMapper.numApplications()); - metrics.removeMetricUpdater(Metrics.createDimensions(applicationId)); + try (Lock lock = applications.lock(applicationId)) { + if (applications.exists(applicationId)) + return; // Application was deployed again. + + if (applicationMapper.hasApplication(applicationId, clock.instant())) { + applicationMapper.remove(applicationId); + hostRegistry.removeHostsForKey(applicationId); + reloadListenersOnRemove(applicationId); + tenantMetricUpdater.setApplications(applicationMapper.numApplications()); + metrics.removeMetricUpdater(Metrics.createDimensions(applicationId)); + } } } @Override public void removeApplicationsExcept(Set<ApplicationId> applications) { for (ApplicationId activeApplication : applicationMapper.listApplicationIds()) { - if (! applications.contains(activeApplication)) { + if ( ! applications.contains(activeApplication)) { log.log(LogLevel.INFO, "Will remove deleted application " + activeApplication.toShortString()); removeApplication(activeApplication); } @@ -237,4 +257,6 @@ public class TenantRequestHandler implements RequestHandler, ReloadHandler, Host } } + TenantApplications applications() { return applications; } + } |