diff options
Diffstat (limited to 'configserver/src')
64 files changed, 1223 insertions, 1797 deletions
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index dd88096c62f..9625bdf7447 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -49,12 +49,10 @@ import com.yahoo.vespa.config.server.http.v2.PrepareResult; import com.yahoo.vespa.config.server.metrics.ApplicationMetricsRetriever; import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; import com.yahoo.vespa.config.server.session.LocalSession; -import com.yahoo.vespa.config.server.session.LocalSessionRepo; import com.yahoo.vespa.config.server.session.PrepareParams; import com.yahoo.vespa.config.server.session.RemoteSession; -import com.yahoo.vespa.config.server.session.RemoteSessionRepo; import com.yahoo.vespa.config.server.session.Session; -import com.yahoo.vespa.config.server.session.SessionFactory; +import com.yahoo.vespa.config.server.session.SessionRepository; import com.yahoo.vespa.config.server.session.SilentDeployLogger; import com.yahoo.vespa.config.server.tenant.ApplicationRolesStore; import com.yahoo.vespa.config.server.tenant.ContainerEndpointsCache; @@ -79,6 +77,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.logging.Level; @@ -216,7 +215,9 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye Slime deployLog = createDeployLog(); DeployLogger logger = new DeployHandlerLogger(deployLog.get().setArray("log"), prepareParams.isVerbose(), applicationId); try (ActionTimer timer = timerFor(applicationId, "deployment.prepareMillis")) { - ConfigChangeActions actions = session.prepare(logger, prepareParams, currentActiveApplicationSet, tenant.getPath(), now); + SessionRepository sessionRepository = tenant.getSessionRepository(); + ConfigChangeActions actions = sessionRepository.prepareLocalSession(session, logger, prepareParams, + currentActiveApplicationSet, tenant.getPath(), now); logConfigChangeActions(actions, logger); log.log(Level.INFO, TenantRepository.logPre(applicationId) + "Session " + sessionId + " prepared successfully. "); return new PrepareResult(sessionId, actions, deployLog); @@ -304,8 +305,8 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye LocalSession activeSession = getActiveLocalSession(tenant, application); if (activeSession == null) return Optional.empty(); TimeoutBudget timeoutBudget = new TimeoutBudget(clock, timeout); - LocalSession newSession = tenant.getSessionFactory().createSessionFromExisting(activeSession, logger, true, timeoutBudget); - tenant.getLocalSessionRepo().addSession(newSession); + LocalSession newSession = tenant.getSessionRepository().createSessionFromExisting(activeSession, logger, true, timeoutBudget); + tenant.getSessionRepository().addSession(newSession); return Optional.of(Deployment.unprepared(newSession, this, hostProvisioner, tenant, timeout, clock, false /* don't validate as this is already deployed */, bootstrap)); @@ -489,7 +490,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye Tenant tenant = tenantRepository.getTenant(applicationId.tenant()); if (tenant == null) throw new NotFoundException("Tenant '" + applicationId.tenant() + "' not found"); long sessionId = getSessionIdForApplication(tenant, applicationId); - RemoteSession session = tenant.getRemoteSessionRepo().getSession(sessionId); + RemoteSession session = tenant.getSessionRepo().getRemoteSession(sessionId); if (session == null) throw new NotFoundException("Remote session " + sessionId + " not found"); return session.ensureApplicationLoaded().getForVersionOrLatest(version, clock.instant()); } catch (NotFoundException e) { @@ -526,10 +527,10 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye } private boolean localSessionHasBeenDeleted(ApplicationId applicationId, long sessionId, Duration waitTime) { - RemoteSessionRepo remoteSessionRepo = tenantRepository.getTenant(applicationId.tenant()).getRemoteSessionRepo(); + SessionRepository sessionRepository = tenantRepository.getTenant(applicationId.tenant()).getSessionRepo(); Instant end = Instant.now().plus(waitTime); do { - if (remoteSessionRepo.getSession(sessionId) == null) return true; + if (sessionRepository.getRemoteSession(sessionId) == null) return true; try { Thread.sleep(10); } catch (InterruptedException e) { /* ignored */} } while (Instant.now().isBefore(end)); @@ -635,11 +636,10 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye boolean internalRedeploy, TimeoutBudget timeoutBudget) { Tenant tenant = tenantRepository.getTenant(applicationId.tenant()); - LocalSessionRepo localSessionRepo = tenant.getLocalSessionRepo(); - SessionFactory sessionFactory = tenant.getSessionFactory(); + SessionRepository sessionRepository = tenant.getSessionRepository(); RemoteSession fromSession = getExistingSession(tenant, applicationId); - LocalSession session = sessionFactory.createSessionFromExisting(fromSession, logger, internalRedeploy, timeoutBudget); - localSessionRepo.addSession(session); + LocalSession session = sessionRepository.createSessionFromExisting(fromSession, logger, internalRedeploy, timeoutBudget); + sessionRepository.addSession(session); return session.getSessionId(); } @@ -658,15 +658,17 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye Tenant tenant = tenantRepository.getTenant(applicationId.tenant()); tenant.getApplicationRepo().createApplication(applicationId); Optional<Long> activeSessionId = tenant.getApplicationRepo().activeSessionOf(applicationId); - LocalSession session = tenant.getSessionFactory().createSession(applicationDirectory, applicationId, - timeoutBudget, activeSessionId); - tenant.getLocalSessionRepo().addSession(session); + LocalSession session = tenant.getSessionRepository().createSession(applicationDirectory, + applicationId, + timeoutBudget, + activeSessionId); + tenant.getSessionRepository().addSession(session); return session.getSessionId(); } public void deleteExpiredLocalSessions() { Map<Tenant, List<LocalSession>> sessionsPerTenant = new HashMap<>(); - tenantRepository.getAllTenants().forEach(tenant -> sessionsPerTenant.put(tenant, tenant.getLocalSessionRepo().getSessions())); + tenantRepository.getAllTenants().forEach(tenant -> sessionsPerTenant.put(tenant, tenant.getSessionRepository().getLocalSessions())); Set<ApplicationId> applicationIds = new HashSet<>(); sessionsPerTenant.values().forEach(sessionList -> sessionList.forEach(s -> applicationIds.add(s.getApplicationId()))); @@ -677,7 +679,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye if (activeSession != null) activeSessions.put(applicationId, activeSession.getSessionId()); }); - sessionsPerTenant.keySet().forEach(tenant -> tenant.getLocalSessionRepo().deleteExpiredSessions(activeSessions)); + sessionsPerTenant.keySet().forEach(tenant -> tenant.getSessionRepository().deleteExpiredSessions(activeSessions)); } public int deleteExpiredRemoteSessions(Duration expiryTime) { @@ -687,7 +689,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye public int deleteExpiredRemoteSessions(Clock clock, Duration expiryTime) { return tenantRepository.getAllTenants() .stream() - .map(tenant -> tenant.getRemoteSessionRepo().deleteExpiredSessions(clock, expiryTime)) + .map(tenant -> tenant.getSessionRepo().deleteExpiredRemoteSessions(clock, expiryTime)) .mapToInt(i -> i) .sum(); } @@ -739,6 +741,14 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye return configserverConfig; } + public ApplicationId getApplicationIdForHostname(String hostname) { + Optional<ApplicationId> applicationId = tenantRepository.getAllTenantNames().stream() + .map(tenantName -> tenantRepository.getTenant(tenantName).getApplicationRepo().getApplicationIdForHostName(hostname)) + .filter(Objects::nonNull) + .findFirst(); + return applicationId.orElse(null); + } + private void validateThatLocalSessionIsNotActive(Tenant tenant, long sessionId) { LocalSession session = getLocalSession(tenant, sessionId); if (Session.Status.ACTIVATE.equals(session.getStatus())) { @@ -747,20 +757,20 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye } private LocalSession getLocalSession(Tenant tenant, long sessionId) { - LocalSession session = tenant.getLocalSessionRepo().getSession(sessionId); + LocalSession session = tenant.getSessionRepository().getLocalSession(sessionId); if (session == null) throw new NotFoundException("Session " + sessionId + " was not found"); return session; } private RemoteSession getRemoteSession(Tenant tenant, long sessionId) { - RemoteSession session = tenant.getRemoteSessionRepo().getSession(sessionId); + RemoteSession session = tenant.getSessionRepo().getRemoteSession(sessionId); if (session == null) throw new NotFoundException("Session " + sessionId + " was not found"); return session; } - private Optional<ApplicationSet> getCurrentActiveApplicationSet(Tenant tenant, ApplicationId appId) { + public Optional<ApplicationSet> getCurrentActiveApplicationSet(Tenant tenant, ApplicationId appId) { Optional<ApplicationSet> currentActiveApplicationSet = Optional.empty(); TenantApplications applicationRepo = tenant.getApplicationRepo(); try { @@ -805,7 +815,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye private RemoteSession getActiveSession(Tenant tenant, ApplicationId applicationId) { TenantApplications applicationRepo = tenant.getApplicationRepo(); if (applicationRepo.activeApplications().contains(applicationId)) { - return tenant.getRemoteSessionRepo().getSession(applicationRepo.requireActiveSessionOf(applicationId)); + return tenant.getSessionRepo().getRemoteSession(applicationRepo.requireActiveSessionOf(applicationId)); } return null; } @@ -813,7 +823,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye public LocalSession getActiveLocalSession(Tenant tenant, ApplicationId applicationId) { TenantApplications applicationRepo = tenant.getApplicationRepo(); if (applicationRepo.activeApplications().contains(applicationId)) { - return tenant.getLocalSessionRepo().getSession(applicationRepo.requireActiveSessionOf(applicationId)); + return tenant.getSessionRepository().getLocalSession(applicationRepo.requireActiveSessionOf(applicationId)); } return null; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java index a4dfec708d6..4feaf51c3e0 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java @@ -213,7 +213,7 @@ public class TenantApplications implements RequestHandler, ReloadHandler, HostVa break; } // We may have lost events and may need to remove applications. - // New applications are added when session is added, not here. See RemoteSessionRepo. + // New applications are added when session is added, not here. See SessionRepository. removeUnusedApplications(); }); } @@ -419,5 +419,17 @@ public class TenantApplications implements RequestHandler, ReloadHandler, HostVa reloadListener.verifyHostsAreAvailable(tenant, newHosts); } + public HostValidator<ApplicationId> getHostValidator() { + return this; + } + + public HostRegistry<ApplicationId> getApplicationHostRegistry() { + return hostRegistry; + } + + public ApplicationId getApplicationIdForHostName(String hostname) { + return hostRegistry.getKeyForHost(hostname); + } + public TenantFileSystemDirs getTenantFileSystemDirs() { return tenantFileSystemDirs; } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java index d90a79795cf..39fac90d242 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java @@ -15,6 +15,7 @@ import com.yahoo.vespa.config.server.ActivationConflictException; import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.ApplicationRepository.ActionTimer; import com.yahoo.vespa.config.server.TimeoutBudget; +import com.yahoo.vespa.config.server.application.ApplicationSet; import com.yahoo.vespa.config.server.http.InternalServerException; import com.yahoo.vespa.config.server.session.LocalSession; import com.yahoo.vespa.config.server.session.PrepareParams; @@ -118,7 +119,9 @@ public class Deployment implements com.yahoo.config.provision.Deployment { .isBootstrap(isBootstrap); dockerImageRepository.ifPresent(params::dockerImageRepository); athenzDomain.ifPresent(params::athenzDomain); - session.prepare(logger, params.build(), Optional.empty(), tenant.getPath(), clock.instant()); + Optional<ApplicationSet> activeApplicationSet = applicationRepository.getCurrentActiveApplicationSet(tenant, session.getApplicationId()); + tenant.getSessionRepository().prepareLocalSession(session, logger, params.build(), activeApplicationSet, + tenant.getPath(), clock.instant()); this.prepared = true; } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index c925157b980..74a9e72e255 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -153,7 +153,7 @@ public class ModelContextImpl implements ModelContext { private final double defaultTermwiseLimit; private final double threadPoolSizeFactor; private final double queueSizefactor; - private final String docprocLoadBalancerType; + private final String jvmGCOPtions; private final Optional<AthenzDomain> athenzDomain; private final Optional<ApplicationRoles> applicationRoles; private final int jdiscHealthCheckProxyClientTimeout; @@ -195,7 +195,7 @@ public class ModelContextImpl implements ModelContext { .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); queueSizefactor = Flags.DEFAULT_QUEUE_SIZE_FACTOR.bindTo(flagSource) .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); - docprocLoadBalancerType = Flags.DOCPROC_LOADBALANCER_TYPE.bindTo(flagSource) + jvmGCOPtions = Flags.JVM_GC_OPTIONS.bindTo(flagSource) .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); this.athenzDomain = athenzDomain; this.applicationRoles = applicationRoles; @@ -257,11 +257,6 @@ public class ModelContextImpl implements ModelContext { } @Override - public String docprocLoadBalancerType() { - return docprocLoadBalancerType; - } - - @Override public boolean useDistributorBtreeDb() { return useDistributorBtreeDb; } @@ -280,6 +275,8 @@ public class ModelContextImpl implements ModelContext { } @Override public Duration jdiscHealthCheckProxyClientTimeout() { return Duration.ofMillis(jdiscHealthCheckProxyClientTimeout); } + @Override public String jvmGCOptions() { return jvmGCOPtions; } + } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java index 0865b72dbbf..07a686a63bd 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java @@ -177,4 +177,10 @@ public class FileDirectory { destChannel.transferFrom(sourceChannel, 0, sourceChannel.size()); } } + + @Override + public String toString() { + return "root dir: " + root.getAbsolutePath(); + } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HostHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HostHandler.java index 2c888df6658..1ea41b85983 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HostHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HostHandler.java @@ -3,61 +3,37 @@ package com.yahoo.vespa.config.server.http.v2; import com.google.inject.Inject; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.jdisc.Response; import com.yahoo.jdisc.application.BindingMatch; -import com.yahoo.vespa.config.server.GlobalComponentRegistry; -import com.yahoo.vespa.config.server.host.HostRegistries; -import com.yahoo.vespa.config.server.host.HostRegistry; +import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.http.HttpErrorResponse; import com.yahoo.vespa.config.server.http.HttpHandler; import com.yahoo.vespa.config.server.http.JSONResponse; -import java.util.logging.Level; - - /** * Handler for getting tenant and application for a given hostname. * * @author hmusum - * @since 5.19 */ public class HostHandler extends HttpHandler { - final HostRegistries hostRegistries; - private final Zone zone; + private final ApplicationRepository applicationRepository; @Inject - public HostHandler(HttpHandler.Context ctx, - GlobalComponentRegistry globalComponentRegistry) { + public HostHandler(HttpHandler.Context ctx, ApplicationRepository applicationRepository) { super(ctx); - this.hostRegistries = globalComponentRegistry.getHostRegistries(); - this.zone = globalComponentRegistry.getZone(); + this.applicationRepository = applicationRepository; } @Override public HttpResponse handleGET(HttpRequest request) { String hostname = getBindingMatch(request).group(2); - log.log(Level.FINE, "hostname=" + hostname); - - HostRegistry<TenantName> tenantHostRegistry = hostRegistries.getTenantHostRegistry(); - log.log(Level.FINE, "hosts in tenant host registry '" + tenantHostRegistry + "' " + tenantHostRegistry.getAllHosts()); - TenantName tenant = tenantHostRegistry.getKeyForHost(hostname); - if (tenant == null) return createError(hostname); - log.log(Level.FINE, "tenant=" + tenant); - HostRegistry<ApplicationId> applicationIdHostRegistry = hostRegistries.getApplicationHostRegistry(tenant); - ApplicationId applicationId; - if (applicationIdHostRegistry == null) return createError(hostname); - applicationId = applicationIdHostRegistry.getKeyForHost(hostname); - log.log(Level.FINE, "applicationId=" + applicationId); - if (applicationId == null) { - return createError(hostname); - } else { - log.log(Level.FINE, "hosts in application host registry '" + applicationIdHostRegistry + "' " + applicationIdHostRegistry.getAllHosts()); - return new HostResponse(Response.Status.OK, applicationId, zone); - } + ApplicationId applicationId = applicationRepository.getApplicationIdForHostname(hostname); + return (applicationId == null) + ? createError(hostname) + : new HostResponse(Response.Status.OK, applicationId, applicationRepository.zone()); } private HttpErrorResponse createError(String hostname) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java index 3ea7959c212..4e6a541793d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java @@ -37,7 +37,7 @@ public class ConfigServerMaintenance extends AbstractComponent { //tenantsMaintainer = new TenantsMaintainer(applicationRepository, curator, defaults.tenantsMaintainerInterval); fileDistributionMaintainer = new FileDistributionMaintainer(applicationRepository, curator, defaults.defaultInterval, configserverConfig); sessionsMaintainer = new SessionsMaintainer(applicationRepository, curator, defaults.defaultInterval); - applicationPackageMaintainer = new ApplicationPackageMaintainer(applicationRepository, curator, defaults.defaultInterval, configserverConfig, flagSource); + applicationPackageMaintainer = new ApplicationPackageMaintainer(applicationRepository, curator, Duration.ofMinutes(1), configserverConfig, flagSource); } @Override diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java index 56e32f7d802..73e7b36c381 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java @@ -4,25 +4,11 @@ package com.yahoo.vespa.config.server.session; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.application.api.ApplicationMetaData; import com.yahoo.config.application.api.ApplicationPackage; -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; -import com.yahoo.io.IOUtils; import com.yahoo.path.Path; -import com.yahoo.transaction.AbstractTransaction; -import com.yahoo.transaction.NestedTransaction; import com.yahoo.transaction.Transaction; import com.yahoo.vespa.config.server.TimeoutBudget; -import com.yahoo.vespa.config.server.application.ApplicationSet; import com.yahoo.vespa.config.server.application.TenantApplications; -import com.yahoo.vespa.config.server.configchange.ConfigChangeActions; -import com.yahoo.vespa.config.server.host.HostValidator; -import com.yahoo.vespa.curator.Curator; - -import java.io.File; -import java.time.Instant; -import java.util.Optional; -import java.util.logging.Level; /** * A LocalSession is a session that has been created locally on this configserver. A local session can be edited and @@ -37,43 +23,18 @@ public class LocalSession extends Session { protected final ApplicationPackage applicationPackage; private final TenantApplications applicationRepo; - private final SessionPreparer sessionPreparer; - private final File serverDBSessionDir; - private final SessionZooKeeperClient sessionZooKeeperClient; - private final HostValidator<ApplicationId> hostValidator; /** - * Create a session. This involves loading the application, validating it and distributing it. + * Creates a session. This involves loading the application, validating it and distributing it. * * @param sessionId The session id for this session. */ - public LocalSession(TenantName tenant, long sessionId, SessionPreparer sessionPreparer, - ApplicationPackage applicationPackage, SessionZooKeeperClient sessionZooKeeperClient, - File serverDBSessionDir, TenantApplications applicationRepo, - HostValidator<ApplicationId> hostValidator) { + public LocalSession(TenantName tenant, long sessionId, ApplicationPackage applicationPackage, + SessionZooKeeperClient sessionZooKeeperClient, + TenantApplications applicationRepo) { super(tenant, sessionId, sessionZooKeeperClient); - this.serverDBSessionDir = serverDBSessionDir; this.applicationPackage = applicationPackage; - this.sessionZooKeeperClient = sessionZooKeeperClient; this.applicationRepo = applicationRepo; - this.sessionPreparer = sessionPreparer; - this.hostValidator = hostValidator; - } - - public ConfigChangeActions prepare(DeployLogger logger, - PrepareParams params, - Optional<ApplicationSet> currentActiveApplicationSet, - Path tenantPath, - Instant now) { - applicationRepo.createApplication(params.getApplicationId()); // TODO jvenstad: This is wrong, but it has to be done now, since preparation can change the application ID of a session :( - logger.log(Level.FINE, "Created application " + params.getApplicationId()); - Curator.CompletionWaiter waiter = zooKeeperClient.createPrepareWaiter(); - ConfigChangeActions actions = sessionPreparer.prepare(hostValidator, logger, params, - currentActiveApplicationSet, tenantPath, now, - serverDBSessionDir, applicationPackage, sessionZooKeeperClient); - setPrepared(); - waiter.awaitCompletion(params.getTimeoutBudget().timeLeft()); - return actions; } public ApplicationFile getApplicationFile(Path relativePath, Mode mode) { @@ -83,22 +44,22 @@ public class LocalSession extends Session { return applicationPackage.getFile(relativePath); } - private void setPrepared() { + void setPrepared() { setStatus(Session.Status.PREPARE); } private Transaction createSetStatusTransaction(Status status) { - return zooKeeperClient.createWriteStatusTransaction(status); + return sessionZooKeeperClient.createWriteStatusTransaction(status); } private void setStatus(Session.Status newStatus) { - zooKeeperClient.writeStatus(newStatus); + sessionZooKeeperClient.writeStatus(newStatus); } public Transaction createActivateTransaction() { - zooKeeperClient.createActiveWaiter(); + sessionZooKeeperClient.createActiveWaiter(); Transaction transaction = createSetStatusTransaction(Status.ACTIVATE); - transaction.add(applicationRepo.createPutTransaction(zooKeeperClient.readApplicationId(), getSessionId()).operations()); + transaction.add(applicationRepo.createPutTransaction(sessionZooKeeperClient.readApplicationId(), getSessionId()).operations()); return transaction; } @@ -110,79 +71,16 @@ public class LocalSession extends Session { return applicationPackage.getMetaData().getPreviousActiveGeneration(); } - /** Add transactions to delete this session to the given nested transaction */ - public void delete(NestedTransaction transaction) { - transaction.add(zooKeeperClient.deleteTransaction(), FileTransaction.class); - transaction.add(FileTransaction.from(FileOperations.delete(serverDBSessionDir.getAbsolutePath()))); - } - public void waitUntilActivated(TimeoutBudget timeoutBudget) { - zooKeeperClient.getActiveWaiter().awaitCompletion(timeoutBudget.timeLeft()); + sessionZooKeeperClient.getActiveWaiter().awaitCompletion(timeoutBudget.timeLeft()); } public enum Mode { READ, WRITE } - public ApplicationMetaData getMetaData() { - return applicationPackage.getMetaData(); - } - - // The rest of this class should be moved elsewhere ... - - private static class FileTransaction extends AbstractTransaction { - - public static FileTransaction from(FileOperation operation) { - FileTransaction transaction = new FileTransaction(); - transaction.add(operation); - return transaction; - } - - @Override - public void prepare() { } + public ApplicationMetaData getMetaData() { return applicationPackage.getMetaData(); } - @Override - public void commit() { - for (Operation operation : operations()) - ((FileOperation)operation).commit(); - } - - } - - /** Factory for file operations */ - private static class FileOperations { - - /** Creates an operation which recursively deletes the given path */ - public static DeleteOperation delete(String pathToDelete) { - return new DeleteOperation(pathToDelete); - } - - } - - private interface FileOperation extends Transaction.Operation { - - void commit(); - - } - - /** - * Recursively deletes this path and everything below. - * Succeeds with no action if the path does not exist. - */ - private static class DeleteOperation implements FileOperation { - - private final String pathToDelete; - - DeleteOperation(String pathToDelete) { - this.pathToDelete = pathToDelete; - } - - @Override - public void commit() { - // TODO: Check delete access in prepare() - IOUtils.recursiveDeleteDir(new File(pathToDelete)); - } - - } + public ApplicationPackage getApplicationPackage() { return applicationPackage; } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionRepo.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionRepo.java deleted file mode 100644 index e23552dee44..00000000000 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionRepo.java +++ /dev/null @@ -1,151 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.session; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.TenantName; -import com.yahoo.path.Path; -import com.yahoo.transaction.NestedTransaction; -import com.yahoo.vespa.config.server.GlobalComponentRegistry; -import com.yahoo.vespa.config.server.deploy.TenantFileSystemDirs; -import com.yahoo.vespa.config.server.tenant.TenantRepository; -import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; -import com.yahoo.vespa.curator.Curator; - -import java.io.File; -import java.io.FilenameFilter; -import java.time.Clock; -import java.time.Duration; -import java.time.Instant; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.Executor; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * - * Contains state for the local instance of the configserver. - * - * @author Ulf Lilleengen - */ -public class LocalSessionRepo { - - private static final Logger log = Logger.getLogger(LocalSessionRepo.class.getName()); - private static final FilenameFilter sessionApplicationsFilter = (dir, name) -> name.matches("\\d+"); - - private final SessionCache<LocalSession> sessionCache; - private final Map<Long, LocalSessionStateWatcher> sessionStateWatchers = new HashMap<>(); - private final Duration sessionLifetime; - private final Clock clock; - private final Curator curator; - private final Executor zkWatcherExecutor; - private final TenantFileSystemDirs tenantFileSystemDirs; - - public LocalSessionRepo(TenantName tenantName, GlobalComponentRegistry componentRegistry, SessionFactory sessionFactory) { - sessionCache = new SessionCache<>(); - this.clock = componentRegistry.getClock(); - this.curator = componentRegistry.getCurator(); - this.sessionLifetime = Duration.ofSeconds(componentRegistry.getConfigserverConfig().sessionLifetime()); - this.zkWatcherExecutor = command -> componentRegistry.getZkWatcherExecutor().execute(tenantName, command); - this.tenantFileSystemDirs = new TenantFileSystemDirs(componentRegistry.getConfigServerDB(), tenantName); - loadSessions(sessionFactory); - } - - public synchronized void addSession(LocalSession session) { - sessionCache.addSession(session); - Path sessionsPath = TenantRepository.getSessionsPath(session.getTenantName()); - long sessionId = session.getSessionId(); - Curator.FileCache fileCache = curator.createFileCache(sessionsPath.append(String.valueOf(sessionId)).append(ConfigCurator.SESSIONSTATE_ZK_SUBPATH).getAbsolute(), false); - sessionStateWatchers.put(sessionId, new LocalSessionStateWatcher(fileCache, session, this, zkWatcherExecutor)); - } - - public LocalSession getSession(long sessionId) { - return sessionCache.getSession(sessionId); - } - - public List<LocalSession> getSessions() { - return sessionCache.getSessions(); - } - - private void loadSessions(SessionFactory sessionFactory) { - File[] sessions = tenantFileSystemDirs.sessionsPath().listFiles(sessionApplicationsFilter); - if (sessions == null) { - return; - } - for (File session : sessions) { - try { - addSession(sessionFactory.createSessionFromId(Long.parseLong(session.getName()))); - } catch (IllegalArgumentException e) { - log.log(Level.WARNING, "Could not load session '" + - session.getAbsolutePath() + "':" + e.getMessage() + ", skipping it."); - } - } - } - - public void deleteExpiredSessions(Map<ApplicationId, Long> activeSessions) { - log.log(Level.FINE, "Purging old sessions"); - try { - for (LocalSession candidate : sessionCache.getSessions()) { - Instant createTime = candidate.getCreateTime(); - log.log(Level.FINE, "Candidate session for deletion: " + candidate.getSessionId() + ", created: " + createTime); - - // Sessions with state other than ACTIVATED - if (hasExpired(candidate) && !isActiveSession(candidate)) { - deleteSession(candidate); - } else if (createTime.plus(Duration.ofDays(1)).isBefore(clock.instant())) { - // Sessions with state ACTIVATE, but which are not actually active - ApplicationId applicationId = candidate.getApplicationId(); - Long activeSession = activeSessions.get(applicationId); - if (activeSession == null || activeSession != candidate.getSessionId()) { - deleteSession(candidate); - log.log(Level.INFO, "Deleted inactive session " + candidate.getSessionId() + " created " + - createTime + " for '" + applicationId + "'"); - } - } - } - // Make sure to catch here, to avoid executor just dying in case of issues ... - } catch (Throwable e) { - log.log(Level.WARNING, "Error when purging old sessions ", e); - } - log.log(Level.FINE, "Done purging old sessions"); - } - - private boolean hasExpired(LocalSession candidate) { - return (candidate.getCreateTime().plus(sessionLifetime).isBefore(clock.instant())); - } - - private boolean isActiveSession(LocalSession candidate) { - return candidate.getStatus() == Session.Status.ACTIVATE; - } - - public void deleteSession(LocalSession session) { - long sessionId = session.getSessionId(); - log.log(Level.FINE, "Deleting local session " + sessionId); - LocalSessionStateWatcher watcher = sessionStateWatchers.remove(sessionId); - if (watcher != null) watcher.close(); - sessionCache.removeSession(sessionId); - NestedTransaction transaction = new NestedTransaction(); - session.delete(transaction); - transaction.commit(); - } - - public void close() { - deleteAllSessions(); - tenantFileSystemDirs.delete(); - } - - private void deleteAllSessions() { - List<LocalSession> sessions = new ArrayList<>(sessionCache.getSessions()); - for (LocalSession session : sessions) { - deleteSession(session); - } - } - - @Override - public String toString() { - return getSessions().toString(); - } - -} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionStateWatcher.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionStateWatcher.java index 662094fc0ca..acbb1dc81ce 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionStateWatcher.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionStateWatcher.java @@ -21,14 +21,14 @@ public class LocalSessionStateWatcher { private final Curator.FileCache fileCache; private final LocalSession session; - private final LocalSessionRepo localSessionRepo; + private final SessionRepository sessionRepository; private final Executor zkWatcherExecutor; LocalSessionStateWatcher(Curator.FileCache fileCache, LocalSession session, - LocalSessionRepo localSessionRepo, Executor zkWatcherExecutor) { + SessionRepository sessionRepository, Executor zkWatcherExecutor) { this.fileCache = fileCache; this.session = session; - this.localSessionRepo = localSessionRepo; + this.sessionRepository = sessionRepository; this.zkWatcherExecutor = zkWatcherExecutor; this.fileCache.start(); this.fileCache.addListener(this::nodeChanged); @@ -40,9 +40,9 @@ public class LocalSessionStateWatcher { log.log(status == Session.Status.DELETE ? Level.INFO : Level.FINE, session.logPre() + "Session change: Local session " + sessionId + " changed status to " + status); - if (status.equals(Session.Status.DELETE) && localSessionRepo.getSession(sessionId) != null) { + if (status.equals(Session.Status.DELETE) && sessionRepository.getLocalSession(sessionId) != null) { log.log(Level.FINE, session.logPre() + "Deleting session " + sessionId); - localSessionRepo.deleteSession(session); + sessionRepository.deleteLocalSession(session); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java index c1179a2dd17..763c77f2088 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java @@ -50,20 +50,20 @@ public class RemoteSession extends Session { } void loadPrepared() { - Curator.CompletionWaiter waiter = zooKeeperClient.getPrepareWaiter(); + Curator.CompletionWaiter waiter = sessionZooKeeperClient.getPrepareWaiter(); ensureApplicationLoaded(); notifyCompletion(waiter); } private ApplicationSet loadApplication() { - ApplicationPackage applicationPackage = zooKeeperClient.loadApplicationPackage(); + ApplicationPackage applicationPackage = sessionZooKeeperClient.loadApplicationPackage(); // Read hosts allocated on the config server instance which created this Optional<AllocatedHosts> allocatedHosts = applicationPackage.getAllocatedHosts(); - return ApplicationSet.fromList(applicationLoader.buildModels(zooKeeperClient.readApplicationId(), - zooKeeperClient.readDockerImageRepository(), - zooKeeperClient.readVespaVersion(), + return ApplicationSet.fromList(applicationLoader.buildModels(sessionZooKeeperClient.readApplicationId(), + sessionZooKeeperClient.readDockerImageRepository(), + sessionZooKeeperClient.readVespaVersion(), applicationPackage, new SettableOptional<>(allocatedHosts), clock.instant())); @@ -78,11 +78,11 @@ public class RemoteSession extends Session { } public Transaction createDeleteTransaction() { - return zooKeeperClient.createWriteStatusTransaction(Status.DELETE); + return sessionZooKeeperClient.createWriteStatusTransaction(Status.DELETE); } void makeActive(ReloadHandler reloadHandler) { - Curator.CompletionWaiter waiter = zooKeeperClient.getActiveWaiter(); + Curator.CompletionWaiter waiter = sessionZooKeeperClient.getActiveWaiter(); log.log(Level.FINE, () -> logPre() + "Getting session from repo: " + getSessionId()); ApplicationSet app = ensureApplicationLoaded(); log.log(Level.FINE, () -> logPre() + "Reloading config for " + getSessionId()); @@ -93,7 +93,7 @@ public class RemoteSession extends Session { } void confirmUpload() { - Curator.CompletionWaiter waiter = zooKeeperClient.getUploadWaiter(); + Curator.CompletionWaiter waiter = sessionZooKeeperClient.getUploadWaiter(); log.log(Level.FINE, "Notifying upload waiter for session " + getSessionId()); notifyCompletion(waiter); log.log(Level.FINE, "Done notifying upload for session " + getSessionId()); @@ -116,13 +116,13 @@ public class RemoteSession extends Session { } public void delete() { - Transaction transaction = zooKeeperClient.deleteTransaction(); + Transaction transaction = sessionZooKeeperClient.deleteTransaction(); transaction.commit(); transaction.close(); } public ApplicationMetaData getMetaData() { - return zooKeeperClient.loadApplicationPackage().getMetaData(); + return sessionZooKeeperClient.loadApplicationPackage().getMetaData(); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionRepo.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionRepo.java deleted file mode 100644 index 0e538b05931..00000000000 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionRepo.java +++ /dev/null @@ -1,241 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.session; - -import com.google.common.collect.HashMultiset; -import com.google.common.collect.Multiset; -import com.yahoo.concurrent.StripedExecutor; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.TenantName; - -import java.time.Clock; -import java.util.logging.Level; -import com.yahoo.path.Path; -import com.yahoo.vespa.config.server.GlobalComponentRegistry; -import com.yahoo.vespa.config.server.ReloadHandler; -import com.yahoo.vespa.config.server.application.TenantApplications; -import com.yahoo.vespa.config.server.monitoring.MetricUpdater; -import com.yahoo.vespa.config.server.monitoring.Metrics; -import com.yahoo.vespa.config.server.tenant.TenantRepository; -import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; -import com.yahoo.vespa.curator.Curator; -import com.yahoo.vespa.flags.BooleanFlag; -import com.yahoo.vespa.flags.FlagSource; -import com.yahoo.vespa.flags.Flags; -import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.recipes.cache.ChildData; -import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; - -import java.time.Duration; -import java.time.Instant; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.Executor; -import java.util.logging.Logger; -import java.util.stream.Collectors; - -/** - * Session repository for RemoteSessions. There is one such repo per tenant. - * Will watch/prepare sessions (applications) based on watched nodes in ZooKeeper. The zookeeper state watched in - * this class is shared between all config servers, so it should not modify any global state, because the operation - * will be performed on all servers. The repo can be regarded as read only from the POV of the configserver. - * - * @author Vegard Havdal - * @author Ulf Lilleengen - */ -public class RemoteSessionRepo { - - private static final Logger log = Logger.getLogger(RemoteSessionRepo.class.getName()); - - private final Curator curator; - private final Path sessionsPath; - private final SessionFactory sessionFactory; - private final Map<Long, RemoteSessionStateWatcher> sessionStateWatchers = new HashMap<>(); - private final ReloadHandler reloadHandler; - private final TenantName tenantName; - private final MetricUpdater metrics; - private final BooleanFlag distributeApplicationPackage; - private final Curator.DirectoryCache directoryCache; - private final TenantApplications applicationRepo; - private final Executor zkWatcherExecutor; - private final SessionCache<RemoteSession> sessionCache; - - public RemoteSessionRepo(GlobalComponentRegistry componentRegistry, - SessionFactory sessionFactory, - ReloadHandler reloadHandler, - TenantName tenantName, - TenantApplications applicationRepo, - FlagSource flagSource) { - this.sessionCache = new SessionCache<>(); - this.curator = componentRegistry.getCurator(); - this.sessionsPath = TenantRepository.getSessionsPath(tenantName); - this.applicationRepo = applicationRepo; - this.sessionFactory = sessionFactory; - this.reloadHandler = reloadHandler; - this.tenantName = tenantName; - this.metrics = componentRegistry.getMetrics().getOrCreateMetricUpdater(Metrics.createDimensions(tenantName)); - this.distributeApplicationPackage = Flags.CONFIGSERVER_DISTRIBUTE_APPLICATION_PACKAGE.bindTo(flagSource); - StripedExecutor<TenantName> zkWatcherExecutor = componentRegistry.getZkWatcherExecutor(); - this.zkWatcherExecutor = command -> zkWatcherExecutor.execute(tenantName, command); - initializeSessions(); - this.directoryCache = curator.createDirectoryCache(sessionsPath.getAbsolute(), false, false, componentRegistry.getZkCacheExecutor()); - this.directoryCache.addListener(this::childEvent); - this.directoryCache.start(); - } - - public RemoteSession getSession(long sessionId) { - return sessionCache.getSession(sessionId); - } - - public List<Long> getSessions() { - return getSessionList(curator.getChildren(sessionsPath)); - } - - public void addSession(RemoteSession session) { - sessionCache.addSession(session); - metrics.incAddedSessions(); - } - - public int deleteExpiredSessions(Clock clock, Duration expiryTime) { - int deleted = 0; - for (long sessionId : getSessions()) { - RemoteSession session = sessionCache.getSession(sessionId); - if (session == null) continue; // Internal sessions not in synch with zk, continue - if (session.getStatus() == Session.Status.ACTIVATE) continue; - if (sessionHasExpired(session.getCreateTime(), expiryTime, clock)) { - log.log(Level.INFO, "Remote session " + sessionId + " for " + tenantName + " has expired, deleting it"); - session.delete(); - deleted++; - } - } - return deleted; - } - - private boolean sessionHasExpired(Instant created, Duration expiryTime, Clock clock) { - return (created.plus(expiryTime).isBefore(clock.instant())); - } - - private List<Long> getSessionListFromDirectoryCache(List<ChildData> children) { - return getSessionList(children.stream() - .map(child -> Path.fromString(child.getPath()).getName()) - .collect(Collectors.toList())); - } - - private List<Long> getSessionList(List<String> children) { - return children.stream().map(Long::parseLong).collect(Collectors.toList()); - } - - private void initializeSessions() throws NumberFormatException { - getSessions().forEach(this::sessionAdded); - } - - private synchronized void sessionsChanged() throws NumberFormatException { - List<Long> sessions = getSessionListFromDirectoryCache(directoryCache.getCurrentData()); - checkForRemovedSessions(sessions); - checkForAddedSessions(sessions); - } - - private void checkForRemovedSessions(List<Long> sessions) { - for (RemoteSession session : sessionCache.getSessions()) - if ( ! sessions.contains(session.getSessionId())) - sessionRemoved(session.getSessionId()); - } - - private void checkForAddedSessions(List<Long> sessions) { - for (Long sessionId : sessions) - if (sessionCache.getSession(sessionId) == null) - sessionAdded(sessionId); - } - - /** - * A session for which we don't have a watcher, i.e. hitherto unknown to us. - * - * @param sessionId session id for the new session - */ - private void sessionAdded(long sessionId) { - log.log(Level.FINE, () -> "Adding session to RemoteSessionRepo: " + sessionId); - RemoteSession session = sessionFactory.createRemoteSession(sessionId); - Path sessionPath = sessionsPath.append(String.valueOf(sessionId)); - Curator.FileCache fileCache = curator.createFileCache(sessionPath.append(ConfigCurator.SESSIONSTATE_ZK_SUBPATH).getAbsolute(), false); - fileCache.addListener(this::nodeChanged); - loadSessionIfActive(session); - addSession(session); - sessionStateWatchers.put(sessionId, new RemoteSessionStateWatcher(fileCache, reloadHandler, session, metrics, zkWatcherExecutor)); - if (distributeApplicationPackage.value()) - sessionFactory.createLocalSessionUsingDistributedApplicationPackage(sessionId); - } - - private void sessionRemoved(long sessionId) { - RemoteSessionStateWatcher watcher = sessionStateWatchers.remove(sessionId); - if (watcher != null) watcher.close(); - sessionCache.removeSession(sessionId); - metrics.incRemovedSessions(); - } - - private void loadSessionIfActive(RemoteSession session) { - for (ApplicationId applicationId : applicationRepo.activeApplications()) { - if (applicationRepo.requireActiveSessionOf(applicationId) == session.getSessionId()) { - log.log(Level.FINE, () -> "Found active application for session " + session.getSessionId() + " , loading it"); - reloadHandler.reloadConfig(session.ensureApplicationLoaded()); - log.log(Level.INFO, session.logPre() + "Application activated successfully: " + applicationId + " (generation " + session.getSessionId() + ")"); - return; - } - } - } - - public synchronized void close() { - try { - if (directoryCache != null) { - directoryCache.close(); - } - } catch (Exception e) { - log.log(Level.WARNING, "Exception when closing path cache", e); - } finally { - checkForRemovedSessions(new ArrayList<>()); - } - } - - private void nodeChanged() { - zkWatcherExecutor.execute(() -> { - Multiset<Session.Status> sessionMetrics = HashMultiset.create(); - for (RemoteSession session : sessionCache.getSessions()) { - sessionMetrics.add(session.getStatus()); - } - metrics.setNewSessions(sessionMetrics.count(Session.Status.NEW)); - metrics.setPreparedSessions(sessionMetrics.count(Session.Status.PREPARE)); - metrics.setActivatedSessions(sessionMetrics.count(Session.Status.ACTIVATE)); - metrics.setDeactivatedSessions(sessionMetrics.count(Session.Status.DEACTIVATE)); - }); - } - - @SuppressWarnings("unused") - private void childEvent(CuratorFramework ignored, PathChildrenCacheEvent event) { - zkWatcherExecutor.execute(() -> { - log.log(Level.FINE, () -> "Got child event: " + event); - switch (event.getType()) { - case CHILD_ADDED: - sessionsChanged(); - synchronizeOnNew(getSessionListFromDirectoryCache(Collections.singletonList(event.getData()))); - break; - case CHILD_REMOVED: - sessionsChanged(); - break; - case CONNECTION_RECONNECTED: - sessionsChanged(); - break; - } - }); - } - - private void synchronizeOnNew(List<Long> sessionList) { - for (long sessionId : sessionList) { - RemoteSession session = sessionCache.getSession(sessionId); - if (session == null) continue; // session might have been deleted after getting session list - log.log(Level.FINE, () -> session.logPre() + "Confirming upload for session " + sessionId); - session.confirmUpload(); - } - } - -} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java index c553133ba12..1e832548342 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java @@ -25,12 +25,12 @@ public abstract class Session implements Comparable<Session> { private final long sessionId; protected final TenantName tenant; - protected final SessionZooKeeperClient zooKeeperClient; + protected final SessionZooKeeperClient sessionZooKeeperClient; - protected Session(TenantName tenant, long sessionId, SessionZooKeeperClient zooKeeperClient) { + protected Session(TenantName tenant, long sessionId, SessionZooKeeperClient sessionZooKeeperClient) { this.tenant = tenant; this.sessionId = sessionId; - this.zooKeeperClient = zooKeeperClient; + this.sessionZooKeeperClient = sessionZooKeeperClient; } /** * Retrieve the session id for this session. @@ -41,7 +41,7 @@ public abstract class Session implements Comparable<Session> { } public Session.Status getStatus() { - return zooKeeperClient.readStatus(); + return sessionZooKeeperClient.readStatus(); } @Override @@ -80,43 +80,43 @@ public abstract class Session implements Comparable<Session> { } public Instant getCreateTime() { - return zooKeeperClient.readCreateTime(); + return sessionZooKeeperClient.readCreateTime(); } public void setApplicationId(ApplicationId applicationId) { - zooKeeperClient.writeApplicationId(applicationId); + sessionZooKeeperClient.writeApplicationId(applicationId); } void setApplicationPackageReference(FileReference applicationPackageReference) { if (applicationPackageReference == null) throw new IllegalArgumentException(String.format( "Null application package FileReference for tenant: %s, session: %d", tenant, sessionId)); - zooKeeperClient.writeApplicationPackageReference(applicationPackageReference); + sessionZooKeeperClient.writeApplicationPackageReference(applicationPackageReference); } public void setVespaVersion(Version version) { - zooKeeperClient.writeVespaVersion(version); + sessionZooKeeperClient.writeVespaVersion(version); } public void setDockerImageRepository(Optional<DockerImage> dockerImageRepository) { - zooKeeperClient.writeDockerImageRepository(dockerImageRepository); + sessionZooKeeperClient.writeDockerImageRepository(dockerImageRepository); } public void setAthenzDomain(Optional<AthenzDomain> athenzDomain) { - zooKeeperClient.writeAthenzDomain(athenzDomain); + sessionZooKeeperClient.writeAthenzDomain(athenzDomain); } - public ApplicationId getApplicationId() { return zooKeeperClient.readApplicationId(); } + public ApplicationId getApplicationId() { return sessionZooKeeperClient.readApplicationId(); } - public FileReference getApplicationPackageReference() {return zooKeeperClient.readApplicationPackageReference(); } + public FileReference getApplicationPackageReference() {return sessionZooKeeperClient.readApplicationPackageReference(); } - public Optional<DockerImage> getDockerImageRepository() { return zooKeeperClient.readDockerImageRepository(); } + public Optional<DockerImage> getDockerImageRepository() { return sessionZooKeeperClient.readDockerImageRepository(); } - public Version getVespaVersion() { return zooKeeperClient.readVespaVersion(); } + public Version getVespaVersion() { return sessionZooKeeperClient.readVespaVersion(); } - public Optional<AthenzDomain> getAthenzDomain() { return zooKeeperClient.readAthenzDomain(); } + public Optional<AthenzDomain> getAthenzDomain() { return sessionZooKeeperClient.readAthenzDomain(); } public AllocatedHosts getAllocatedHosts() { - return zooKeeperClient.getAllocatedHosts(); + return sessionZooKeeperClient.getAllocatedHosts(); } public Transaction createDeactivateTransaction() { @@ -124,7 +124,7 @@ public abstract class Session implements Comparable<Session> { } private Transaction createSetStatusTransaction(Status status) { - return zooKeeperClient.createWriteStatusTransaction(status); + return sessionZooKeeperClient.createWriteStatusTransaction(status); } // Note: Assumes monotonically increasing session ids diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionCache.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionCache.java index 8808dc0cf75..501d4918996 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionCache.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionCache.java @@ -16,9 +16,7 @@ public class SessionCache<SESSIONTYPE extends Session> { private final HashMap<Long, SESSIONTYPE> sessions = new HashMap<>(); public synchronized void addSession(SESSIONTYPE session) { - if (sessions.containsKey(session.getSessionId())) - throw new IllegalArgumentException("There already exists a session with id '" + session.getSessionId() + "'"); - sessions.put(session.getSessionId(), session); + sessions.putIfAbsent(session.getSessionId(), session); } synchronized void removeSession(long id) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactory.java deleted file mode 100644 index 8aba9fa465d..00000000000 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactory.java +++ /dev/null @@ -1,280 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.session; - -import com.yahoo.config.FileReference; -import com.yahoo.config.application.api.ApplicationPackage; -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.config.model.application.provider.DeployData; -import com.yahoo.config.model.application.provider.FilesApplicationPackage; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.NodeFlavors; -import com.yahoo.config.provision.TenantName; -import com.yahoo.io.IOUtils; -import com.yahoo.path.Path; -import com.yahoo.vespa.config.server.GlobalComponentRegistry; -import com.yahoo.vespa.config.server.TimeoutBudget; -import com.yahoo.vespa.config.server.application.TenantApplications; -import com.yahoo.vespa.config.server.deploy.TenantFileSystemDirs; -import com.yahoo.vespa.config.server.filedistribution.FileDirectory; -import com.yahoo.vespa.config.server.host.HostValidator; -import com.yahoo.vespa.config.server.tenant.TenantRepository; -import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; -import com.yahoo.vespa.config.server.zookeeper.SessionCounter; -import com.yahoo.vespa.curator.Curator; -import com.yahoo.vespa.defaults.Defaults; -import com.yahoo.vespa.flags.BooleanFlag; -import com.yahoo.vespa.flags.Flags; - -import java.io.File; -import java.io.IOException; -import java.time.Clock; -import java.util.List; -import java.util.Optional; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * Serves as the factory of sessions. Takes care of copying files to the correct folder and initializing the - * session state. There is one SessionFactory per tenant. - * - * @author Ulf Lilleengen - */ -public class SessionFactory { - - private static final Logger log = Logger.getLogger(SessionFactory.class.getName()); - private static final long nonExistingActiveSession = 0; - - private final SessionPreparer sessionPreparer; - private final Curator curator; - private final ConfigCurator configCurator; - private final TenantApplications applicationRepo; - private final Path sessionsPath; - private final GlobalComponentRegistry componentRegistry; - private final HostValidator<ApplicationId> hostRegistry; - private final TenantName tenant; - private final String serverId; - private final Optional<NodeFlavors> nodeFlavors; - private final Clock clock; - private final BooleanFlag distributeApplicationPackage; - - public SessionFactory(GlobalComponentRegistry globalComponentRegistry, - TenantApplications applicationRepo, - HostValidator<ApplicationId> hostRegistry, - TenantName tenant) { - this.hostRegistry = hostRegistry; - this.tenant = tenant; - this.sessionPreparer = globalComponentRegistry.getSessionPreparer(); - this.curator = globalComponentRegistry.getCurator(); - this.configCurator = globalComponentRegistry.getConfigCurator(); - this.sessionsPath = TenantRepository.getSessionsPath(tenant); - this.applicationRepo = applicationRepo; - this.componentRegistry = globalComponentRegistry; - this.serverId = globalComponentRegistry.getConfigserverConfig().serverId(); - this.nodeFlavors = globalComponentRegistry.getZone().nodeFlavors(); - this.clock = globalComponentRegistry.getClock(); - this.distributeApplicationPackage = Flags.CONFIGSERVER_DISTRIBUTE_APPLICATION_PACKAGE - .bindTo(globalComponentRegistry.getFlagSource()); - } - - /** - * Creates a new deployment session from an application package. - * - * @param applicationDirectory a File pointing to an application. - * @param applicationId application id for this new session. - * @param timeoutBudget Timeout for creating session and waiting for other servers. - * @return a new session - */ - public LocalSession createSession(File applicationDirectory, ApplicationId applicationId, - TimeoutBudget timeoutBudget, Optional<Long> activeSessionId) { - return create(applicationDirectory, applicationId, activeSessionId.orElse(nonExistingActiveSession), false, timeoutBudget); - } - - public RemoteSession createRemoteSession(long sessionId) { - SessionZooKeeperClient sessionZKClient = createSessionZooKeeperClient(getSessionPath(sessionId)); - return new RemoteSession(tenant, sessionId, componentRegistry, sessionZKClient); - } - - private void ensureSessionPathDoesNotExist(long sessionId) { - Path sessionPath = getSessionPath(sessionId); - if (configCurator.exists(sessionPath.getAbsolute())) { - throw new IllegalArgumentException("Path " + sessionPath.getAbsolute() + " already exists in ZooKeeper"); - } - } - - private ApplicationPackage createApplication(File userDir, - File configApplicationDir, - ApplicationId applicationId, - long sessionId, - long currentlyActiveSessionId, - boolean internalRedeploy) { - long deployTimestamp = System.currentTimeMillis(); - String user = System.getenv("USER"); - if (user == null) { - user = "unknown"; - } - DeployData deployData = new DeployData(user, userDir.getAbsolutePath(), applicationId, deployTimestamp, internalRedeploy, sessionId, currentlyActiveSessionId); - return FilesApplicationPackage.fromFileWithDeployData(configApplicationDir, deployData); - } - - private LocalSession createSessionFromApplication(ApplicationPackage applicationPackage, - long sessionId, - TimeoutBudget timeoutBudget, - Clock clock) { - log.log(Level.FINE, TenantRepository.logPre(tenant) + "Creating session " + sessionId + " in ZooKeeper"); - SessionZooKeeperClient sessionZKClient = createSessionZooKeeperClient(getSessionPath(sessionId)); - sessionZKClient.createNewSession(clock.instant()); - Curator.CompletionWaiter waiter = sessionZKClient.getUploadWaiter(); - LocalSession session = new LocalSession(tenant, sessionId, sessionPreparer, applicationPackage, sessionZKClient, - getSessionAppDir(sessionId), applicationRepo, hostRegistry); - waiter.awaitCompletion(timeoutBudget.timeLeft()); - return session; - } - - /** - * Creates a new deployment session from an already existing session. - * - * @param existingSession the session to use as base - * @param logger a deploy logger where the deploy log will be written. - * @param internalRedeploy whether this session is for a system internal redeploy — not an application package change - * @param timeoutBudget timeout for creating session and waiting for other servers. - * @return a new session - */ - public LocalSession createSessionFromExisting(Session existingSession, - DeployLogger logger, - boolean internalRedeploy, - TimeoutBudget timeoutBudget) { - File existingApp = getSessionAppDir(existingSession.getSessionId()); - ApplicationId existingApplicationId = existingSession.getApplicationId(); - - long activeSessionId = getActiveSessionId(existingApplicationId); - logger.log(Level.FINE, "Create new session for application id '" + existingApplicationId + "' from existing active session " + activeSessionId); - LocalSession session = create(existingApp, existingApplicationId, activeSessionId, internalRedeploy, timeoutBudget); - // Note: Needs to be kept in sync with calls in SessionPreparer.writeStateToZooKeeper() - session.setApplicationId(existingApplicationId); - if (distributeApplicationPackage.value() && existingSession.getApplicationPackageReference() != null) { - session.setApplicationPackageReference(existingSession.getApplicationPackageReference()); - } - session.setVespaVersion(existingSession.getVespaVersion()); - session.setDockerImageRepository(existingSession.getDockerImageRepository()); - session.setAthenzDomain(existingSession.getAthenzDomain()); - return session; - } - - private LocalSession create(File applicationFile, ApplicationId applicationId, long currentlyActiveSessionId, - boolean internalRedeploy, TimeoutBudget timeoutBudget) { - long sessionId = getNextSessionId(); - try { - ensureSessionPathDoesNotExist(sessionId); - ApplicationPackage app = createApplicationPackage(applicationFile, applicationId, - sessionId, currentlyActiveSessionId, internalRedeploy); - return createSessionFromApplication(app, sessionId, timeoutBudget, clock); - } catch (Exception e) { - throw new RuntimeException("Error creating session " + sessionId, e); - } - } - - /** - * This method is used when creating a session based on a remote session and the distributed application package - * It does not wait for session being created on other servers - */ - private LocalSession createLocalSession(File applicationFile, ApplicationId applicationId, - long sessionId, long currentlyActiveSessionId) { - try { - ApplicationPackage applicationPackage = createApplicationPackage(applicationFile, applicationId, - sessionId, currentlyActiveSessionId, false); - SessionZooKeeperClient sessionZooKeeperClient = createSessionZooKeeperClient(getSessionPath(sessionId)); - return new LocalSession(tenant, sessionId, sessionPreparer, applicationPackage, sessionZooKeeperClient, - getSessionAppDir(sessionId), applicationRepo, hostRegistry); - } catch (Exception e) { - throw new RuntimeException("Error creating session " + sessionId, e); - } - } - - private ApplicationPackage createApplicationPackage(File applicationFile, ApplicationId applicationId, - long sessionId, long currentlyActiveSessionId, - boolean internalRedeploy) throws IOException { - File userApplicationDir = getSessionAppDir(sessionId); - IOUtils.copyDirectory(applicationFile, userApplicationDir); - ApplicationPackage applicationPackage = createApplication(applicationFile, - userApplicationDir, - applicationId, - sessionId, - currentlyActiveSessionId, - internalRedeploy); - applicationPackage.writeMetaData(); - return applicationPackage; - } - - /** - * Returns a new session instance for the given session id. - */ - LocalSession createSessionFromId(long sessionId) { - File sessionDir = getAndValidateExistingSessionAppDir(sessionId); - ApplicationPackage applicationPackage = FilesApplicationPackage.fromFile(sessionDir); - Path sessionIdPath = sessionsPath.append(String.valueOf(sessionId)); - SessionZooKeeperClient sessionZKClient = createSessionZooKeeperClient(sessionIdPath); - return new LocalSession(tenant, sessionId, sessionPreparer, applicationPackage, sessionZKClient, - getSessionAppDir(sessionId), applicationRepo, hostRegistry); - } - - /** - * Returns a new session instance for the given session id. - */ - LocalSession createLocalSessionUsingDistributedApplicationPackage(long sessionId) { - if (applicationRepo.hasLocalSession(sessionId)) { - log.log(Level.FINE, "Local session for session id " + sessionId + " already exists"); - return createSessionFromId(sessionId); - } - - log.log(Level.INFO, "Creating local session for session id " + sessionId); - SessionZooKeeperClient sessionZKClient = createSessionZooKeeperClient(getSessionPath(sessionId)); - FileReference fileReference = sessionZKClient.readApplicationPackageReference(); - log.log(Level.FINE, "File reference for session id " + sessionId + ": " + fileReference); - if (fileReference != null) { - File rootDir = new File(Defaults.getDefaults().underVespaHome(componentRegistry.getConfigserverConfig().fileReferencesDir())); - File sessionDir = new FileDirectory(rootDir).getFile(fileReference); - if (!sessionDir.exists()) - throw new RuntimeException("File reference for session " + sessionId + " not found (" + sessionDir.getAbsolutePath() + ")"); - ApplicationId applicationId = sessionZKClient.readApplicationId(); - return createLocalSession(sessionDir, - applicationId, - sessionId, - applicationRepo.activeSessionOf(applicationId).orElse(nonExistingActiveSession)); - } - return null; - } - - // Return Optional instead of faking it with nonExistingActiveSession - private long getActiveSessionId(ApplicationId applicationId) { - List<ApplicationId> applicationIds = applicationRepo.activeApplications(); - if (applicationIds.contains(applicationId)) { - return applicationRepo.requireActiveSessionOf(applicationId); - } - return nonExistingActiveSession; - } - - private long getNextSessionId() { - return new SessionCounter(componentRegistry.getConfigCurator(), tenant).nextSessionId(); - } - - private Path getSessionPath(long sessionId) { - return sessionsPath.append(String.valueOf(sessionId)); - } - - private SessionZooKeeperClient createSessionZooKeeperClient(Path sessionPath) { - return new SessionZooKeeperClient(curator, configCurator, sessionPath, serverId, nodeFlavors); - } - - private File getAndValidateExistingSessionAppDir(long sessionId) { - File appDir = getSessionAppDir(sessionId); - if (!appDir.exists() || !appDir.isDirectory()) { - throw new IllegalArgumentException("Unable to find correct application directory for session " + sessionId); - } - return appDir; - } - - private File getSessionAppDir(long sessionId) { - return new TenantFileSystemDirs(componentRegistry.getConfigServerDB(), tenant).getUserApplicationDir(sessionId); - } - -} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java index 4a2e7cb405b..3c83263f781 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java @@ -55,6 +55,7 @@ import java.io.File; import java.io.IOException; import java.net.URI; import java.time.Instant; +import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Optional; @@ -115,7 +116,7 @@ public class SessionPreparer { * @param logger For storing logs returned in response to client. * @param params parameters controlling behaviour of prepare. * @param currentActiveApplicationSet Set of currently active applications. - * @param tenantPath Zookeeper path for the tenant for this session + * @param tenantPath Zookeeper path for the tenant for this session * @return the config change actions that must be done to handle the activation of the models prepared. */ public ConfigChangeActions prepare(HostValidator<ApplicationId> hostValidator, DeployLogger logger, PrepareParams params, @@ -317,7 +318,7 @@ public class SessionPreparer { } private List<ContainerEndpoint> readEndpointsIfNull(List<ContainerEndpoint> endpoints) { - if (endpoints == null) { // endpoints is only set when prepared via HTTP + if (endpoints == null) { // endpoints are only set when prepared via HTTP endpoints = this.containerEndpointsCache.read(applicationId); } return List.copyOf(endpoints); @@ -383,7 +384,7 @@ public class SessionPreparer { */ public ConfigChangeActions getConfigChangeActions() { return new ConfigChangeActions(results.stream().map(result -> result.actions) - .flatMap(actions -> actions.stream()) + .flatMap(Collection::stream) .collect(Collectors.toList())); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java new file mode 100644 index 00000000000..8317dc02e90 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java @@ -0,0 +1,659 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.session; + +import com.google.common.collect.HashMultiset; +import com.google.common.collect.Multiset; +import com.yahoo.config.FileReference; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.application.provider.DeployData; +import com.yahoo.config.model.application.provider.FilesApplicationPackage; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.NodeFlavors; +import com.yahoo.config.provision.TenantName; +import com.yahoo.io.IOUtils; +import com.yahoo.path.Path; +import com.yahoo.transaction.AbstractTransaction; +import com.yahoo.transaction.NestedTransaction; +import com.yahoo.transaction.Transaction; +import com.yahoo.vespa.config.server.GlobalComponentRegistry; +import com.yahoo.vespa.config.server.ReloadHandler; +import com.yahoo.vespa.config.server.TimeoutBudget; +import com.yahoo.vespa.config.server.application.ApplicationSet; +import com.yahoo.vespa.config.server.application.TenantApplications; +import com.yahoo.vespa.config.server.configchange.ConfigChangeActions; +import com.yahoo.vespa.config.server.deploy.TenantFileSystemDirs; +import com.yahoo.vespa.config.server.filedistribution.FileDirectory; +import com.yahoo.vespa.config.server.monitoring.MetricUpdater; +import com.yahoo.vespa.config.server.monitoring.Metrics; +import com.yahoo.vespa.config.server.tenant.TenantRepository; +import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; +import com.yahoo.vespa.config.server.zookeeper.SessionCounter; +import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.defaults.Defaults; +import com.yahoo.vespa.flags.BooleanFlag; +import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.Flags; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.recipes.cache.ChildData; +import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; + +import java.io.File; +import java.io.FilenameFilter; +import java.io.IOException; +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.Executor; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +/** + * + * Session repository for a tenant. Stores session state in zookeeper and file system. There are two + * different session types (RemoteSession and LocalSession). + * + * @author Ulf Lilleengen + * @author hmusum + * + */ +public class SessionRepository { + + private static final Logger log = Logger.getLogger(SessionRepository.class.getName()); + private static final FilenameFilter sessionApplicationsFilter = (dir, name) -> name.matches("\\d+"); + private static final long nonExistingActiveSession = 0; + + private final SessionCache<LocalSession> localSessionCache = new SessionCache<>(); + private final SessionCache<RemoteSession> remoteSessionCache = new SessionCache<>(); + private final Map<Long, LocalSessionStateWatcher> localSessionStateWatchers = new HashMap<>(); + private final Map<Long, RemoteSessionStateWatcher> remoteSessionStateWatchers = new HashMap<>(); + private final Duration sessionLifetime; + private final Clock clock; + private final Curator curator; + private final Executor zkWatcherExecutor; + private final TenantFileSystemDirs tenantFileSystemDirs; + private final BooleanFlag distributeApplicationPackage; + private final ReloadHandler reloadHandler; + private final MetricUpdater metrics; + private final Curator.DirectoryCache directoryCache; + private final TenantApplications applicationRepo; + private final SessionPreparer sessionPreparer; + private final Path sessionsPath; + private final TenantName tenantName; + private final GlobalComponentRegistry componentRegistry; + + public SessionRepository(TenantName tenantName, + GlobalComponentRegistry componentRegistry, + TenantApplications applicationRepo, + ReloadHandler reloadHandler, + FlagSource flagSource, + SessionPreparer sessionPreparer) { + this.tenantName = tenantName; + this.componentRegistry = componentRegistry; + this.sessionsPath = TenantRepository.getSessionsPath(tenantName); + this.clock = componentRegistry.getClock(); + this.curator = componentRegistry.getCurator(); + this.sessionLifetime = Duration.ofSeconds(componentRegistry.getConfigserverConfig().sessionLifetime()); + this.zkWatcherExecutor = command -> componentRegistry.getZkWatcherExecutor().execute(tenantName, command); + this.tenantFileSystemDirs = new TenantFileSystemDirs(componentRegistry.getConfigServerDB(), tenantName); + this.applicationRepo = applicationRepo; + this.sessionPreparer = sessionPreparer; + this.distributeApplicationPackage = Flags.CONFIGSERVER_DISTRIBUTE_APPLICATION_PACKAGE.bindTo(flagSource); + this.reloadHandler = reloadHandler; + this.metrics = componentRegistry.getMetrics().getOrCreateMetricUpdater(Metrics.createDimensions(tenantName)); + + loadLocalSessions(); + initializeRemoteSessions(); + this.directoryCache = curator.createDirectoryCache(sessionsPath.getAbsolute(), false, false, componentRegistry.getZkCacheExecutor()); + this.directoryCache.addListener(this::childEvent); + this.directoryCache.start(); + } + + // ---------------- Local sessions ---------------------------------------------------------------- + + public synchronized void addSession(LocalSession session) { + localSessionCache.addSession(session); + Path sessionsPath = TenantRepository.getSessionsPath(session.getTenantName()); + long sessionId = session.getSessionId(); + Curator.FileCache fileCache = curator.createFileCache(sessionsPath.append(String.valueOf(sessionId)).append(ConfigCurator.SESSIONSTATE_ZK_SUBPATH).getAbsolute(), false); + localSessionStateWatchers.put(sessionId, new LocalSessionStateWatcher(fileCache, session, this, zkWatcherExecutor)); + } + + public LocalSession getLocalSession(long sessionId) { + return localSessionCache.getSession(sessionId); + } + + public List<LocalSession> getLocalSessions() { + return localSessionCache.getSessions(); + } + + private void loadLocalSessions() { + File[] sessions = tenantFileSystemDirs.sessionsPath().listFiles(sessionApplicationsFilter); + if (sessions == null) { + return; + } + for (File session : sessions) { + try { + addSession(createSessionFromId(Long.parseLong(session.getName()))); + } catch (IllegalArgumentException e) { + log.log(Level.WARNING, "Could not load session '" + + session.getAbsolutePath() + "':" + e.getMessage() + ", skipping it."); + } + } + } + + public ConfigChangeActions prepareLocalSession(LocalSession session, + DeployLogger logger, + PrepareParams params, + Optional<ApplicationSet> currentActiveApplicationSet, + Path tenantPath, + Instant now) { + applicationRepo.createApplication(params.getApplicationId()); // TODO jvenstad: This is wrong, but it has to be done now, since preparation can change the application ID of a session :( + logger.log(Level.FINE, "Created application " + params.getApplicationId()); + long sessionId = session.getSessionId(); + SessionZooKeeperClient sessionZooKeeperClient = createSessionZooKeeperClient(sessionId); + Curator.CompletionWaiter waiter = sessionZooKeeperClient.createPrepareWaiter(); + ConfigChangeActions actions = sessionPreparer.prepare(applicationRepo.getHostValidator(), logger, params, + currentActiveApplicationSet, tenantPath, now, + getSessionAppDir(sessionId), + session.getApplicationPackage(), sessionZooKeeperClient); + session.setPrepared(); + waiter.awaitCompletion(params.getTimeoutBudget().timeLeft()); + return actions; + } + + public void deleteExpiredSessions(Map<ApplicationId, Long> activeSessions) { + log.log(Level.FINE, "Purging old sessions"); + try { + for (LocalSession candidate : localSessionCache.getSessions()) { + Instant createTime = candidate.getCreateTime(); + log.log(Level.FINE, "Candidate session for deletion: " + candidate.getSessionId() + ", created: " + createTime); + + // Sessions with state other than ACTIVATED + if (hasExpired(candidate) && !isActiveSession(candidate)) { + deleteLocalSession(candidate); + } else if (createTime.plus(Duration.ofDays(1)).isBefore(clock.instant())) { + // Sessions with state ACTIVATE, but which are not actually active + ApplicationId applicationId = candidate.getApplicationId(); + Long activeSession = activeSessions.get(applicationId); + if (activeSession == null || activeSession != candidate.getSessionId()) { + deleteLocalSession(candidate); + log.log(Level.INFO, "Deleted inactive session " + candidate.getSessionId() + " created " + + createTime + " for '" + applicationId + "'"); + } + } + } + // Make sure to catch here, to avoid executor just dying in case of issues ... + } catch (Throwable e) { + log.log(Level.WARNING, "Error when purging old sessions ", e); + } + log.log(Level.FINE, "Done purging old sessions"); + } + + private boolean hasExpired(LocalSession candidate) { + return (candidate.getCreateTime().plus(sessionLifetime).isBefore(clock.instant())); + } + + private boolean isActiveSession(LocalSession candidate) { + return candidate.getStatus() == Session.Status.ACTIVATE; + } + + public void deleteLocalSession(LocalSession session) { + long sessionId = session.getSessionId(); + log.log(Level.FINE, "Deleting local session " + sessionId); + LocalSessionStateWatcher watcher = localSessionStateWatchers.remove(sessionId); + if (watcher != null) watcher.close(); + localSessionCache.removeSession(sessionId); + NestedTransaction transaction = new NestedTransaction(); + deleteLocalSession(session, transaction); + transaction.commit(); + } + + /** Add transactions to delete this session to the given nested transaction */ + public void deleteLocalSession(LocalSession session, NestedTransaction transaction) { + long sessionId = session.getSessionId(); + SessionZooKeeperClient sessionZooKeeperClient = createSessionZooKeeperClient(sessionId); + transaction.add(sessionZooKeeperClient.deleteTransaction(), FileTransaction.class); + transaction.add(FileTransaction.from(FileOperations.delete(getSessionAppDir(sessionId).getAbsolutePath()))); + } + + public void close() { + deleteAllSessions(); + tenantFileSystemDirs.delete(); + try { + if (directoryCache != null) { + directoryCache.close(); + } + } catch (Exception e) { + log.log(Level.WARNING, "Exception when closing path cache", e); + } finally { + checkForRemovedSessions(new ArrayList<>()); + } + } + + private void deleteAllSessions() { + List<LocalSession> sessions = new ArrayList<>(localSessionCache.getSessions()); + for (LocalSession session : sessions) { + deleteLocalSession(session); + } + } + + // ---------------- Remote sessions ---------------------------------------------------------------- + + public RemoteSession getRemoteSession(long sessionId) { + return remoteSessionCache.getSession(sessionId); + } + + public List<Long> getRemoteSessions() { + return getSessionList(curator.getChildren(sessionsPath)); + } + + public void addRemoteSession(RemoteSession session) { + remoteSessionCache.addSession(session); + metrics.incAddedSessions(); + } + + public int deleteExpiredRemoteSessions(Clock clock, Duration expiryTime) { + int deleted = 0; + for (long sessionId : getRemoteSessions()) { + RemoteSession session = remoteSessionCache.getSession(sessionId); + if (session == null) continue; // Internal sessions not in synch with zk, continue + if (session.getStatus() == Session.Status.ACTIVATE) continue; + if (sessionHasExpired(session.getCreateTime(), expiryTime, clock)) { + log.log(Level.INFO, "Remote session " + sessionId + " for " + tenantName + " has expired, deleting it"); + session.delete(); + deleted++; + } + } + return deleted; + } + + private boolean sessionHasExpired(Instant created, Duration expiryTime, Clock clock) { + return (created.plus(expiryTime).isBefore(clock.instant())); + } + + private List<Long> getSessionListFromDirectoryCache(List<ChildData> children) { + return getSessionList(children.stream() + .map(child -> Path.fromString(child.getPath()).getName()) + .collect(Collectors.toList())); + } + + private List<Long> getSessionList(List<String> children) { + return children.stream().map(Long::parseLong).collect(Collectors.toList()); + } + + private void initializeRemoteSessions() throws NumberFormatException { + getRemoteSessions().forEach(this::sessionAdded); + } + + private synchronized void sessionsChanged() throws NumberFormatException { + List<Long> sessions = getSessionListFromDirectoryCache(directoryCache.getCurrentData()); + checkForRemovedSessions(sessions); + checkForAddedSessions(sessions); + } + + private void checkForRemovedSessions(List<Long> sessions) { + for (RemoteSession session : remoteSessionCache.getSessions()) + if ( ! sessions.contains(session.getSessionId())) + sessionRemoved(session.getSessionId()); + } + + private void checkForAddedSessions(List<Long> sessions) { + for (Long sessionId : sessions) + if (remoteSessionCache.getSession(sessionId) == null) + sessionAdded(sessionId); + } + + /** + * A session for which we don't have a watcher, i.e. hitherto unknown to us. + * + * @param sessionId session id for the new session + */ + public void sessionAdded(long sessionId) { + log.log(Level.FINE, () -> "Adding session to SessionRepository: " + sessionId); + RemoteSession session = createRemoteSession(sessionId); + Path sessionPath = sessionsPath.append(String.valueOf(sessionId)); + Curator.FileCache fileCache = curator.createFileCache(sessionPath.append(ConfigCurator.SESSIONSTATE_ZK_SUBPATH).getAbsolute(), false); + fileCache.addListener(this::nodeChanged); + loadSessionIfActive(session); + addRemoteSession(session); + remoteSessionStateWatchers.put(sessionId, new RemoteSessionStateWatcher(fileCache, reloadHandler, session, metrics, zkWatcherExecutor)); + if (distributeApplicationPackage.value()) { + Optional<LocalSession> localSession = createLocalSessionUsingDistributedApplicationPackage(sessionId); + localSession.ifPresent(this::addSession); + } + } + + private void sessionRemoved(long sessionId) { + RemoteSessionStateWatcher watcher = remoteSessionStateWatchers.remove(sessionId); + if (watcher != null) watcher.close(); + remoteSessionCache.removeSession(sessionId); + metrics.incRemovedSessions(); + } + + private void loadSessionIfActive(RemoteSession session) { + for (ApplicationId applicationId : applicationRepo.activeApplications()) { + if (applicationRepo.requireActiveSessionOf(applicationId) == session.getSessionId()) { + log.log(Level.FINE, () -> "Found active application for session " + session.getSessionId() + " , loading it"); + reloadHandler.reloadConfig(session.ensureApplicationLoaded()); + log.log(Level.INFO, session.logPre() + "Application activated successfully: " + applicationId + " (generation " + session.getSessionId() + ")"); + return; + } + } + } + + private void nodeChanged() { + zkWatcherExecutor.execute(() -> { + Multiset<Session.Status> sessionMetrics = HashMultiset.create(); + for (RemoteSession session : remoteSessionCache.getSessions()) { + sessionMetrics.add(session.getStatus()); + } + metrics.setNewSessions(sessionMetrics.count(Session.Status.NEW)); + metrics.setPreparedSessions(sessionMetrics.count(Session.Status.PREPARE)); + metrics.setActivatedSessions(sessionMetrics.count(Session.Status.ACTIVATE)); + metrics.setDeactivatedSessions(sessionMetrics.count(Session.Status.DEACTIVATE)); + }); + } + + @SuppressWarnings("unused") + private void childEvent(CuratorFramework ignored, PathChildrenCacheEvent event) { + zkWatcherExecutor.execute(() -> { + log.log(Level.FINE, () -> "Got child event: " + event); + switch (event.getType()) { + case CHILD_ADDED: + sessionsChanged(); + synchronizeOnNew(getSessionListFromDirectoryCache(Collections.singletonList(event.getData()))); + break; + case CHILD_REMOVED: + case CONNECTION_RECONNECTED: + sessionsChanged(); + break; + } + }); + } + + private void synchronizeOnNew(List<Long> sessionList) { + for (long sessionId : sessionList) { + RemoteSession session = remoteSessionCache.getSession(sessionId); + if (session == null) continue; // session might have been deleted after getting session list + log.log(Level.FINE, () -> session.logPre() + "Confirming upload for session " + sessionId); + session.confirmUpload(); + } + } + + /** + * Creates a new deployment session from an application package. + * + * @param applicationDirectory a File pointing to an application. + * @param applicationId application id for this new session. + * @param timeoutBudget Timeout for creating session and waiting for other servers. + * @return a new session + */ + public LocalSession createSession(File applicationDirectory, ApplicationId applicationId, + TimeoutBudget timeoutBudget, Optional<Long> activeSessionId) { + return create(applicationDirectory, applicationId, activeSessionId.orElse(nonExistingActiveSession), false, timeoutBudget); + } + + public RemoteSession createRemoteSession(long sessionId) { + SessionZooKeeperClient sessionZKClient = createSessionZooKeeperClient(sessionId); + return new RemoteSession(tenantName, sessionId, componentRegistry, sessionZKClient); + } + + private void ensureSessionPathDoesNotExist(long sessionId) { + Path sessionPath = getSessionPath(sessionId); + if (componentRegistry.getConfigCurator().exists(sessionPath.getAbsolute())) { + throw new IllegalArgumentException("Path " + sessionPath.getAbsolute() + " already exists in ZooKeeper"); + } + } + + private ApplicationPackage createApplication(File userDir, + File configApplicationDir, + ApplicationId applicationId, + long sessionId, + long currentlyActiveSessionId, + boolean internalRedeploy) { + long deployTimestamp = System.currentTimeMillis(); + String user = System.getenv("USER"); + if (user == null) { + user = "unknown"; + } + DeployData deployData = new DeployData(user, userDir.getAbsolutePath(), applicationId, deployTimestamp, internalRedeploy, sessionId, currentlyActiveSessionId); + return FilesApplicationPackage.fromFileWithDeployData(configApplicationDir, deployData); + } + + private LocalSession createSessionFromApplication(ApplicationPackage applicationPackage, + long sessionId, + TimeoutBudget timeoutBudget, + Clock clock) { + log.log(Level.FINE, TenantRepository.logPre(tenantName) + "Creating session " + sessionId + " in ZooKeeper"); + SessionZooKeeperClient sessionZKClient = createSessionZooKeeperClient(sessionId); + sessionZKClient.createNewSession(clock.instant()); + Curator.CompletionWaiter waiter = sessionZKClient.getUploadWaiter(); + LocalSession session = new LocalSession(tenantName, sessionId, applicationPackage, sessionZKClient, applicationRepo); + waiter.awaitCompletion(timeoutBudget.timeLeft()); + return session; + } + + /** + * Creates a new deployment session from an already existing session. + * + * @param existingSession the session to use as base + * @param logger a deploy logger where the deploy log will be written. + * @param internalRedeploy whether this session is for a system internal redeploy — not an application package change + * @param timeoutBudget timeout for creating session and waiting for other servers. + * @return a new session + */ + public LocalSession createSessionFromExisting(Session existingSession, + DeployLogger logger, + boolean internalRedeploy, + TimeoutBudget timeoutBudget) { + File existingApp = getSessionAppDir(existingSession.getSessionId()); + ApplicationId existingApplicationId = existingSession.getApplicationId(); + + long activeSessionId = getActiveSessionId(existingApplicationId); + logger.log(Level.FINE, "Create new session for application id '" + existingApplicationId + "' from existing active session " + activeSessionId); + LocalSession session = create(existingApp, existingApplicationId, activeSessionId, internalRedeploy, timeoutBudget); + // Note: Needs to be kept in sync with calls in SessionPreparer.writeStateToZooKeeper() + session.setApplicationId(existingApplicationId); + if (distributeApplicationPackage.value() && existingSession.getApplicationPackageReference() != null) { + session.setApplicationPackageReference(existingSession.getApplicationPackageReference()); + } + session.setVespaVersion(existingSession.getVespaVersion()); + session.setDockerImageRepository(existingSession.getDockerImageRepository()); + session.setAthenzDomain(existingSession.getAthenzDomain()); + return session; + } + + private LocalSession create(File applicationFile, ApplicationId applicationId, long currentlyActiveSessionId, + boolean internalRedeploy, TimeoutBudget timeoutBudget) { + long sessionId = getNextSessionId(); + try { + ensureSessionPathDoesNotExist(sessionId); + ApplicationPackage app = createApplicationPackage(applicationFile, applicationId, + sessionId, currentlyActiveSessionId, internalRedeploy); + return createSessionFromApplication(app, sessionId, timeoutBudget, clock); + } catch (Exception e) { + throw new RuntimeException("Error creating session " + sessionId, e); + } + } + + /** + * This method is used when creating a session based on a remote session and the distributed application package + * It does not wait for session being created on other servers + */ + private LocalSession createLocalSession(File applicationFile, ApplicationId applicationId, + long sessionId, long currentlyActiveSessionId) { + try { + ApplicationPackage applicationPackage = createApplicationPackage(applicationFile, applicationId, + sessionId, currentlyActiveSessionId, false); + SessionZooKeeperClient sessionZooKeeperClient = createSessionZooKeeperClient(sessionId); + return new LocalSession(tenantName, sessionId, applicationPackage, sessionZooKeeperClient, applicationRepo); + } catch (Exception e) { + throw new RuntimeException("Error creating session " + sessionId, e); + } + } + + private ApplicationPackage createApplicationPackage(File applicationFile, ApplicationId applicationId, + long sessionId, long currentlyActiveSessionId, + boolean internalRedeploy) throws IOException { + File userApplicationDir = getSessionAppDir(sessionId); + IOUtils.copyDirectory(applicationFile, userApplicationDir); + ApplicationPackage applicationPackage = createApplication(applicationFile, + userApplicationDir, + applicationId, + sessionId, + currentlyActiveSessionId, + internalRedeploy); + applicationPackage.writeMetaData(); + return applicationPackage; + } + + /** + * Returns a new session instance for the given session id. + */ + LocalSession createSessionFromId(long sessionId) { + File sessionDir = getAndValidateExistingSessionAppDir(sessionId); + ApplicationPackage applicationPackage = FilesApplicationPackage.fromFile(sessionDir); + SessionZooKeeperClient sessionZKClient = createSessionZooKeeperClient(sessionId); + return new LocalSession(tenantName, sessionId, applicationPackage, sessionZKClient, applicationRepo); + } + + /** + * Returns a new session instance for the given session id. + */ + Optional<LocalSession> createLocalSessionUsingDistributedApplicationPackage(long sessionId) { + if (applicationRepo.hasLocalSession(sessionId)) { + log.log(Level.FINE, "Local session for session id " + sessionId + " already exists"); + return Optional.of(createSessionFromId(sessionId)); + } + + log.log(Level.INFO, "Creating local session for session id " + sessionId); + SessionZooKeeperClient sessionZKClient = createSessionZooKeeperClient(sessionId); + FileReference fileReference = sessionZKClient.readApplicationPackageReference(); + log.log(Level.FINE, "File reference for session id " + sessionId + ": " + fileReference); + if (fileReference != null) { + File rootDir = new File(Defaults.getDefaults().underVespaHome(componentRegistry.getConfigserverConfig().fileReferencesDir())); + File sessionDir; + FileDirectory fileDirectory = new FileDirectory(rootDir); + try { + sessionDir = fileDirectory.getFile(fileReference); + } catch (IllegalArgumentException e) { + // We cannot be guaranteed that the file reference exists (it could be that it has not + // been downloaded yet), and e.g when bootstrapping we cannot throw an exception in that case + log.log(Level.INFO, "File reference for session id " + sessionId + ": " + fileReference + " not found in " + fileDirectory); + return Optional.empty(); + } + ApplicationId applicationId = sessionZKClient.readApplicationId(); + return Optional.of(createLocalSession(sessionDir, + applicationId, + sessionId, + applicationRepo.activeSessionOf(applicationId).orElse(nonExistingActiveSession))); + } + return Optional.empty(); + } + + // Return Optional instead of faking it with nonExistingActiveSession + private long getActiveSessionId(ApplicationId applicationId) { + List<ApplicationId> applicationIds = applicationRepo.activeApplications(); + if (applicationIds.contains(applicationId)) { + return applicationRepo.requireActiveSessionOf(applicationId); + } + return nonExistingActiveSession; + } + + private long getNextSessionId() { + return new SessionCounter(componentRegistry.getConfigCurator(), tenantName).nextSessionId(); + } + + private Path getSessionPath(long sessionId) { + return sessionsPath.append(String.valueOf(sessionId)); + } + + + private SessionZooKeeperClient createSessionZooKeeperClient(long sessionId) { + String serverId = componentRegistry.getConfigserverConfig().serverId(); + Optional<NodeFlavors> nodeFlavors = componentRegistry.getZone().nodeFlavors(); + Path sessionPath = getSessionPath(sessionId); + return new SessionZooKeeperClient(curator, componentRegistry.getConfigCurator(), sessionPath, serverId, nodeFlavors); + } + + private File getAndValidateExistingSessionAppDir(long sessionId) { + File appDir = getSessionAppDir(sessionId); + if (!appDir.exists() || !appDir.isDirectory()) { + throw new IllegalArgumentException("Unable to find correct application directory for session " + sessionId); + } + return appDir; + } + + private File getSessionAppDir(long sessionId) { + return new TenantFileSystemDirs(componentRegistry.getConfigServerDB(), tenantName).getUserApplicationDir(sessionId); + } + + @Override + public String toString() { + return getLocalSessions().toString(); + } + + private static class FileTransaction extends AbstractTransaction { + + public static FileTransaction from(FileOperation operation) { + FileTransaction transaction = new FileTransaction(); + transaction.add(operation); + return transaction; + } + + @Override + public void prepare() { } + + @Override + public void commit() { + for (Operation operation : operations()) + ((FileOperation)operation).commit(); + } + + } + + /** Factory for file operations */ + private static class FileOperations { + + /** Creates an operation which recursively deletes the given path */ + public static DeleteOperation delete(String pathToDelete) { + return new DeleteOperation(pathToDelete); + } + + } + + private interface FileOperation extends Transaction.Operation { + + void commit(); + + } + + /** + * Recursively deletes this path and everything below. + * Succeeds with no action if the path does not exist. + */ + private static class DeleteOperation implements FileOperation { + + private final String pathToDelete; + + DeleteOperation(String pathToDelete) { + this.pathToDelete = pathToDelete; + } + + @Override + public void commit() { + // TODO: Check delete access in prepare() + IOUtils.recursiveDeleteDir(new File(pathToDelete)); + } + + } + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java index 33001d2996c..807629a2148 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java @@ -14,7 +14,6 @@ import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.NodeFlavors; import com.yahoo.path.Path; import com.yahoo.text.Utf8; -import com.yahoo.transaction.NestedTransaction; import com.yahoo.transaction.Transaction; import com.yahoo.vespa.config.server.UserConfigDefinitionRepo; import com.yahoo.vespa.config.server.deploy.ZooKeeperClient; @@ -129,18 +128,6 @@ public class SessionZooKeeperClient { return curator.getCompletionWaiter(path, getNumberOfMembers(), serverId); } - public void delete(NestedTransaction transaction ) { - try { - log.log(Level.FINE, "Deleting " + sessionPath.getAbsolute()); - CuratorTransaction curatorTransaction = new CuratorTransaction(curator); - CuratorOperations.deleteAll(sessionPath.getAbsolute(), curator).forEach(curatorTransaction::add); - transaction.add(curatorTransaction); - transaction.commit(); - } catch (RuntimeException e) { - log.log(Level.INFO, "Error deleting session (" + sessionPath.getAbsolute() + ") from zookeeper", e); - } - } - /** Returns a transaction deleting this session on commit */ public CuratorTransaction deleteTransaction() { return CuratorTransaction.from(CuratorOperations.deleteAll(sessionPath.getAbsolute(), curator), curator); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenant.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenant.java index f0aab8b2312..f7c8ae9d5c3 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenant.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenant.java @@ -6,9 +6,7 @@ import com.yahoo.path.Path; import com.yahoo.vespa.config.server.ReloadHandler; import com.yahoo.vespa.config.server.RequestHandler; import com.yahoo.vespa.config.server.application.TenantApplications; -import com.yahoo.vespa.config.server.session.LocalSessionRepo; -import com.yahoo.vespa.config.server.session.RemoteSessionRepo; -import com.yahoo.vespa.config.server.session.SessionFactory; +import com.yahoo.vespa.config.server.session.SessionRepository; import com.yahoo.vespa.curator.Curator; import org.apache.zookeeper.data.Stat; @@ -28,19 +26,15 @@ public class Tenant implements TenantHandlerProvider { static final String APPLICATIONS = "applications"; private final TenantName name; - private final RemoteSessionRepo remoteSessionRepo; private final Path path; - private final SessionFactory sessionFactory; - private final LocalSessionRepo localSessionRepo; + private final SessionRepository sessionRepository; private final TenantApplications applicationRepo; private final RequestHandler requestHandler; private final ReloadHandler reloadHandler; private final Curator curator; Tenant(TenantName name, - SessionFactory sessionFactory, - LocalSessionRepo localSessionRepo, - RemoteSessionRepo remoteSessionRepo, + SessionRepository sessionRepository, RequestHandler requestHandler, ReloadHandler reloadHandler, TenantApplications applicationRepo, @@ -49,9 +43,7 @@ public class Tenant implements TenantHandlerProvider { this.path = TenantRepository.getTenantPath(name); this.requestHandler = requestHandler; this.reloadHandler = reloadHandler; - this.remoteSessionRepo = remoteSessionRepo; - this.sessionFactory = sessionFactory; - this.localSessionRepo = localSessionRepo; + this.sessionRepository = sessionRepository; this.applicationRepo = applicationRepo; this.curator = curator; } @@ -74,13 +66,8 @@ public class Tenant implements TenantHandlerProvider { return requestHandler; } - /** - * The RemoteSessionRepo for this - * - * @return repo - */ - public RemoteSessionRepo getRemoteSessionRepo() { - return remoteSessionRepo; + public SessionRepository getSessionRepo() { + return sessionRepository; } public TenantName getName() { @@ -91,13 +78,7 @@ public class Tenant implements TenantHandlerProvider { return path; } - public SessionFactory getSessionFactory() { - return sessionFactory; - } - - public LocalSessionRepo getLocalSessionRepo() { - return localSessionRepo; - } + public SessionRepository getSessionRepository() { return sessionRepository; } @Override public String toString() { @@ -140,9 +121,8 @@ public class Tenant implements TenantHandlerProvider { * Called by watchers as a reaction to {@link #delete()}. */ void close() { - remoteSessionRepo.close(); // Closes watchers and clears memory. applicationRepo.close(); // Closes watchers. - localSessionRepo.close(); // Closes watchers, clears memory, and deletes local files and ZK session state. + sessionRepository.close(); // Closes watchers, clears memory, and deletes local files and ZK session state. } /** Deletes the tenant tree from ZooKeeper (application and session status for the tenant) and triggers {@link #close()}. */ diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java index d34f89a179b..304fbb6786a 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java @@ -14,9 +14,7 @@ import com.yahoo.vespa.config.server.RequestHandler; import com.yahoo.vespa.config.server.application.TenantApplications; import com.yahoo.vespa.config.server.deploy.TenantFileSystemDirs; import com.yahoo.vespa.config.server.monitoring.MetricUpdater; -import com.yahoo.vespa.config.server.session.LocalSessionRepo; -import com.yahoo.vespa.config.server.session.RemoteSessionRepo; -import com.yahoo.vespa.config.server.session.SessionFactory; +import com.yahoo.vespa.config.server.session.SessionRepository; import com.yahoo.vespa.curator.Curator; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; @@ -49,7 +47,7 @@ import java.util.stream.Collectors; * This component will monitor the set of tenants in the config server by watching in ZooKeeper. * It will set up Tenant objects accordingly, which will manage the config sessions per tenant. * This class will read the preexisting set of tenants from ZooKeeper at startup. (For now it will also - * create a default tenant since that will be used for API that do no know about tenants or have not yet + * create a default tenant since that will be used for APIs that do no know about tenants or have not yet * implemented support for it). * * This instance is called from two different threads, the http handler threads and the zookeeper watcher threads. @@ -223,16 +221,12 @@ public class TenantRepository { requestHandler = applicationRepo; if (reloadHandler == null) reloadHandler = applicationRepo; - SessionFactory sessionFactory = new SessionFactory(componentRegistry, applicationRepo, applicationRepo, tenantName); - LocalSessionRepo localSessionRepo = new LocalSessionRepo(tenantName, componentRegistry, sessionFactory); - RemoteSessionRepo remoteSessionRepo = new RemoteSessionRepo(componentRegistry, - sessionFactory, - reloadHandler, - tenantName, - applicationRepo, - componentRegistry.getFlagSource()); + SessionRepository sessionRepository = new SessionRepository(tenantName, componentRegistry, + applicationRepo, reloadHandler, + componentRegistry.getFlagSource(), + componentRegistry.getSessionPreparer()); log.log(Level.INFO, "Creating tenant '" + tenantName + "'"); - Tenant tenant = new Tenant(tenantName, sessionFactory, localSessionRepo, remoteSessionRepo, requestHandler, + Tenant tenant = new Tenant(tenantName, sessionRepository, requestHandler, reloadHandler, applicationRepo, componentRegistry.getCurator()); notifyNewTenant(tenant); tenants.putIfAbsent(tenantName, tenant); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ConfigCurator.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ConfigCurator.java index 35e8b0917cf..20ac4b65c64 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ConfigCurator.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ConfigCurator.java @@ -3,16 +3,11 @@ package com.yahoo.vespa.config.server.zookeeper; import com.google.inject.Inject; import com.yahoo.cloud.config.ZookeeperServerConfig; -import com.yahoo.io.IOUtils; -import java.util.logging.Level; import com.yahoo.text.Utf8; import com.yahoo.vespa.curator.Curator; -import java.io.File; -import java.io.FilenameFilter; -import java.io.IOException; -import java.util.ArrayList; import java.util.List; +import java.util.logging.Level; /** * A (stateful) curator wrapper for the config server. This simplifies Curator method calls used by the config server @@ -49,8 +44,6 @@ public class ConfigCurator { /** Path for session state */ public static final String SESSIONSTATE_ZK_SUBPATH = "/sessionState"; - private static final FilenameFilter acceptsAllFileNameFilter = (dir, name) -> true; - private final Curator curator; public static final java.util.logging.Logger log = java.util.logging.Logger.getLogger(ConfigCurator.class.getName()); @@ -229,65 +222,6 @@ public class ConfigCurator { putData(path, name, data); } - /** - * Takes for instance the dir /app and puts the contents into the given ZK path. Ignores files starting with dot, - * and dirs called CVS. - * - * @param dir directory which holds the summary class part files - * @param path zookeeper path - * @param filenameFilter A FilenameFilter which decides which files in dir are fed to zookeeper - * @param recurse recurse subdirectories - */ - void feedZooKeeper(File dir, String path, FilenameFilter filenameFilter, boolean recurse) { - try { - if (filenameFilter == null) { - filenameFilter = acceptsAllFileNameFilter; - } - if (!dir.isDirectory()) { - log.fine(dir.getCanonicalPath() + " is not a directory. Not feeding the files into ZooKeeper."); - return; - } - for (File file : listFiles(dir, filenameFilter)) { - if (file.getName().startsWith(".")) continue; //.svn , .git ... - if ("CVS".equals(file.getName())) continue; - if (file.isFile()) { - byte[] contents = IOUtils.readFileBytes(file); - putData(path, file.getName(), contents); - } else if (recurse && file.isDirectory()) { - createNode(path, file.getName()); - feedZooKeeper(file, path + '/' + file.getName(), filenameFilter, recurse); - } - } - } - catch (IOException e) { - throw new RuntimeException("Exception feeding ZooKeeper at path " + path, e); - } - } - - /** - * Same as normal listFiles, but use the filter only for normal files - * - * @param dir directory to list files in - * @param filter A FilenameFilter which decides which files in dir are listed - * @return an array of Files - */ - protected File[] listFiles(File dir, FilenameFilter filter) { - File[] rawList = dir.listFiles(); - List<File> ret = new ArrayList<>(); - if (rawList != null) { - for (File f : rawList) { - if (f.isDirectory()) { - ret.add(f); - } else { - if (filter.accept(dir, f.getName())) { - ret.add(f); - } - } - } - } - return ret.toArray(new File[0]); - } - /** Deletes the node at the given path, and any children it may have. If the node does not exist this does nothing */ public void deleteRecurse(String path) { try { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java index a3d072be38b..f0aa38a228e 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.config.server; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.Version; import com.yahoo.config.ConfigInstance; +import com.yahoo.config.FileReference; import com.yahoo.config.SimpletypesConfig; import com.yahoo.config.application.api.ApplicationMetaData; import com.yahoo.config.model.NullConfigModelRegistry; @@ -33,11 +34,12 @@ import com.yahoo.vespa.config.protocol.VespaVersion; import com.yahoo.vespa.config.server.application.OrchestratorMock; import com.yahoo.vespa.config.server.application.TenantApplications; import com.yahoo.vespa.config.server.deploy.DeployTester; +import com.yahoo.vespa.config.server.deploy.TenantFileSystemDirs; import com.yahoo.vespa.config.server.http.InternalServerException; import com.yahoo.vespa.config.server.http.SessionHandlerTest; import com.yahoo.vespa.config.server.http.v2.PrepareResult; import com.yahoo.vespa.config.server.session.LocalSession; -import com.yahoo.vespa.config.server.session.LocalSessionRepo; +import com.yahoo.vespa.config.server.session.SessionRepository; import com.yahoo.vespa.config.server.session.PrepareParams; import com.yahoo.vespa.config.server.session.RemoteSession; import com.yahoo.vespa.config.server.session.Session; @@ -45,8 +47,12 @@ import com.yahoo.vespa.config.server.session.SilentDeployLogger; import com.yahoo.vespa.config.server.tenant.ApplicationRolesStore; import com.yahoo.vespa.config.server.tenant.Tenant; import com.yahoo.vespa.config.server.tenant.TenantRepository; +import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.Flags; +import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.model.VespaModelFactory; import org.hamcrest.core.Is; import org.jetbrains.annotations.NotNull; @@ -103,6 +109,7 @@ public class ApplicationRepositoryTest { private SessionHandlerTest.MockProvisioner provisioner; private OrchestratorMock orchestrator; private TimeoutBudget timeoutBudget; + private ConfigCurator configCurator; @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); @@ -110,19 +117,23 @@ public class ApplicationRepositoryTest { @Rule public ExpectedException exceptionRule = ExpectedException.none(); - @Rule - public TemporaryFolder tempFolder = new TemporaryFolder(); - @Before public void setup() throws IOException { + setup(new InMemoryFlagSource()); + } + + public void setup(FlagSource flagSource) throws IOException { Curator curator = new MockCurator(); + configCurator = ConfigCurator.create(curator); TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder() .curator(curator) .configServerConfig(new ConfigserverConfig.Builder() .payloadCompressionType(ConfigserverConfig.PayloadCompressionType.Enum.UNCOMPRESSED) - .configServerDBDir(tempFolder.newFolder("configserverdb").getAbsolutePath()) - .configDefinitionsDir(tempFolder.newFolder("configdefinitions").getAbsolutePath()) + .configServerDBDir(temporaryFolder.newFolder().getAbsolutePath()) + .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) + .fileReferencesDir(temporaryFolder.newFolder().getAbsolutePath()) .build()) + .flagSource(flagSource) .build(); tenantRepository = new TenantRepository(componentRegistry, false); tenantRepository.addTenant(TenantRepository.HOSTED_VESPA_TENANT); @@ -146,7 +157,7 @@ public class ApplicationRepositoryTest { TenantName tenantName = applicationId().tenant(); Tenant tenant = tenantRepository.getTenant(tenantName); - LocalSession session = tenant.getLocalSessionRepo().getSession(tenant.getApplicationRepo() + LocalSession session = tenant.getSessionRepository().getLocalSession(tenant.getApplicationRepo() .requireActiveSessionOf(applicationId())); session.getAllocatedHosts(); } @@ -178,7 +189,7 @@ public class ApplicationRepositoryTest { TenantName tenantName = applicationId().tenant(); Tenant tenant = tenantRepository.getTenant(tenantName); - LocalSession session = tenant.getLocalSessionRepo().getSession( + LocalSession session = tenant.getSessionRepository().getLocalSession( tenant.getApplicationRepo().requireActiveSessionOf(applicationId())); assertEquals(firstSessionId, session.getMetaData().getPreviousActiveGeneration()); } @@ -291,24 +302,32 @@ public class ApplicationRepositoryTest { @Test public void delete() { + TenantName tenantName = applicationId().tenant(); + Tenant tenant = tenantRepository.getTenant(tenantName); { PrepareResult result = deployApp(testApp); long sessionId = result.sessionId(); - Tenant tenant = tenantRepository.getTenant(applicationId().tenant()); - LocalSession applicationData = tenant.getLocalSessionRepo().getSession(sessionId); + LocalSession applicationData = tenant.getSessionRepository().getLocalSession(sessionId); assertNotNull(applicationData); assertNotNull(applicationData.getApplicationId()); - assertNotNull(tenant.getRemoteSessionRepo().getSession(sessionId)); + assertNotNull(tenant.getSessionRepo().getLocalSession(sessionId)); assertNotNull(applicationRepository.getActiveSession(applicationId())); + String sessionNode = TenantRepository.getSessionsPath(tenantName).append(String.valueOf(sessionId)).getAbsolute(); + assertTrue(configCurator.exists(sessionNode)); + TenantFileSystemDirs tenantFileSystemDirs = tenant.getApplicationRepo().getTenantFileSystemDirs(); + File sessionFile = new File(tenantFileSystemDirs.sessionsPath(), String.valueOf(sessionId)); + assertTrue(sessionFile.exists()); // Delete app and verify that it has been deleted from repos and provisioner assertTrue(applicationRepository.delete(applicationId())); assertNull(applicationRepository.getActiveSession(applicationId())); - assertNull(tenant.getLocalSessionRepo().getSession(sessionId)); - assertNull(tenant.getRemoteSessionRepo().getSession(sessionId)); + assertNull(tenant.getSessionRepository().getLocalSession(sessionId)); + assertNull(tenant.getSessionRepo().getLocalSession(sessionId)); assertTrue(provisioner.removed); assertEquals(tenant.getName(), provisioner.lastApplicationId.tenant()); assertEquals(applicationId(), provisioner.lastApplicationId); + assertFalse(configCurator.exists(sessionNode)); + assertFalse(sessionFile.exists()); assertFalse(applicationRepository.delete(applicationId())); } @@ -345,8 +364,7 @@ public class ApplicationRepositoryTest { // A new delete should cleanup and be successful RemoteSession activeSession = applicationRepository.getActiveSession(applicationId()); assertNull(activeSession); - Tenant tenant = tenantRepository.getTenant(applicationId().tenant()); - assertNull(tenant.getRemoteSessionRepo().getSession(prepareResult.sessionId())); + assertNull(tenant.getSessionRepo().getLocalSession(prepareResult.sessionId())); assertTrue(applicationRepository.delete(applicationId())); } @@ -379,14 +397,14 @@ public class ApplicationRepositoryTest { assertNotEquals(activeSessionId, deployment3session); // No change to active session id assertEquals(activeSessionId, tester.tenant().getApplicationRepo().requireActiveSessionOf(tester.applicationId())); - LocalSessionRepo localSessionRepo = tester.tenant().getLocalSessionRepo(); - assertEquals(3, localSessionRepo.getSessions().size()); + SessionRepository sessionRepository = tester.tenant().getSessionRepository(); + assertEquals(3, sessionRepository.getLocalSessions().size()); clock.advance(Duration.ofHours(1)); // longer than session lifetime // All sessions except 3 should be removed after the call to deleteExpiredLocalSessions tester.applicationRepository().deleteExpiredLocalSessions(); - Collection<LocalSession> sessions = localSessionRepo.getSessions(); + Collection<LocalSession> sessions = sessionRepository.getLocalSessions(); assertEquals(1, sessions.size()); ArrayList<LocalSession> localSessions = new ArrayList<>(sessions); LocalSession localSession = localSessions.get(0); @@ -400,9 +418,9 @@ public class ApplicationRepositoryTest { assertTrue(deployment4.isPresent()); deployment4.get().prepare(); // session 5 (not activated) - assertEquals(2, localSessionRepo.getSessions().size()); - localSessionRepo.deleteSession(localSession); - assertEquals(1, localSessionRepo.getSessions().size()); + assertEquals(2, sessionRepository.getLocalSessions().size()); + sessionRepository.deleteLocalSession(localSession); + assertEquals(1, sessionRepository.getLocalSessions().size()); // Check that trying to expire when there are no active sessions works tester.applicationRepository().deleteExpiredLocalSessions(); @@ -457,7 +475,7 @@ public class ApplicationRepositoryTest { TenantName tenantName = applicationId().tenant(); Tenant tenant = tenantRepository.getTenant(tenantName); - LocalSession session = tenant.getLocalSessionRepo().getSession(tenant.getApplicationRepo().requireActiveSessionOf(applicationId())); + LocalSession session = tenant.getSessionRepository().getLocalSession(tenant.getApplicationRepo().requireActiveSessionOf(applicationId())); List<NetworkPorts.Allocation> list = new ArrayList<>(); list.add(new NetworkPorts.Allocation(8080, "container", "container/container.0", "http")); @@ -658,6 +676,14 @@ public class ApplicationRepositoryTest { resolve(SimpletypesConfig.class, requestHandler, applicationId(), vespaVersion); } + @Test + public void testDistributionOfApplicationPackage() throws IOException { + FlagSource flagSource = new InMemoryFlagSource() + .withBooleanFlag(Flags.CONFIGSERVER_DISTRIBUTE_APPLICATION_PACKAGE.id(), true); + setup(flagSource); + applicationRepository.deploy(app1, prepareParams()); + } + private ApplicationRepository createApplicationRepository() { return new ApplicationRepository(tenantRepository, provisioner, diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/TestComponentRegistry.java b/configserver/src/test/java/com/yahoo/vespa/config/server/TestComponentRegistry.java index ec5648757f1..bc16f44b405 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/TestComponentRegistry.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/TestComponentRegistry.java @@ -60,6 +60,7 @@ public class TestComponentRegistry implements GlobalComponentRegistry { private final StripedExecutor<TenantName> zkWatcherExecutor; private final ExecutorService zkCacheExecutor; private final SecretStore secretStore; + private final FlagSource flagSource; private TestComponentRegistry(Curator curator, ConfigCurator configCurator, Metrics metrics, ModelFactoryRegistry modelFactoryRegistry, @@ -74,7 +75,8 @@ public class TestComponentRegistry implements GlobalComponentRegistry { TenantListener tenantListener, Zone zone, Clock clock, - SecretStore secretStore) { + SecretStore secretStore, + FlagSource flagSource) { this.curator = curator; this.configCurator = configCurator; this.metrics = metrics; @@ -94,6 +96,7 @@ public class TestComponentRegistry implements GlobalComponentRegistry { this.zkWatcherExecutor = new StripedExecutor<>(new InThreadExecutorService()); this.zkCacheExecutor = new InThreadExecutorService(); this.secretStore = secretStore; + this.flagSource = flagSource; } public static class Builder { @@ -115,6 +118,7 @@ public class TestComponentRegistry implements GlobalComponentRegistry { private Optional<Provisioner> hostProvisioner = Optional.empty(); private Zone zone = Zone.defaultZone(); private Clock clock = Clock.systemUTC(); + private FlagSource flagSource = new InMemoryFlagSource(); public Builder configServerConfig(ConfigserverConfig configserverConfig) { this.configserverConfig = configserverConfig; @@ -161,6 +165,11 @@ public class TestComponentRegistry implements GlobalComponentRegistry { return this; } + public Builder flagSource(FlagSource flagSource) { + this.flagSource = flagSource; + return this; + } + public TestComponentRegistry build() { final PermanentApplicationPackage permApp = this.permanentApplicationPackage .orElse(new PermanentApplicationPackage(configserverConfig)); @@ -172,11 +181,11 @@ public class TestComponentRegistry implements GlobalComponentRegistry { SessionPreparer sessionPreparer = new SessionPreparer(modelFactoryRegistry, fileDistributionProvider, hostProvisionerProvider, permApp, configserverConfig, defRepo, curator, - zone, new InMemoryFlagSource(), secretStore); + zone, flagSource, secretStore); return new TestComponentRegistry(curator, ConfigCurator.create(curator), metrics, modelFactoryRegistry, permApp, fileDistributionProvider, hostRegistries, configserverConfig, sessionPreparer, hostProvisioner, defRepo, reloadListener, tenantListener, - zone, clock, secretStore); + zone, clock, secretStore, flagSource); } } @@ -221,7 +230,7 @@ public class TestComponentRegistry implements GlobalComponentRegistry { } @Override - public FlagSource getFlagSource() { return new InMemoryFlagSource(); } + public FlagSource getFlagSource() { return flagSource; } @Override public ExecutorService getZkCacheExecutor() { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/a-music-indexer-correct.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/a-music-indexer-correct.cfg deleted file mode 100644 index 8b43ff9c793..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/a-music-indexer-correct.cfg +++ /dev/null @@ -1,78 +0,0 @@ -accesslog "/home/vespa/logs/vespa/foo.log" -partialsd "sd" -partialsd2 "global2" -asyncfetchocc 10 -a 0 -b 1 -c 2 -d 3 -e 4 -onlyindef 45 -listenport 13700 -rangecheck2 10 -rangecheck3 10 -kanon -78.56 -rangecheck1 10.0 -testref search/cluster.music/c0/r0/indexer.4 -testref2 some/babbel -mode BATCH -functionmodules[0] -storage[2] -storage[0].feeder[1] -storage[0].feeder[0] "test" -storage[1].id search/cluster.music/c0/r0/indexer.4 -storage[1].id2 pjatt -storage[1].feeder[2] -storage[1].feeder[0] "me" -storage[1].feeder[1] "now" -search[3] -search[0].feeder[1] -search[0].feeder[0] "foofeeder" -search[1].feeder[4] -search[1].feeder[0] "barfeeder1_1" -search[1].feeder[1] "barfeeder2" -search[1].feeder[2] "" -search[1].feeder[3] "barfeeder2_1" -search[2].feeder[2] -search[2].feeder[0] "" -search[2].feeder[1] "bazfeeder" -f[1] -f[0].a "A" -f[0].b "B" -f[0].c "C" -f[0].h "H" -f[0].f "F" -config[1] -config[0].role "rtx" -config[0].usewrapper false -config[0].id search/cluster.music/rtx/0 -routingtable[1] -routingtable[0].hop[3] -routingtable[0].hop[0].name "docproc/cluster.music.indexing/chain.music.indexing" -routingtable[0].hop[0].selector "docproc/cluster.music.indexing/*/chain.music.indexing" -routingtable[0].hop[0].recipient[0] -routingtable[0].hop[1].name "search/cluster.music" -routingtable[0].hop[1].selector "search/cluster.music/[SearchColumn]/[SearchRow]/feed-destination" -routingtable[0].hop[1].recipient[1] -routingtable[0].hop[1].recipient[0] "search/cluster.music/c0/r0/feed-destination" -routingtable[0].hop[2].selector "[DocumentRouteSelector]" -routingtable[0].hop[2].name "indexing" -routingtable[0].hop[2].recipient[1] -routingtable[0].hop[2].recipient[0] "search/cluster.music" -speciallog[1] -speciallog[0].filehandler.name "QueryAccessLog" -speciallog[0].filehandler.pattern "logs/vespa/qrs/QueryAccessLog.%Y%m%d%H%M%S" -speciallog[0].filehandler.rotation "0 1 ..." -speciallog[0].cachehandler.name "QueryAccessLog" -speciallog[0].name "QueryAccessLog" -speciallog[0].type "file" -speciallog[0].cachehandler.size 1000 -rulebase[4] -rulebase[0].name "cjk" -rulebase[0].rules "# Use unicode equivalents in java source:\n#\n# 佳:\u4f73\n# 能:\u80fd\n# 索:\u7d22\n# 尼:\u5c3c\n# 惠:\u60e0\n# 普:\u666e\n\n@default\n\na索 -> 索a;\n\n[brand] -> brand:[brand];\n\n[brand] :- 索尼,惠普,佳能;\n" -rulebase[1].name "common" -rulebase[1].rules "## Some test rules\n\n# Spelling correction\nbahc -> bach;\n\n# Stopwords\nsomelongstopword -> ;\n[stopword] -> ;\n[stopword] :- someotherlongstopword, yetanotherstopword;\n\n# \n[song] by [artist] -> song:[song] artist:[artist];\n\n[song] :- together, imagine, tinseltown;\n[artist] :- youngbloods, beatles, zappa;\n\n# Negative\nvarious +> -kingz;\n\n\n" -rulebase[2].name "egyik" -rulebase[2].rules "@include(common.sr)\n@automata(/home/vespa/etc/vespa/fsa/stopwords.fsa)\n[stopwords] -> ;\n\n" -rulebase[3].name "masik" -rulebase[3].rules "@include(common.sr)\n[stopwords] :- etaoin, shrdlu;\n[stopwords] -> ;\n\n" diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/a-sports-indexer-correct.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/a-sports-indexer-correct.cfg deleted file mode 100644 index 927ff8a26c9..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/a-sports-indexer-correct.cfg +++ /dev/null @@ -1,48 +0,0 @@ -accesslog "/home/vespa/logs/vespa/foo.log" -partialsd "global" -partialsd2 "global2" -asyncfetchocc 10 -a 0 -b 1 -c 67 -d 89 -e 4 -onlyindef 45 -listenport 13700 -rangecheck2 10 -rangecheck3 10 -rangecheck1 10.0 -mode BATCH -functionmodules[0] -storage[0] -search[3] -search[0].feeder[1] -search[0].feeder[0] "foofeeder" -search[1].feeder[4] -search[1].feeder[0] "barfeeder1_1" -search[1].feeder[1] "sportsfeeder1" -search[1].feeder[2] "" -search[1].feeder[3] "barfeeder2_1" -search[2].feeder[2] -search[2].feeder[0] "" -search[2].feeder[1] "bazfeeder" -f[0] -config[0] -routingtable[0] -speciallog[1] -speciallog[0].filehandler.name "QueryAccessLog" -speciallog[0].filehandler.pattern "logs/vespa/qrs/QueryAccessLog.%Y%m%d%H%M%S" -speciallog[0].filehandler.rotation "0 1 ..." -speciallog[0].cachehandler.name "QueryAccessLog" -speciallog[0].name "QueryAccessLog" -speciallog[0].type "file" -speciallog[0].cachehandler.size 1000 -rulebase[4] -rulebase[0].name "cjk" -rulebase[0].rules "# Use unicode equivalents in java source:\n#\n# 佳:\u4f73\n# 能:\u80fd\n# 索:\u7d22\n# 尼:\u5c3c\n# 惠:\u60e0\n# 普:\u666e\n\n@default\n\na索 -> 索a;\n\n[brand] -> brand:[brand];\n\n[brand] :- 索尼,惠普,佳能;\n" -rulebase[1].name "common" -rulebase[1].rules "## Some test rules\n\n# Spelling correction\nbahc -> bach;\n\n# Stopwords\nsomelongstopword -> ;\n[stopword] -> ;\n[stopword] :- someotherlongstopword, yetanotherstopword;\n\n# \n[song] by [artist] -> song:[song] artist:[artist];\n\n[song] :- together, imagine, tinseltown;\n[artist] :- youngbloods, beatles, zappa;\n\n# Negative\nvarious +> -kingz;\n\n\n" -rulebase[2].name "egyik" -rulebase[2].rules "@include(common.sr)\n@automata(/home/vespa/etc/vespa/fsa/stopwords.fsa)\n[stopwords] -> ;\n\n" -rulebase[3].name "masik" -rulebase[3].rules "@include(common.sr)\n[stopwords] :- etaoin, shrdlu;\n[stopwords] -> ;\n\n" diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/app_stripped/components/testbundle.jar b/configserver/src/test/java/com/yahoo/vespa/config/server/app_stripped/components/testbundle.jar Binary files differdeleted file mode 100644 index 69f6e335092..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/app_stripped/components/testbundle.jar +++ /dev/null diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/app_stripped/services.xml b/configserver/src/test/java/com/yahoo/vespa/config/server/app_stripped/services.xml deleted file mode 100644 index b7924e73e7a..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/app_stripped/services.xml +++ /dev/null @@ -1,18 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<services version="1.0"> - - <admin version="2.0"> - <adminserver hostalias="node1"/> - </admin> - - <content version="1.0"> - <redundancy>1</redundancy> - <documents> - <document type="music" mode="index"/> - </documents> - <nodes>> - <node hostalias="node1" distribution-key="0"/> - </nodes> - </content> -</services> diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/HttpProxyTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/HttpProxyTest.java index e64921e3ea0..cf7740f133f 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/HttpProxyTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/HttpProxyTest.java @@ -1,25 +1,32 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.application; +import com.yahoo.config.model.api.HostInfo; import com.yahoo.config.model.api.Model; +import com.yahoo.config.model.api.ServiceInfo; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.vespa.config.server.http.HttpFetcher; -import com.yahoo.vespa.config.server.http.StaticResponse; import com.yahoo.vespa.config.server.http.RequestTimeoutException; +import com.yahoo.vespa.config.server.http.StaticResponse; import org.junit.Before; import org.junit.Test; import org.mockito.ArgumentCaptor; import java.net.URL; +import java.util.Arrays; +import java.util.Collections; import static com.yahoo.config.model.api.container.ContainerServiceType.CLUSTERCONTROLLER_CONTAINER; +import static com.yahoo.vespa.config.server.application.MockModel.createServiceInfo; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertSame; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; public class HttpProxyTest { + private final HttpFetcher fetcher = mock(HttpFetcher.class); private final HttpProxy proxy = new HttpProxy(fetcher); @@ -29,7 +36,7 @@ public class HttpProxyTest { @Before public void setup() { - Model modelMock = MockModel.createClusterController(hostname, port); + Model modelMock = createClusterController(); when(applicationMock.getModel()).thenReturn(modelMock); } @@ -52,7 +59,7 @@ public class HttpProxyTest { // The HttpResponse returned by the fetcher IS the same object as the one returned by the proxy, // when everything goes well. - assertTrue(actualResponse == response); + assertSame(actualResponse, response); } @Test(expected = RequestTimeoutException.class) @@ -62,4 +69,20 @@ public class HttpProxyTest { proxy.get(applicationMock, hostname, CLUSTERCONTROLLER_CONTAINER.serviceName, "clustercontroller-status/v1/clusterName"); } + + private static MockModel createClusterController() { + ServiceInfo container = createServiceInfo( + hostname, + "foo", // name + CLUSTERCONTROLLER_CONTAINER.serviceName, + ClusterSpec.Type.container, + port, + "state http external query"); + ServiceInfo serviceNoStatePort = createServiceInfo(hostname, "storagenode", "storagenode", + ClusterSpec.Type.content, 1234, "rpc"); + HostInfo hostInfo = new HostInfo(hostname, Arrays.asList(container, serviceNoStatePort)); + + return new MockModel(Collections.singleton(hostInfo)); + } + } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/MockModel.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/MockModel.java index 0da96f9f01d..c9f25451ea4 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/MockModel.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/MockModel.java @@ -23,8 +23,6 @@ import java.util.Map; import java.util.Set; import java.util.stream.Collectors; -import static com.yahoo.config.model.api.container.ContainerServiceType.CLUSTERCONTROLLER_CONTAINER; - /** * Model with two services, one that does not have a state port * @@ -45,22 +43,6 @@ public class MockModel implements Model { return new HostInfo(hostname, Arrays.asList(container, serviceNoStatePort)); } - // TODO: Move to caller - static MockModel createClusterController(String hostname, int statePort) { - ServiceInfo container = createServiceInfo( - hostname, - "foo", // name - CLUSTERCONTROLLER_CONTAINER.serviceName, - ClusterSpec.Type.container, - statePort, - "state http external query"); - ServiceInfo serviceNoStatePort = createServiceInfo(hostname, "storagenode", "storagenode", - ClusterSpec.Type.content, 1234, "rpc"); - HostInfo hostInfo = new HostInfo(hostname, Arrays.asList(container, serviceNoStatePort)); - - return new MockModel(Collections.singleton(hostInfo)); - } - static MockModel createConfigProxies(List<String> hostnames, int rpcPort) { Set<HostInfo> hostInfos = new HashSet<>(); hostnames.forEach(hostname -> { @@ -71,7 +53,7 @@ public class MockModel implements Model { return new MockModel(hostInfos); } - static private ServiceInfo createServiceInfo( + static ServiceInfo createServiceInfo( String hostname, String name, String type, @@ -121,4 +103,5 @@ public class MockModel implements Model { public AllocatedHosts allocatedHosts() { throw new UnsupportedOperationException(); } + } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java index b2091d6e537..7a8bad3d199 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java @@ -249,7 +249,7 @@ public class DeployTester { public AllocatedHosts getAllocatedHostsOf(ApplicationId applicationId) { Tenant tenant = tenant(); - LocalSession session = tenant.getLocalSessionRepo().getSession(tenant.getApplicationRepo() + LocalSession session = tenant.getSessionRepository().getLocalSession(tenant.getApplicationRepo() .requireActiveSessionOf(applicationId)); return session.getAllocatedHosts(); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/RedeployTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/RedeployTest.java index 36467a2ca64..c07c7316930 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/RedeployTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/RedeployTest.java @@ -33,11 +33,11 @@ public class RedeployTest { assertTrue(deployment.isPresent()); long activeSessionIdBefore = tester.applicationRepository().getActiveSession(tester.applicationId()).getSessionId(); - assertEquals(tester.applicationId(), tester.tenant().getLocalSessionRepo().getSession(activeSessionIdBefore).getApplicationId()); + assertEquals(tester.applicationId(), tester.tenant().getSessionRepository().getLocalSession(activeSessionIdBefore).getApplicationId()); deployment.get().activate(); long activeSessionIdAfter = tester.applicationRepository().getActiveSession(tester.applicationId()).getSessionId(); assertEquals(activeSessionIdAfter, activeSessionIdBefore + 1); - assertEquals(tester.applicationId(), tester.tenant().getLocalSessionRepo().getSession(activeSessionIdAfter).getApplicationId()); + assertEquals(tester.applicationId(), tester.tenant().getSessionRepository().getLocalSession(activeSessionIdAfter).getApplicationId()); } /** No deployment is done because there is no local active session. */ diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionExampleHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionExampleHandlerTest.java deleted file mode 100644 index b6d9ab5d618..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionExampleHandlerTest.java +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.http; - -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.JsonFormat; -import com.yahoo.slime.Slime; -import org.junit.Test; - -import java.io.IOException; -import java.io.OutputStream; -import java.util.concurrent.Executor; -import java.util.concurrent.Executors; - -import static com.yahoo.jdisc.http.HttpResponse.Status.*; -import static com.yahoo.jdisc.http.HttpRequest.Method.*; - -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; - -/** - * @author hmusum - * @since 5.1.14 - */ -public class SessionExampleHandlerTest { - private static final String URI = "http://localhost:19071/session/example"; - - @Test - public void basicPut() throws IOException { - final SessionExampleHandler handler = new SessionExampleHandler(Executors.newCachedThreadPool()); - final HttpRequest request = HttpRequest.createTestRequest(URI, PUT); - HttpResponse response = handler.handle(request); - assertThat(response.getStatus(), is(OK)); - assertThat(SessionHandlerTest.getRenderedString(response), is("{\"test\":\"PUT received\"}")); - } - - @Test - public void invalidMethod() { - final SessionExampleHandler handler = new SessionExampleHandler(Executors.newCachedThreadPool()); - final HttpRequest request = HttpRequest.createTestRequest(URI, GET); - HttpResponse response = handler.handle(request); - assertThat(response.getStatus(), is(METHOD_NOT_ALLOWED)); - } - - - /** - * A handler that prepares a session given by an id in the request. - * - * @author hmusum - * @since 5.1.14 - */ - public static class SessionExampleHandler extends ThreadedHttpRequestHandler { - - public SessionExampleHandler(Executor executor) { - super(executor, null); - } - - @Override - public HttpResponse handle(HttpRequest request) { - final com.yahoo.jdisc.http.HttpRequest.Method method = request.getMethod(); - switch (method) { - case PUT: - return handlePUT(request); - case GET: - return new SessionExampleResponse(METHOD_NOT_ALLOWED, "Method '" + method + "' is not supported"); - default: - return new SessionExampleResponse(INTERNAL_SERVER_ERROR); - } - } - - @SuppressWarnings({"UnusedDeclaration"}) - HttpResponse handlePUT(HttpRequest request) { - return new SessionExampleResponse(OK, "PUT received"); - } - - private static class SessionExampleResponse extends HttpResponse { - private final Slime slime = new Slime(); - private final Cursor root = slime.setObject(); - private final String message; - - - private SessionExampleResponse(int status) { - this(status, ""); - headers().put("Cache-Control","max-age=120"); - } - - private SessionExampleResponse(int status, String message) { - super(status); - this.message = message; - } - - @Override - public void render(OutputStream outputStream) throws IOException { - root.setString("test", message); - new JsonFormat(true).encode(outputStream, slime); - } - } - } -} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionHandlerTest.java index 91a40bd6083..0b9a780d9e1 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionHandlerTest.java @@ -3,11 +3,9 @@ package com.yahoo.vespa.config.server.http; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.application.api.ApplicationPackage; -import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.ProvisionLogger; @@ -18,14 +16,9 @@ import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.path.Path; import com.yahoo.transaction.NestedTransaction; import com.yahoo.transaction.Transaction; -import com.yahoo.vespa.config.server.application.ApplicationSet; -import com.yahoo.vespa.config.server.configchange.ConfigChangeActions; -import com.yahoo.vespa.config.server.host.HostRegistry; import com.yahoo.vespa.config.server.session.DummyTransaction; import com.yahoo.vespa.config.server.session.LocalSession; import com.yahoo.vespa.config.server.session.MockSessionZKClient; -import com.yahoo.vespa.config.server.session.PrepareParams; -import com.yahoo.vespa.config.server.session.RemoteSession; import com.yahoo.vespa.config.server.session.Session; import java.io.ByteArrayOutputStream; @@ -36,7 +29,6 @@ import java.time.Instant; import java.util.Collection; import java.util.List; import java.util.Map; -import java.util.Optional; /** * Base class for session handler tests @@ -90,13 +82,11 @@ public class SessionHandlerTest { public static class MockLocalSession extends LocalSession { public Session.Status status; - private ConfigChangeActions actions = new ConfigChangeActions(); private Instant createTime = Instant.now(); private ApplicationId applicationId; - private Optional<DockerImage> dockerImageRepository; public MockLocalSession(long sessionId, ApplicationPackage app) { - super(TenantName.defaultName(), sessionId, null, app, new MockSessionZKClient(app), null, null, new HostRegistry<>()); + super(TenantName.defaultName(), sessionId, app, new MockSessionZKClient(app), null); } public MockLocalSession(long sessionId, ApplicationPackage app, ApplicationId applicationId) { @@ -104,13 +94,6 @@ public class SessionHandlerTest { this.applicationId = applicationId; } - @Override - public ConfigChangeActions prepare(DeployLogger logger, PrepareParams params, Optional<ApplicationSet> application, Path tenantPath, Instant now) { - status = Session.Status.PREPARE; - this.dockerImageRepository = params.dockerImageRepository(); - return actions; - } - public void setStatus(Session.Status status) { this.status = status; } @@ -140,13 +123,6 @@ public class SessionHandlerTest { return createTime; } - @Override - public void delete(NestedTransaction transaction) { } - - @Override - public Optional<DockerImage> getDockerImageRepository() { - return dockerImageRepository; - } } public enum Cmd { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java index 078dc47af51..4cf81d22e3c 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java @@ -54,13 +54,13 @@ public class ApplicationContentHandlerTest extends ContentHandlerTestBase { session2 = new MockLocalSession(2, FilesApplicationPackage.fromFile(new File("src/test/apps/content"))); Tenant tenant1 = tenantRepository.getTenant(tenantName1); - tenant1.getLocalSessionRepo().addSession(session2); + tenant1.getSessionRepository().addSession(session2); tenant1.getApplicationRepo().createApplication(idTenant1); tenant1.getApplicationRepo().createPutTransaction(idTenant1, 2).commit(); MockLocalSession session3 = new MockLocalSession(3, FilesApplicationPackage.fromFile(new File("src/test/apps/content2"))); Tenant tenant2 = tenantRepository.getTenant(tenantName2); - tenant2.getLocalSessionRepo().addSession(session3); + tenant2.getSessionRepository().addSession(session3); tenant2.getApplicationRepo().createApplication(idTenant2); tenant2.getApplicationRepo().createPutTransaction(idTenant2, 3).commit(); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java index ff8f7a291ad..0a5221b2c97 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java @@ -286,13 +286,13 @@ public class ApplicationHandlerTest { private void deleteAndAssertOKResponseMocked(ApplicationId applicationId, boolean fullAppIdInUrl) throws IOException { long sessionId = tenantRepository.getTenant(applicationId.tenant()).getApplicationRepo().requireActiveSessionOf(applicationId); deleteAndAssertResponse(applicationId, Zone.defaultZone(), Response.Status.OK, null, fullAppIdInUrl); - assertNull(tenantRepository.getTenant(applicationId.tenant()).getLocalSessionRepo().getSession(sessionId)); + assertNull(tenantRepository.getTenant(applicationId.tenant()).getSessionRepository().getLocalSession(sessionId)); } private void deleteAndAssertOKResponse(Tenant tenant, ApplicationId applicationId) throws IOException { long sessionId = tenant.getApplicationRepo().requireActiveSessionOf(applicationId); deleteAndAssertResponse(applicationId, Zone.defaultZone(), Response.Status.OK, null, true); - assertNull(tenant.getLocalSessionRepo().getSession(sessionId)); + assertNull(tenant.getSessionRepository().getLocalSession(sessionId)); } private void deleteAndAssertResponse(ApplicationId applicationId, Zone zone, int expectedStatus, HttpErrorResponse.errorCodes errorCode, boolean fullAppIdInUrl) throws IOException { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java index 2eaa5d75ba7..37181abfcf4 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java @@ -12,8 +12,9 @@ import com.yahoo.config.provision.Zone; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.jdisc.Response; +import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.TestComponentRegistry; -import com.yahoo.vespa.config.server.host.HostRegistries; +import com.yahoo.vespa.config.server.application.OrchestratorMock; import com.yahoo.vespa.config.server.host.HostRegistry; import com.yahoo.vespa.config.server.http.HandlerTest; import com.yahoo.vespa.config.server.http.HttpErrorResponse; @@ -29,14 +30,13 @@ import org.junit.Test; import java.io.File; import java.io.IOException; +import java.time.Clock; import java.util.Collections; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; - /** * @author hmusum */ +// TODO: Try to move testing to ApplicationRepositoryTest and avoid all the low-level setup code here public class HostHandlerTest { private static final String urlPrefix = "http://myhost:14000/application/v2/host/"; private static File testApp = new File("src/test/apps/app"); @@ -44,49 +44,45 @@ public class HostHandlerTest { private HostHandler handler; private final static TenantName mytenant = TenantName.from("mytenant"); private final static String hostname = "testhost"; + private final static Zone zone = Zone.defaultZone(); private TenantRepository tenantRepository; - private HostRegistries hostRegistries; - private HostHandler hostHandler; static void addMockApplication(Tenant tenant, ApplicationId applicationId, long sessionId) { tenant.getApplicationRepo().createApplication(applicationId); tenant.getApplicationRepo().createPutTransaction(applicationId, sessionId).commit(); ApplicationPackage app = FilesApplicationPackage.fromFile(testApp); - tenant.getLocalSessionRepo().addSession(new SessionHandlerTest.MockLocalSession(sessionId, app, applicationId)); + tenant.getSessionRepository().addSession(new SessionHandlerTest.MockLocalSession(sessionId, app, applicationId)); TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder() .modelFactoryRegistry(new ModelFactoryRegistry(Collections.singletonList(new VespaModelFactory(new NullConfigModelRegistry())))) .build(); - tenant.getRemoteSessionRepo().addSession(new RemoteSession(tenant.getName(), sessionId, componentRegistry, new MockSessionZKClient(app))); + tenant.getSessionRepo().addRemoteSession(new RemoteSession(tenant.getName(), sessionId, componentRegistry, new MockSessionZKClient(app))); } @Before public void setup() { - TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder().build(); - tenantRepository = new TenantRepository(componentRegistry, false); - tenantRepository.addTenant(mytenant); - handler = createHostHandler(); - } - - private HostHandler createHostHandler() { final HostRegistry<TenantName> hostRegistry = new HostRegistry<>(); hostRegistry.update(mytenant, Collections.singletonList(hostname)); - TestComponentRegistry testComponentRegistry = new TestComponentRegistry.Builder().build(); - hostRegistries = testComponentRegistry.getHostRegistries(); - hostRegistries.createApplicationHostRegistry(mytenant).update(ApplicationId.from(mytenant, ApplicationName.defaultName(), InstanceName.defaultName()), Collections.singletonList(hostname)); - hostRegistries.getTenantHostRegistry().update(mytenant, Collections.singletonList(hostname)); - hostHandler = new HostHandler( - HostHandler.testOnlyContext(), - testComponentRegistry); - return hostHandler; + TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder() + .zone(zone) + .build(); + tenantRepository = new TenantRepository(componentRegistry, false); + tenantRepository.addTenant(mytenant); + Tenant tenant = tenantRepository.getTenant(mytenant); + HostRegistry<ApplicationId> applicationHostRegistry = tenant.getApplicationRepo().getApplicationHostRegistry(); + applicationHostRegistry.update(ApplicationId.from(mytenant, ApplicationName.defaultName(), InstanceName.defaultName()), Collections.singletonList(hostname)); + ApplicationRepository applicationRepository = new ApplicationRepository(tenantRepository, + new SessionHandlerTest.MockProvisioner(), + new OrchestratorMock(), + Clock.systemUTC()); + handler = new HostHandler(HostHandler.testOnlyContext(), applicationRepository); } @Test public void require_correct_tenant_and_application_for_hostname() throws Exception { - assertThat(hostRegistries, is(hostHandler.hostRegistries)); long sessionId = 1; ApplicationId id = ApplicationId.from(mytenant, ApplicationName.defaultName(), InstanceName.defaultName()); addMockApplication(tenantRepository.getTenant(mytenant), id, sessionId); - assertApplicationForHost(hostname, mytenant, id, Zone.defaultZone()); + assertApplicationForHost(hostname, mytenant, id, zone); } @Test diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java index 88bf6fb7172..f639843ac08 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java @@ -43,7 +43,7 @@ public class SessionContentHandlerTest extends ContentHandlerTestBase { public void setupHandler() throws Exception { tenantRepository = new TenantRepository(componentRegistry, false); tenantRepository.addTenant(tenant); - tenantRepository.getTenant(tenant).getLocalSessionRepo().addSession(new MockLocalSession(1L, FilesApplicationPackage.fromFile(createTestApp()))); + tenantRepository.getTenant(tenant).getSessionRepository().addSession(new MockLocalSession(1L, FilesApplicationPackage.fromFile(createTestApp()))); handler = createHandler(); pathPrefix = "/application/v2/tenant/" + tenant + "/session/"; baseUrl = "http://foo:1337/application/v2/tenant/" + tenant + "/session/1/content/"; diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/a.search-clustermusic-c0-r0-indexer4.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/a.search-clustermusic-c0-r0-indexer4.cfg deleted file mode 100644 index d3970ee48eb..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/a.search-clustermusic-c0-r0-indexer4.cfg +++ /dev/null @@ -1,44 +0,0 @@ -include: search/cluster.music -include: search/cluster.music -c 2 -storage[2] -storage[0].feeder[1] -storage[0].feeder[0] "test" -storage[1].feeder[2] -storage[1].feeder[0] "me" -storage[1].feeder[1] now -storage[1].id :parent: -storage[1].id2 pjatt -testref :parent: -testref2 some/babbel -config[1] -config[0].role "rtx" -#config[0].usewrapper false -config[0].id search/cluster.music/rtx/0 -f[1] -f[0].a "A" -f[0].b "B" -f[0].c "C" -f[0].h "H" -f[0].f "F" -f[0].notindef "notindef" -routingtable[1] -routingtable[0].hop[3] -routingtable[0].hop[0].name "docproc/cluster.music.indexing/chain.music.indexing" -routingtable[0].hop[0].selector "docproc/cluster.music.indexing/*/chain.music.indexing" -routingtable[0].hop[1].name "search/cluster.music" -routingtable[0].hop[1].selector "search/cluster.music/[SearchColumn]/[SearchRow]/feed-destination" -routingtable[0].hop[1].recipient[1] -routingtable[0].hop[1].recipient[0] "search/cluster.music/c0/r0/feed-destination" -routingtable[0].hop[2].selector "[DocumentRouteSelector]" -routingtable[0].hop[2].name "indexing" -routingtable[0].hop[2].notindef "not in def" -routingtable[0].hop[2].recipient[1] -routingtable[0].hop[2].recipient[0] "search/cluster.music" -notindef "dfsd" -nopenotindef[0] "boo" -nadaindef[0].naw 98 -mode NOTINDEF -rangecheck1 100 -rangecheck2 10000 -rangecheck3 20 diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/a.search-clustersports-c0-r0-indexer4.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/a.search-clustersports-c0-r0-indexer4.cfg deleted file mode 100644 index 727a5052ed6..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/a.search-clustersports-c0-r0-indexer4.cfg +++ /dev/null @@ -1,2 +0,0 @@ -include: search/cluster.sports -c 67 diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/a.vespamodel.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/a.vespamodel.cfg deleted file mode 100644 index f4996027f60..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/a.vespamodel.cfg +++ /dev/null @@ -1 +0,0 @@ -model vespa diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/c.search-clustersports-c0-r0-indexer4.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/c.search-clustersports-c0-r0-indexer4.cfg deleted file mode 100644 index d75d76810f9..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/c.search-clustersports-c0-r0-indexer4.cfg +++ /dev/null @@ -1,2 +0,0 @@ -foo "bar" -gaz -78 diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/compositeinclude.search-qrservers-0.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/compositeinclude.search-qrservers-0.cfg deleted file mode 100644 index 7ccdb73eb9a..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/compositeinclude.search-qrservers-0.cfg +++ /dev/null @@ -1,2 +0,0 @@ -include: search/cluster.logical/* -include: search/cluster.video/* diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/recursiveinclude.search-clustermusic-c0-r0.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/recursiveinclude.search-clustermusic-c0-r0.cfg deleted file mode 100644 index 5b07d3a2890..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/recursiveinclude.search-clustermusic-c0-r0.cfg +++ /dev/null @@ -1 +0,0 @@ -include: search/cluster.music diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/a.search-clustermusic.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/a.search-clustermusic.cfg deleted file mode 100644 index f3acd4cf8b9..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/a.search-clustermusic.cfg +++ /dev/null @@ -1,5 +0,0 @@ -asyncfetchocc 9 -d 3 -kanon -78.56 - -partialsd "sd" diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/a.search-clustersports.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/a.search-clustersports.cfg deleted file mode 100644 index 5d8a01a18ea..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/a.search-clustersports.cfg +++ /dev/null @@ -1,2 +0,0 @@ -d 89 -search[1].feeder[1] "sportsfeeder1" diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/b.search-clustersports.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/b.search-clustersports.cfg deleted file mode 100644 index f6c35df398d..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/b.search-clustersports.cfg +++ /dev/null @@ -1 +0,0 @@ -gaff -89 diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/compositeinclude.search-clusterlogical.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/compositeinclude.search-clusterlogical.cfg deleted file mode 100644 index c3d9b1e45a1..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/compositeinclude.search-clusterlogical.cfg +++ /dev/null @@ -1,8 +0,0 @@ -classes[1] -classes[logical].id 1906788747 -classes[logical].name logical -classes[logical].fields[2] -classes[logical].fields[0].name sddocnameNAM -classes[logical].fields[0].type longstring -classes[logical].fields[1].name title -classes[logical].fields[1].type longstringSTRIN diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/compositeinclude.search-clustervideo.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/compositeinclude.search-clustervideo.cfg deleted file mode 100644 index 12a21671b4a..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/compositeinclude.search-clustervideo.cfg +++ /dev/null @@ -1,8 +0,0 @@ -classes[1] -classes[music].id 1906788746 -classes[music].name music -classes[music].fields[2] -classes[music].fields[0].name sddocnameNAME -classes[music].fields[0].type longstring -classes[music].fields[1].name title -classes[music].fields[1].type longstringSTRING diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/compositeinclude.search-part-clusterlogical.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/compositeinclude.search-part-clusterlogical.cfg deleted file mode 100644 index 4001c59adbc..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/compositeinclude.search-part-clusterlogical.cfg +++ /dev/null @@ -1,14 +0,0 @@ -classes[0] -classes[smallsum614540714].id 614540714 -classes[smallsum614540714].name smallsum -classes[smallsum614540714].fields[5] -classes[smallsum614540714].fields[0].name s_13 -classes[smallsum614540714].fields[0].type longstring -classes[smallsum614540714].fields[1].name ranklog -classes[smallsum614540714].fields[1].type longstring -classes[smallsum614540714].fields[2].name rankfeatures -classes[smallsum614540714].fields[2].type longstring -classes[smallsum614540714].fields[3].name summaryfeatures -classes[smallsum614540714].fields[3].type longstring -classes[smallsum614540714].fields[4].name sddocname -classes[smallsum614540714].fields[4].type longstring diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/compositeinclude.search-part-clustervideo.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/compositeinclude.search-part-clustervideo.cfg deleted file mode 100644 index 33d07b99ab6..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/compositeinclude.search-part-clustervideo.cfg +++ /dev/null @@ -1,14 +0,0 @@ -classes[0] -classes[smallsum507688128].id 507688128 -classes[smallsum507688128].name smallsum -classes[smallsum507688128].fields[5] -classes[smallsum507688128].fields[0].name title -classes[smallsum507688128].fields[0].type longstring -classes[smallsum507688128].fields[1].name ranklog -classes[smallsum507688128].fields[1].type longstring -classes[smallsum507688128].fields[2].name rankfeatures -classes[smallsum507688128].fields[2].type longstring -classes[smallsum507688128].fields[3].name summaryfeatures -classes[smallsum507688128].fields[3].type longstring -classes[smallsum507688128].fields[4].name sddocname -classes[smallsum507688128].fields[4].type longstring diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/recursiveinclude.search-clustermusic-conf1.4.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/recursiveinclude.search-clustermusic-conf1.4.cfg deleted file mode 100644 index de9fbdd39f4..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/recursiveinclude.search-clustermusic-conf1.4.cfg +++ /dev/null @@ -1,7 +0,0 @@ -rec 56 -national 77 -ilscript[1] -ilscript[music].name music -ilscript[music].doctype music -ilscript[music].content[1] -ilscript[music].content[0] "input year | summary s_3 | tokenize \"stemming,normalizing\" { index f_3 | index f_4; };" diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/recursiveinclude.search-clustermusic-conf2.4.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/recursiveinclude.search-clustermusic-conf2.4.cfg deleted file mode 100644 index e95f976a43a..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/recursiveinclude.search-clustermusic-conf2.4.cfg +++ /dev/null @@ -1,7 +0,0 @@ -ursive -50 -teatern 78 -ilscript[1] -ilscript[father].name father -ilscript[father].doctype father -ilscript[father].content[6] -ilscript[father].content[0] "input year | summary s_3 | tokenize \"stemming,normalizing\" { index f_3 | index f_5; };" diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/recursiveinclude.search-clustermusic.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/recursiveinclude.search-clustermusic.cfg deleted file mode 100644 index cea943d5bc9..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/sdconfigs/recursiveinclude.search-clustermusic.cfg +++ /dev/null @@ -1,2 +0,0 @@ -include: search/cluster.music/conf1.sd.derived -include: search/cluster.music/conf2.sd.derived diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionRepoTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionRepoTest.java deleted file mode 100644 index a758698d3b5..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionRepoTest.java +++ /dev/null @@ -1,106 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.session; - -import com.yahoo.cloud.config.ConfigserverConfig; -import com.yahoo.config.model.application.provider.FilesApplicationPackage; -import com.yahoo.config.provision.TenantName; -import com.yahoo.io.IOUtils; -import com.yahoo.vespa.config.server.GlobalComponentRegistry; -import com.yahoo.vespa.config.server.MockReloadHandler; -import com.yahoo.vespa.config.server.TestComponentRegistry; -import com.yahoo.vespa.config.server.application.TenantApplications; -import com.yahoo.vespa.config.server.host.HostRegistry; -import com.yahoo.vespa.config.server.http.SessionHandlerTest; -import com.yahoo.vespa.curator.mock.MockCurator; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - -import java.io.File; -import java.nio.file.Path; -import java.nio.file.Paths; - -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.fail; - -/** - * @author Ulf Lilleengen - */ -public class LocalSessionRepoTest { - - private File testApp = new File("src/test/apps/app"); - private LocalSessionRepo repo; - private static final TenantName tenantName = TenantName.defaultName(); - - @Rule - public TemporaryFolder temporaryFolder = new TemporaryFolder(); - - @Before - public void setupSessions() throws Exception { - setupSessions(tenantName, true); - } - - private void setupSessions(TenantName tenantName, boolean createInitialSessions) throws Exception { - File configserverDbDir = temporaryFolder.newFolder().getAbsoluteFile(); - if (createInitialSessions) { - Path sessionsPath = Paths.get(configserverDbDir.getAbsolutePath(), "tenants", tenantName.value(), "sessions"); - IOUtils.copyDirectory(testApp, sessionsPath.resolve("1").toFile()); - IOUtils.copyDirectory(testApp, sessionsPath.resolve("2").toFile()); - IOUtils.copyDirectory(testApp, sessionsPath.resolve("3").toFile()); - } - GlobalComponentRegistry globalComponentRegistry = new TestComponentRegistry.Builder() - .curator(new MockCurator()) - .configServerConfig(new ConfigserverConfig.Builder() - .configServerDBDir(configserverDbDir.getAbsolutePath()) - .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) - .sessionLifetime(5) - .build()) - .build(); - SessionFactory sessionFactory = new SessionFactory(globalComponentRegistry, - TenantApplications.create(globalComponentRegistry, tenantName), - new HostRegistry<>(), - tenantName); - repo = new LocalSessionRepo(tenantName, globalComponentRegistry, sessionFactory); - } - - @Test - public void require_that_sessions_can_be_loaded_from_disk() { - assertNotNull(repo.getSession(1L)); - assertNotNull(repo.getSession(2L)); - assertNotNull(repo.getSession(3L)); - assertNull(repo.getSession(4L)); - } - - @Test - public void require_that_all_sessions_are_deleted() { - repo.close(); - assertNull(repo.getSession(1L)); - assertNull(repo.getSession(2L)); - assertNull(repo.getSession(3L)); - } - - @Test - public void require_that_sessions_belong_to_a_tenant() { - // tenant is "default" - assertNotNull(repo.getSession(1L)); - assertNotNull(repo.getSession(2L)); - assertNotNull(repo.getSession(3L)); - assertNull(repo.getSession(4L)); - - // tenant is "newTenant" - try { - setupSessions(TenantName.from("newTenant"), false); - } catch (Exception e) { - fail(); - } - assertNull(repo.getSession(1L)); - - repo.addSession(new SessionHandlerTest.MockLocalSession(1L, FilesApplicationPackage.fromFile(testApp))); - repo.addSession(new SessionHandlerTest.MockLocalSession(2L, FilesApplicationPackage.fromFile(testApp))); - assertNotNull(repo.getSession(1L)); - assertNotNull(repo.getSession(2L)); - assertNull(repo.getSession(3L)); - } -} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java index b072f20414f..c1377ae439b 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.session; +import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.Version; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.model.application.provider.BaseDeployLogger; @@ -11,27 +12,25 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; import com.yahoo.path.Path; import com.yahoo.slime.Slime; -import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.config.server.TestComponentRegistry; import com.yahoo.vespa.config.server.application.TenantApplications; import com.yahoo.vespa.config.server.deploy.DeployHandlerLogger; -import com.yahoo.vespa.config.server.deploy.TenantFileSystemDirs; import com.yahoo.vespa.config.server.deploy.ZooKeeperClient; -import com.yahoo.vespa.config.server.host.HostRegistry; import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; import java.io.File; -import java.nio.file.Files; +import java.io.IOException; import java.time.Instant; import java.util.Collections; import java.util.Optional; -import static com.yahoo.yolean.Exceptions.uncheck; import static org.hamcrest.core.Is.is; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertThat; @@ -43,18 +42,29 @@ import static org.junit.Assert.assertTrue; public class LocalSessionTest { private static final File testApp = new File("src/test/apps/app"); + private static final TenantName tenantName = TenantName.from("test_tenant"); + private static final Path tenantPath = Path.createRoot(); - private Path tenantPath = Path.createRoot(); + private TenantRepository tenantRepository; private Curator curator; private ConfigCurator configCurator; - private TenantFileSystemDirs tenantFileSystemDirs; + + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); @Before - public void setupTest() { + public void setupTest() throws IOException { curator = new MockCurator(); + TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder() + .curator(curator) + .configServerConfig(new ConfigserverConfig.Builder() + .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) + .configServerDBDir(temporaryFolder.newFolder().getAbsolutePath()) + .build()) + .build(); + tenantRepository = new TenantRepository(componentRegistry, false); + tenantRepository.addTenant(tenantName); configCurator = ConfigCurator.create(curator); - tenantFileSystemDirs = new TenantFileSystemDirs(uncheck(() -> Files.createTempDirectory("serverdb")).toFile(), - TenantName.from("test_tenant")); } @Test @@ -96,35 +106,16 @@ public class LocalSessionTest { assertFalse(f2.exists()); } - @Test - public void require_that_session_can_be_deleted() throws Exception { - TenantName tenantName = TenantName.defaultName(); - LocalSession session = createSession(tenantName, 3); - String sessionNode = TenantRepository.getSessionsPath(tenantName).append(String.valueOf(3)).getAbsolute(); - assertTrue(configCurator.exists(sessionNode)); - assertTrue(new File(tenantFileSystemDirs.sessionsPath(), "3").exists()); - NestedTransaction transaction = new NestedTransaction(); - session.delete(transaction); - transaction.commit(); - assertFalse(configCurator.exists(sessionNode)); - assertFalse(new File(tenantFileSystemDirs.sessionsPath(), "3").exists()); - } - @Test(expected = IllegalStateException.class) public void require_that_no_provision_info_throws_exception() throws Exception { createSession(TenantName.defaultName(), 3).getAllocatedHosts(); } private LocalSession createSession(TenantName tenant, long sessionId) throws Exception { - SessionTest.MockSessionPreparer preparer = new SessionTest.MockSessionPreparer(); - return createSession(tenant, sessionId, preparer); - } - - private LocalSession createSession(TenantName tenant, long sessionId, SessionTest.MockSessionPreparer preparer) throws Exception { - return createSession(tenant, sessionId, preparer, Optional.empty()); + return createSession(tenant, sessionId, Optional.empty()); } - private LocalSession createSession(TenantName tenant, long sessionId, SessionTest.MockSessionPreparer preparer, + private LocalSession createSession(TenantName tenant, long sessionId, Optional<AllocatedHosts> allocatedHosts) throws Exception { SessionZooKeeperClient zkc = new MockSessionZKClient(curator, tenant, sessionId, allocatedHosts); zkc.createWriteStatusTransaction(Session.Status.NEW).commit(); @@ -134,13 +125,9 @@ public class LocalSessionTest { zkClient.write(allocatedHosts.get()); } zkClient.write(Collections.singletonMap(new Version(0, 0, 0), new MockFileRegistry())); - File sessionDir = new File(tenantFileSystemDirs.sessionsPath(), String.valueOf(sessionId)); - sessionDir.createNewFile(); - TenantApplications applications = TenantApplications.create( - new TestComponentRegistry.Builder().curator(curator).build(), tenant); + TenantApplications applications = tenantRepository.getTenant(tenantName).getApplicationRepo(); applications.createApplication(zkc.readApplicationId()); - return new LocalSession(tenant, sessionId, preparer, FilesApplicationPackage.fromFile(testApp), - zkc, sessionDir, applications, new HostRegistry<>()); + return new LocalSession(tenant, sessionId, FilesApplicationPackage.fromFile(testApp), zkc, applications); } private void doPrepare(LocalSession session) { @@ -148,12 +135,13 @@ public class LocalSessionTest { } private void doPrepare(LocalSession session, PrepareParams params) { - session.prepare(getLogger(), params, Optional.empty(), tenantPath, Instant.now()); + SessionRepository sessionRepository = tenantRepository.getTenant(tenantName).getSessionRepository(); + sessionRepository.prepareLocalSession(session, getLogger(), params, Optional.empty(), tenantPath, Instant.now()); } private DeployHandlerLogger getLogger() { return new DeployHandlerLogger(new Slime().get(), false, - new ApplicationId.Builder().tenant("testtenant").applicationName("testapp").build()); + new ApplicationId.Builder().tenant(tenantName).applicationName("testapp").build()); } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/RemoteSessionRepoTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/RemoteSessionRepoTest.java deleted file mode 100644 index 468dd5a15a7..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/RemoteSessionRepoTest.java +++ /dev/null @@ -1,145 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.session; - -import com.yahoo.config.provision.TenantName; -import com.yahoo.path.Path; -import com.yahoo.text.Utf8; -import com.yahoo.vespa.config.server.TestComponentRegistry; -import com.yahoo.vespa.config.server.tenant.Tenant; -import com.yahoo.vespa.config.server.tenant.TenantRepository; -import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; -import com.yahoo.vespa.curator.Curator; -import com.yahoo.vespa.curator.mock.MockCurator; -import org.junit.Before; -import org.junit.Test; - -import java.time.Duration; -import java.time.Instant; -import java.util.function.LongPredicate; - -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertThat; - -/** - * @author Ulf Lilleengen - */ -public class RemoteSessionRepoTest { - - private static final TenantName tenantName = TenantName.defaultName(); - - private RemoteSessionRepo remoteSessionRepo; - private Curator curator; - TenantRepository tenantRepository; - - @Before - public void setupFacade() { - curator = new MockCurator(); - TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder() - .curator(curator) - .build(); - tenantRepository = new TenantRepository(componentRegistry, false); - tenantRepository.addTenant(tenantName); - this.remoteSessionRepo = tenantRepository.getTenant(tenantName).getRemoteSessionRepo(); - curator.create(TenantRepository.getTenantPath(tenantName).append("/applications")); - curator.create(TenantRepository.getSessionsPath(tenantName)); - createSession(1L, false); - createSession(2L, false); - } - - private void createSession(long sessionId, boolean wait) { - createSession(sessionId, wait, tenantName); - } - - private void createSession(long sessionId, boolean wait, TenantName tenantName) { - Path sessionsPath = TenantRepository.getSessionsPath(tenantName); - SessionZooKeeperClient zkc = new SessionZooKeeperClient(curator, sessionsPath.append(String.valueOf(sessionId))); - zkc.createNewSession(Instant.now()); - if (wait) { - Curator.CompletionWaiter waiter = zkc.getUploadWaiter(); - waiter.awaitCompletion(Duration.ofSeconds(120)); - } - } - - @Test - public void testInitialize() { - assertSessionExists(1L); - assertSessionExists(2L); - } - - @Test - public void testCreateSession() { - createSession(3L, true); - assertSessionExists(3L); - } - - @Test - public void testSessionStateChange() throws Exception { - long sessionId = 3L; - createSession(sessionId, true); - assertSessionStatus(sessionId, Session.Status.NEW); - assertStatusChange(sessionId, Session.Status.PREPARE); - assertStatusChange(sessionId, Session.Status.ACTIVATE); - - Path session = TenantRepository.getSessionsPath(tenantName).append("" + sessionId); - curator.delete(session); - assertSessionRemoved(sessionId); - assertNull(remoteSessionRepo.getSession(sessionId)); - } - - // If reading a session throws an exception it should be handled and not prevent other applications - // from loading. In this test we just show that we end up with one session in remote session - // repo even if it had bad data (by making getSessionIdForApplication() in FailingTenantApplications - // throw an exception). - @Test - public void testBadApplicationRepoOnActivate() { - long sessionId = 3L; - TenantName mytenant = TenantName.from("mytenant"); - curator.set(TenantRepository.getApplicationsPath(mytenant).append("mytenant:appX:default"), new byte[0]); // Invalid data - tenantRepository.addTenant(mytenant); - Tenant tenant = tenantRepository.getTenant(mytenant); - curator.create(TenantRepository.getSessionsPath(mytenant)); - remoteSessionRepo = tenant.getRemoteSessionRepo(); - assertThat(remoteSessionRepo.getSessions().size(), is(0)); - createSession(sessionId, true, mytenant); - assertThat(remoteSessionRepo.getSessions().size(), is(1)); - } - - private void assertStatusChange(long sessionId, Session.Status status) throws Exception { - Path statePath = TenantRepository.getSessionsPath(tenantName).append("" + sessionId).append(ConfigCurator.SESSIONSTATE_ZK_SUBPATH); - curator.create(statePath); - curator.framework().setData().forPath(statePath.getAbsolute(), Utf8.toBytes(status.toString())); - assertSessionStatus(sessionId, status); - } - - private void assertSessionRemoved(long sessionId) { - waitFor(p -> remoteSessionRepo.getSession(sessionId) == null, sessionId); - assertNull(remoteSessionRepo.getSession(sessionId)); - } - - private void assertSessionExists(long sessionId) { - assertSessionStatus(sessionId, Session.Status.NEW); - } - - private void assertSessionStatus(long sessionId, Session.Status status) { - waitFor(p -> remoteSessionRepo.getSession(sessionId) != null && - remoteSessionRepo.getSession(sessionId).getStatus() == status, sessionId); - assertNotNull(remoteSessionRepo.getSession(sessionId)); - assertThat(remoteSessionRepo.getSession(sessionId).getStatus(), is(status)); - } - - private void waitFor(LongPredicate predicate, long sessionId) { - long endTime = System.currentTimeMillis() + 60_000; - boolean ok; - do { - ok = predicate.test(sessionId); - try { - Thread.sleep(10); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } while (System.currentTimeMillis() < endTime && !ok); - } - -} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionRepositoryTest.java new file mode 100644 index 00000000000..b9e872261c5 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionRepositoryTest.java @@ -0,0 +1,217 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.session; + +import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.TenantName; +import com.yahoo.text.Utf8; +import com.yahoo.vespa.config.server.ApplicationRepository; +import com.yahoo.vespa.config.server.GlobalComponentRegistry; +import com.yahoo.vespa.config.server.TestComponentRegistry; +import com.yahoo.vespa.config.server.application.OrchestratorMock; +import com.yahoo.vespa.config.server.http.SessionHandlerTest; +import com.yahoo.vespa.config.server.tenant.Tenant; +import com.yahoo.vespa.config.server.tenant.TenantRepository; +import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; +import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.InMemoryFlagSource; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.util.function.LongPredicate; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThat; + +/** + * @author Ulf Lilleengen + */ +public class SessionRepositoryTest { + + private static final TenantName tenantName = TenantName.defaultName(); + private static final ApplicationId applicationId = ApplicationId.from(tenantName.value(), "testApp", "default"); + private static final File testApp = new File("src/test/apps/app"); + + private MockCurator curator; + private TenantRepository tenantRepository; + private ApplicationRepository applicationRepository; + + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + public void setup() throws Exception { + setup(new InMemoryFlagSource()); + } + + private void setup(FlagSource flagSource) throws Exception { + curator = new MockCurator(); + File configserverDbDir = temporaryFolder.newFolder().getAbsoluteFile(); + GlobalComponentRegistry globalComponentRegistry = new TestComponentRegistry.Builder() + .curator(curator) + .configServerConfig(new ConfigserverConfig.Builder() + .configServerDBDir(configserverDbDir.getAbsolutePath()) + .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) + .fileReferencesDir(temporaryFolder.newFolder().getAbsolutePath()) + .sessionLifetime(5) + .build()) + .flagSource(flagSource) + .build(); + tenantRepository = new TenantRepository(globalComponentRegistry, false); + tenantRepository.addTenant(SessionRepositoryTest.tenantName); + applicationRepository = new ApplicationRepository(tenantRepository, + new SessionHandlerTest.MockProvisioner(), + new OrchestratorMock(), + Clock.systemUTC()); + } + + @Test + public void require_that_local_sessions_are_created_and_deleted() throws Exception { + setup(); + long firstSessionId = deploy(); + long secondSessionId = deploy(); + SessionRepository sessionRepository = tenantRepository.getTenant(tenantName).getSessionRepository(); + assertNotNull(sessionRepository.getLocalSession(firstSessionId)); + assertNotNull(sessionRepository.getLocalSession(secondSessionId)); + assertNull(sessionRepository.getLocalSession(secondSessionId + 1)); + + sessionRepository.close(); + // All created sessions are deleted + assertNull(sessionRepository.getLocalSession(firstSessionId)); + assertNull(sessionRepository.getLocalSession(secondSessionId)); + } + + @Test + public void require_that_local_sessions_belong_to_a_tenant() throws Exception { + setup(); + // tenant is "default" + + long firstSessionId = deploy(); + long secondSessionId = deploy(); + SessionRepository sessionRepository = tenantRepository.getTenant(tenantName).getSessionRepository(); + assertNotNull(sessionRepository.getLocalSession(firstSessionId)); + assertNotNull(sessionRepository.getLocalSession(secondSessionId)); + assertNull(sessionRepository.getLocalSession(secondSessionId + 1)); + + // tenant is "newTenant" + TenantName newTenant = TenantName.from("newTenant"); + tenantRepository.addTenant(newTenant); + long sessionId = deploy(ApplicationId.from(newTenant.value(), "testapp", "default")); + SessionRepository sessionRepository2 = tenantRepository.getTenant(newTenant).getSessionRepository(); + assertNotNull(sessionRepository2.getLocalSession(sessionId)); + } + + @Test + public void testInitialize() throws Exception { + setup(); + createSession(10L, false); + createSession(11L, false); + assertRemoteSessionExists(10L); + assertRemoteSessionExists(11L); + } + + @Test + public void testSessionStateChange() throws Exception { + setup(); + long sessionId = 3L; + createSession(sessionId, true); + assertRemoteSessionStatus(sessionId, Session.Status.NEW); + assertStatusChange(sessionId, Session.Status.PREPARE); + assertStatusChange(sessionId, Session.Status.ACTIVATE); + + com.yahoo.path.Path session = TenantRepository.getSessionsPath(tenantName).append("" + sessionId); + curator.delete(session); + assertSessionRemoved(sessionId); + SessionRepository sessionRepository = tenantRepository.getTenant(tenantName).getSessionRepository(); + assertNull(sessionRepository.getRemoteSession(sessionId)); + } + + // If reading a session throws an exception it should be handled and not prevent other applications + // from loading. In this test we just show that we end up with one session in remote session + // repo even if it had bad data (by making getSessionIdForApplication() in FailingTenantApplications + // throw an exception). + @Test + public void testBadApplicationRepoOnActivate() throws Exception { + setup(); + long sessionId = 3L; + TenantName mytenant = TenantName.from("mytenant"); + curator.set(TenantRepository.getApplicationsPath(mytenant).append("mytenant:appX:default"), new byte[0]); // Invalid data + tenantRepository.addTenant(mytenant); + Tenant tenant = tenantRepository.getTenant(mytenant); + curator.create(TenantRepository.getSessionsPath(mytenant)); + SessionRepository sessionRepository = tenant.getSessionRepo(); + assertThat(sessionRepository.getRemoteSessions().size(), is(0)); + createSession(sessionId, true, mytenant); + assertThat(sessionRepository.getRemoteSessions().size(), is(1)); + } + + private void createSession(long sessionId, boolean wait) { + createSession(sessionId, wait, tenantName); + } + + private void createSession(long sessionId, boolean wait, TenantName tenantName) { + com.yahoo.path.Path sessionsPath = TenantRepository.getSessionsPath(tenantName); + SessionZooKeeperClient zkc = new SessionZooKeeperClient(curator, sessionsPath.append(String.valueOf(sessionId))); + zkc.createNewSession(Instant.now()); + if (wait) { + Curator.CompletionWaiter waiter = zkc.getUploadWaiter(); + waiter.awaitCompletion(Duration.ofSeconds(120)); + } + } + + private void assertStatusChange(long sessionId, Session.Status status) throws Exception { + com.yahoo.path.Path statePath = TenantRepository.getSessionsPath(tenantName).append("" + sessionId).append(ConfigCurator.SESSIONSTATE_ZK_SUBPATH); + curator.create(statePath); + curator.framework().setData().forPath(statePath.getAbsolute(), Utf8.toBytes(status.toString())); + assertRemoteSessionStatus(sessionId, status); + } + + private void assertSessionRemoved(long sessionId) { + SessionRepository sessionRepository = tenantRepository.getTenant(tenantName).getSessionRepository(); + waitFor(p -> sessionRepository.getRemoteSession(sessionId) == null, sessionId); + assertNull(sessionRepository.getRemoteSession(sessionId)); + } + + private void assertRemoteSessionExists(long sessionId) { + assertRemoteSessionStatus(sessionId, Session.Status.NEW); + } + + private void assertRemoteSessionStatus(long sessionId, Session.Status status) { + SessionRepository sessionRepository = tenantRepository.getTenant(tenantName).getSessionRepository(); + waitFor(p -> sessionRepository.getRemoteSession(sessionId) != null && + sessionRepository.getRemoteSession(sessionId).getStatus() == status, sessionId); + assertNotNull(sessionRepository.getRemoteSession(sessionId)); + assertThat(sessionRepository.getRemoteSession(sessionId).getStatus(), is(status)); + } + + private void waitFor(LongPredicate predicate, long sessionId) { + long endTime = System.currentTimeMillis() + 60_000; + boolean ok; + do { + ok = predicate.test(sessionId); + try { + Thread.sleep(10); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } while (System.currentTimeMillis() < endTime && !ok); + } + + private long deploy() { + return deploy(applicationId); + } + + private long deploy(ApplicationId applicationId) { + applicationRepository.deploy(testApp, new PrepareParams.Builder().applicationId(applicationId).build()); + return applicationRepository.getActiveSession(applicationId).getSessionId(); + } + +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/userconfigs/a.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/userconfigs/a.cfg deleted file mode 100644 index 0bc17bae65e..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/userconfigs/a.cfg +++ /dev/null @@ -1,18 +0,0 @@ -asyncfetchocc 10 -e 4 -search[2].feeder[1] "bazfeeder" -search[1].feeder[0] "barfeeder1_1" -search[1].feeder[3] "barfeeder2_1" -onlyindef 45 - -speciallog[0].filehandler.rotation "0 1 ..." - -rulebase[4] -rulebase[0].name "cjk" -rulebase[0].rules "# Use unicode equivalents in java source:\n#\n# 佳:\u4f73\n# 能:\u80fd\n# 索:\u7d22\n# 尼:\u5c3c\n# 惠:\u60e0\n# 普:\u666e\n\n@default\n\na索 -> 索a;\n\n[brand] -> brand:[brand];\n\n[brand] :- 索尼,惠普,佳能;\n" -rulebase[1].name "common" -rulebase[1].rules "## Some test rules\n\n# Spelling correction\nbahc -> bach;\n\n# Stopwords\nsomelongstopword -> ;\n[stopword] -> ;\n[stopword] :- someotherlongstopword, yetanotherstopword;\n\n# \n[song] by [artist] -> song:[song] artist:[artist];\n\n[song] :- together, imagine, tinseltown;\n[artist] :- youngbloods, beatles, zappa;\n\n# Negative\nvarious +> -kingz;\n\n\n" -rulebase[2].name "egyik" -rulebase[2].rules "@include(common.sr)\n@automata(/home/vespa/etc/vespa/fsa/stopwords.fsa)\n[stopwords] -> ;\n\n" -rulebase[3].name "masik" -rulebase[3].rules "@include(common.sr)\n[stopwords] :- etaoin, shrdlu;\n[stopwords] -> ;\n\n" diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/userconfigs/b.search#cluster.sports#c0#r0#indexer4.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/userconfigs/b.search#cluster.sports#c0#r0#indexer4.cfg deleted file mode 100644 index 88b50384058..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/userconfigs/b.search#cluster.sports#c0#r0#indexer4.cfg +++ /dev/null @@ -1 +0,0 @@ -usercfgwithid 86 diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/userconfigs/c.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/userconfigs/c.cfg deleted file mode 100644 index b34c4ed311e..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/userconfigs/c.cfg +++ /dev/null @@ -1 +0,0 @@ -foo "test" diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/userconfigs/d.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/userconfigs/d.cfg deleted file mode 100644 index 12c5b53de7d..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/userconfigs/d.cfg +++ /dev/null @@ -1 +0,0 @@ -theint 34 diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/userconfigs/spooler.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/userconfigs/spooler.cfg deleted file mode 100644 index 73ed41667f9..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/userconfigs/spooler.cfg +++ /dev/null @@ -1 +0,0 @@ -keepsuccess true diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFileTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFileTest.java index a73818cda12..a34c17dc909 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFileTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFileTest.java @@ -24,7 +24,7 @@ public class ZKApplicationFileTest extends ApplicationFileTest { private void feed(ConfigCurator zk, File dirToFeed) { assertTrue(dirToFeed.isDirectory()); String appPath = "/0"; - zk.feedZooKeeper(dirToFeed, appPath + ConfigCurator.USERAPP_ZK_SUBPATH, null, true); + ZKApplicationPackageTest.feedZooKeeper(zk, dirToFeed, appPath + ConfigCurator.USERAPP_ZK_SUBPATH, null, true); zk.putData(appPath, ZKApplicationPackage.fileRegistryNode, "dummyfiles"); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java index 4397e087fb7..4b4605cce7d 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java @@ -22,9 +22,12 @@ import org.junit.Test; import org.junit.rules.TemporaryFolder; import java.io.File; +import java.io.FilenameFilter; import java.io.IOException; import java.io.Reader; +import java.util.ArrayList; import java.util.Collections; +import java.util.List; import java.util.Optional; import java.util.regex.Pattern; @@ -37,6 +40,7 @@ import static org.junit.Assert.assertTrue; public class ZKApplicationPackageTest { + private static final FilenameFilter acceptsAllFileNameFilter = (dir, name) -> true; private static final String APP = "src/test/apps/zkapp"; private static final String TEST_FLAVOR_NAME = "test-flavor"; private static final Optional<Flavor> TEST_FLAVOR = new MockNodeFlavors().getFlavor(TEST_FLAVOR_NAME); @@ -97,7 +101,7 @@ public class ZKApplicationPackageTest { private void feed(ConfigCurator zk, File dirToFeed) throws IOException { assertTrue(dirToFeed.isDirectory()); - zk.feedZooKeeper(dirToFeed, "/0" + ConfigCurator.USERAPP_ZK_SUBPATH, null, true); + feedZooKeeper(zk, dirToFeed, "/0" + ConfigCurator.USERAPP_ZK_SUBPATH, null, true); String metaData = "{\"deploy\":{\"user\":\"foo\",\"from\":\"bar\",\"timestamp\":1},\"application\":{\"id\":\"foo:foo:default\",\"checksum\":\"abc\",\"generation\":4,\"previousActiveGeneration\":3}}"; zk.putData("/0", ConfigCurator.META_ZK_PATH, metaData); zk.putData("/0/" + ZKApplicationPackage.fileRegistryNode + "/3.0.0", "dummyfiles"); @@ -115,4 +119,63 @@ public class ZKApplicationPackageTest { } } + /** + * Takes for instance the dir /app and puts the contents into the given ZK path. Ignores files starting with dot, + * and dirs called CVS. + * + * @param dir directory which holds the summary class part files + * @param path zookeeper path + * @param filenameFilter A FilenameFilter which decides which files in dir are fed to zookeeper + * @param recurse recurse subdirectories + */ + static void feedZooKeeper(ConfigCurator zk, File dir, String path, FilenameFilter filenameFilter, boolean recurse) { + try { + if (filenameFilter == null) { + filenameFilter = acceptsAllFileNameFilter; + } + if (!dir.isDirectory()) { + throw new IllegalArgumentException(dir + " is not a directory"); + } + for (File file : listFiles(dir, filenameFilter)) { + if (file.getName().startsWith(".")) continue; //.svn , .git ... + if ("CVS".equals(file.getName())) continue; + if (file.isFile()) { + String contents = IOUtils.readFile(file); + zk.putData(path, file.getName(), contents); + } else if (recurse && file.isDirectory()) { + zk.createNode(path, file.getName()); + feedZooKeeper(zk, file, path + '/' + file.getName(), filenameFilter, recurse); + } + } + } + catch (IOException e) { + throw new RuntimeException("Exception feeding ZooKeeper at path " + path, e); + } + } + + /** + * Same as normal listFiles, but use the filter only for normal files + * + * @param dir directory to list files in + * @param filter A FilenameFilter which decides which files in dir are listed + * @return an array of Files + */ + protected static File[] listFiles(File dir, FilenameFilter filter) { + File[] rawList = dir.listFiles(); + List<File> ret = new ArrayList<>(); + if (rawList != null) { + for (File f : rawList) { + if (f.isDirectory()) { + ret.add(f); + } else { + if (filter.accept(dir, f.getName())) { + ret.add(f); + } + } + } + } + return ret.toArray(new File[0]); + } + + } |