diff options
| author | Harald Musum <musum@verizonmedia.com> | 2023-08-28 15:20:27 +0200 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2023-08-28 15:20:27 +0200 |
| commit | c5d373464251e232c99ba70be72b851b2b5bda7f (patch) | |
| tree | a475b2383fa12774928e83177b2c7fa82c8a5a46 | |
| parent | 8b2d42d659d9e0492a30691979179a6f3239376e (diff) | |
| parent | 12bba1e9531d8237e103a8850969baf99da950df (diff) | |
Merge pull request #28178 from vespa-engine/hmusum/configserver-refactoring-2
Support reading and writing application data as json
31 files changed, 497 insertions, 257 deletions
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index c7e4022c668..138d963b250 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -44,7 +44,7 @@ import com.yahoo.vespa.applicationmodel.InfrastructureApplication; import com.yahoo.vespa.config.server.application.Application; import com.yahoo.vespa.config.server.application.ApplicationCuratorDatabase; import com.yahoo.vespa.config.server.application.ApplicationReindexing; -import com.yahoo.vespa.config.server.application.ApplicationSet; +import com.yahoo.vespa.config.server.application.ApplicationVersions; import com.yahoo.vespa.config.server.application.ClusterReindexing; import com.yahoo.vespa.config.server.application.ClusterReindexingStatusClient; import com.yahoo.vespa.config.server.application.CompressedApplicationInputStream; @@ -658,10 +658,10 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye Tenant tenant = getTenant(applicationId); if (tenant == null) throw new NotFoundException("Tenant '" + applicationId.tenant() + "' not found"); - Optional<ApplicationSet> activeApplicationSet = tenant.getSessionRepository().getActiveApplicationSet(applicationId); - if (activeApplicationSet.isEmpty()) throw new NotFoundException("Unknown application id '" + applicationId + "'"); + Optional<ApplicationVersions> activeApplicationVersions = tenant.getSessionRepository().activeApplicationVersions(applicationId); + if (activeApplicationVersions.isEmpty()) throw new NotFoundException("Unknown application id '" + applicationId + "'"); - return activeApplicationSet.get().getForVersionOrLatest(version, clock.instant()); + return activeApplicationVersions.get().getForVersionOrLatest(version, clock.instant()); } // Will return Optional.empty() if getting application fails (instead of throwing an exception) @@ -706,10 +706,10 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye } public List<Version> getAllVersions(ApplicationId applicationId) { - Optional<ApplicationSet> applicationSet = getActiveApplicationSet(applicationId); + Optional<ApplicationVersions> applicationSet = getActiveApplicationSet(applicationId); return applicationSet.isEmpty() ? List.of() - : applicationSet.get().getAllVersions(applicationId); + : applicationSet.get().versions(applicationId); } public HttpResponse validateSecretStore(ApplicationId applicationId, SystemName systemName, Slime slime) { @@ -1018,8 +1018,8 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye return session; } - public Optional<ApplicationSet> getActiveApplicationSet(ApplicationId appId) { - return getTenant(appId).getSessionRepository().getActiveApplicationSet(appId); + public Optional<ApplicationVersions> getActiveApplicationSet(ApplicationId appId) { + return getTenant(appId).getSessionRepository().activeApplicationVersions(appId); } public Application getActiveApplication(ApplicationId applicationId) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigActivationListener.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigActivationListener.java index e52089f5400..94ff60a29c1 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigActivationListener.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigActivationListener.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.config.server; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.vespa.config.server.application.ApplicationSet; +import com.yahoo.vespa.config.server.application.ApplicationVersions; /** * A ConfigActivationListener is used to signal to a component that config has been @@ -20,7 +20,7 @@ public interface ConfigActivationListener { * * Must be thread-safe. */ - void configActivated(ApplicationSet application); + void configActivated(ApplicationVersions application); /** * Application has been removed. diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java index 16423889d01..aee61fa9a44 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java @@ -11,7 +11,7 @@ import com.yahoo.config.model.api.SuperModelProvider; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.config.GenerationCounter; -import com.yahoo.vespa.config.server.application.ApplicationSet; +import com.yahoo.vespa.config.server.application.ApplicationVersions; import com.yahoo.vespa.config.server.model.SuperModelConfigProvider; import com.yahoo.vespa.flags.FlagSource; @@ -89,10 +89,10 @@ public class SuperModelManager implements SuperModelProvider { } } - public void configActivated(ApplicationSet applicationSet) { + public void configActivated(ApplicationVersions applicationVersions) { synchronized (monitor) { // TODO: Should supermodel care about multiple versions? - ApplicationInfo applicationInfo = applicationSet + ApplicationInfo applicationInfo = applicationVersions .getForVersionOrLatest(Optional.empty(), Instant.now()) .toApplicationInfo(); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelRequestHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelRequestHandler.java index 93bb44e25d3..d43d898f8c3 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelRequestHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelRequestHandler.java @@ -10,7 +10,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.GetConfigRequest; import com.yahoo.vespa.config.protocol.ConfigResponse; -import com.yahoo.vespa.config.server.application.ApplicationSet; +import com.yahoo.vespa.config.server.application.ApplicationVersions; import com.yahoo.vespa.config.server.rpc.ConfigResponseFactory; import java.util.Optional; @@ -46,10 +46,10 @@ public class SuperModelRequestHandler implements RequestHandler { * Signals that config has been activated for an {@link com.yahoo.vespa.config.server.application.Application} * belonging to a tenant. * - * @param applicationSet The activated set of {@link com.yahoo.vespa.config.server.application.Application}. + * @param applicationVersions The activated set of {@link com.yahoo.vespa.config.server.application.Application}s. */ - public synchronized void activateConfig(ApplicationSet applicationSet) { - superModelManager.configActivated(applicationSet); + public synchronized void activateConfig(ApplicationVersions applicationVersions) { + superModelManager.configActivated(applicationVersions); updateHandler(); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationCuratorDatabase.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationCuratorDatabase.java index d757421c23a..70b8dc5f51a 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationCuratorDatabase.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationCuratorDatabase.java @@ -23,10 +23,11 @@ import java.time.Duration; import java.time.Instant; import java.util.List; import java.util.Optional; +import java.util.OptionalLong; import java.util.concurrent.ExecutorService; import java.util.function.UnaryOperator; -import java.util.stream.Collectors; +import static com.yahoo.vespa.curator.transaction.CuratorOperations.setData; import static java.util.stream.Collectors.toUnmodifiableMap; /** @@ -73,13 +74,18 @@ public class ApplicationCuratorDatabase { /** * Creates a node for the given application, marking its existence. */ - public void createApplication(ApplicationId id) { + public void createApplication(ApplicationId id, boolean writeAsJson) { if ( ! id.tenant().equals(tenant)) throw new IllegalArgumentException("Cannot write application id '" + id + "' for tenant '" + tenant + "'"); - try (Lock lock = lock(id)) { - if (curator.exists(applicationPath(id))) return; - curator.create(applicationPath(id)); + try (Lock lock = lock(id)) { + if (writeAsJson) { + var applicationData = new ApplicationData(id, OptionalLong.empty(), OptionalLong.empty()); + curator.set(applicationPath(id), applicationData.toJson()); + } else { + if (curator.exists(applicationPath(id))) return; + curator.create(applicationPath(id)); + } modifyReindexing(id, ApplicationReindexing.empty(), UnaryOperator.identity()); } } @@ -88,10 +94,34 @@ public class ApplicationCuratorDatabase { * Returns a transaction which writes the given session id as the currently active for the given application. * * @param applicationId An {@link ApplicationId} that represents an active application. - * @param sessionId Id of the session containing the application package for this id. + * @param sessionId session id belonging to the application package for this application id. */ - public Transaction createPutTransaction(ApplicationId applicationId, long sessionId) { - return new CuratorTransaction(curator).add(CuratorOperations.setData(applicationPath(applicationId).getAbsolute(), Utf8.toAsciiBytes(sessionId))); + public Transaction createWriteActiveTransaction(Transaction transaction, ApplicationId applicationId, long sessionId, boolean writeAsJson) { + String path = applicationPath(applicationId).getAbsolute(); + return transaction.add(writeAsJson + ? setData(path, new ApplicationData(applicationId, OptionalLong.of(sessionId), OptionalLong.of(sessionId)).toJson()) + : setData(path, Utf8.toAsciiBytes(sessionId))); + } + + /** + * Returns a transaction which writes the given session id as the currently active for the given application. + * + * @param applicationId An {@link ApplicationId} that represents an active application. + * @param sessionId session id belonging to the application package for this application id. + */ + public Transaction createWritePrepareTransaction(Transaction transaction, + ApplicationId applicationId, + long sessionId, + OptionalLong activeSessionId, + boolean writeAsJson) { + + // Needs to read or be supplied current active session id, to avoid overwriting a newer session id. + + String path = applicationPath(applicationId).getAbsolute(); + if (writeAsJson) + return transaction.add(setData(path, new ApplicationData(applicationId, activeSessionId, OptionalLong.of(sessionId)).toJson())); + else + return transaction; // Do nothing, as there is nothing to write in this case } /** @@ -113,6 +143,46 @@ public class ApplicationCuratorDatabase { } /** + * Returns application data for the given application. + * Returns Optional.empty() if application not found or no application data exists. + */ + public Optional<ApplicationData> applicationData(ApplicationId id) { + return applicationData(id, false); + } + + /** + * Returns application data for the given application. + * Returns Optional.empty() if application not found or no application data exists. + */ + public Optional<ApplicationData> applicationData(ApplicationId id, boolean readAsJson) { + Optional<byte[]> data = curator.getData(applicationPath(id)); + if (data.isEmpty() || data.get().length == 0) return Optional.empty(); + + if (readAsJson) { + try { + return Optional.of(ApplicationData.fromBytes(data.get())); + } catch (IllegalArgumentException e) { + return applicationDataOldFormat(id, readAsJson); + } + } else { + return applicationDataOldFormat(id, readAsJson); + } + } + + /** + * Returns application data for the given application. + * Returns Optional.empty() if application not found or no application data exists. + */ + public Optional<ApplicationData> applicationDataOldFormat(ApplicationId id, boolean readAsJson) { + Optional<byte[]> data = curator.getData(applicationPath(id)); + if (data.isEmpty() || data.get().length == 0) return Optional.empty(); + + return Optional.of(new ApplicationData(id, + OptionalLong.of(data.map(bytes -> Long.parseLong(Utf8.toString(bytes))).get()), + OptionalLong.empty())); + } + + /** * List the active applications of a tenant in this config server. * * @return a list of {@link ApplicationId}s that are active. @@ -134,13 +204,11 @@ public class ApplicationCuratorDatabase { curator.set(reindexingDataPath(id), ReindexingStatusSerializer.toBytes(status)); } - /** Sets up a listenable cache with the given listener, over the applications path of this tenant. */ public Curator.DirectoryCache createApplicationsPathCache(ExecutorService zkCacheExecutor) { return curator.createDirectoryCache(applicationsPath.getAbsolute(), false, false, zkCacheExecutor); } - private Path reindexingLockPath(ApplicationId id) { return locksPath.append(id.serializedForm()).append("reindexing"); } @@ -153,16 +221,10 @@ public class ApplicationCuratorDatabase { return applicationsPath.append(id.serializedForm()); } - // Used to determine whether future preparations of this application should use a dedicated CCC. - private Path dedicatedClusterControllerClusterPath(ApplicationId id) { - return applicationPath(id).append("dedicatedClusterControllerCluster"); - } - private Path reindexingDataPath(ApplicationId id) { return applicationPath(id).append("reindexing"); } - private static class ReindexingStatusSerializer { private static final String ENABLED = "enabled"; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationData.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationData.java new file mode 100644 index 00000000000..868ea060fba --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationData.java @@ -0,0 +1,72 @@ +package com.yahoo.vespa.config.server.application; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Slime; +import com.yahoo.slime.SlimeUtils; + +import java.io.IOException; +import java.util.OptionalLong; + +import static com.yahoo.slime.SlimeUtils.optionalLong; + +/** + * Data class for application id, active session and last deployed session + * + * @author hmusum + */ +public class ApplicationData { + + private static final String APPLICATION_ID_FIELD = "applicationId"; + private static final String ACTIVE_SESSION_FIELD = "activeSession"; + private static final String LAST_DEPLOYED_SESSION_FIELD = "lastDeployedSession"; + + private final ApplicationId applicationId; + private final OptionalLong activeSession; + private final OptionalLong lastDeployedSession; + + ApplicationData(ApplicationId applicationId, OptionalLong activeSession, OptionalLong lastDeployedSession) { + this.applicationId = applicationId; + this.activeSession = activeSession; + this.lastDeployedSession = lastDeployedSession; + } + + static ApplicationData fromBytes(byte[] data) { + return fromSlime(SlimeUtils.jsonToSlime(data)); + } + + static ApplicationData fromSlime(Slime slime) { + Cursor cursor = slime.get(); + return new ApplicationData(ApplicationId.fromSerializedForm(cursor.field(APPLICATION_ID_FIELD).asString()), + optionalLong(cursor.field(ACTIVE_SESSION_FIELD)), + optionalLong(cursor.field(LAST_DEPLOYED_SESSION_FIELD))); + } + + public byte[] toJson() { + try { + Slime slime = new Slime(); + toSlime(slime.setObject()); + return SlimeUtils.toJsonBytes(slime); + } catch (IOException e) { + throw new RuntimeException("Serialization of application data to json failed", e); + } + } + + public ApplicationId applicationId() { return applicationId; } + + public OptionalLong activeSession() { return activeSession; } + + public OptionalLong lastDeployedSession() { return lastDeployedSession; } + + @Override + public String toString() { + return "application '" + applicationId + "', active session " + activeSession + ", last deployed session " + lastDeployedSession; + } + + private void toSlime(Cursor object) { + object.setString(APPLICATION_ID_FIELD, applicationId.serializedForm()); + activeSession.ifPresent(session -> object.setLong(ACTIVE_SESSION_FIELD, session)); + lastDeployedSession.ifPresent(session -> object.setLong(LAST_DEPLOYED_SESSION_FIELD, session)); + } + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationMapper.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationMapper.java index c195e5cc8c4..6db01c91dea 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationMapper.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationMapper.java @@ -21,20 +21,20 @@ import java.util.concurrent.ConcurrentHashMap; */ public final class ApplicationMapper { - private final Map<ApplicationId, ApplicationSet> requestHandlers = new ConcurrentHashMap<>(); + private final Map<ApplicationId, ApplicationVersions> requestHandlers = new ConcurrentHashMap<>(); - private ApplicationSet getApplicationSet(ApplicationId applicationId) { - ApplicationSet set = requestHandlers.get(applicationId); - if (set == null) throw new NotFoundException("No such application id: " + applicationId); + private ApplicationVersions applicationVersions(ApplicationId applicationId) { + ApplicationVersions versions = requestHandlers.get(applicationId); + if (versions == null) throw new NotFoundException("No such application id: " + applicationId); - return set; + return versions; } /** - * Register a Application to an application id and specific vespa version + * Register an Application to an application id and specific vespa version */ - public void register(ApplicationId applicationId, ApplicationSet applicationSet) { - requestHandlers.put(applicationId, applicationSet); + public void register(ApplicationId applicationId, ApplicationVersions applicationVersions) { + requestHandlers.put(applicationId, applicationVersions); } /** @@ -45,12 +45,12 @@ public final class ApplicationMapper { } /** - * Retrieve the Application corresponding to this application id and specific vespa version. + * Retrieve the Application corresponding to this application id and specified vespa version. * * @return the matching application, or null if none matches */ public Application getForVersion(ApplicationId applicationId, Optional<Version> vespaVersion, Instant now) throws VersionDoesNotExistException { - return getApplicationSet(applicationId).getForVersionOrLatest(vespaVersion, now); + return applicationVersions(applicationId).getForVersionOrLatest(vespaVersion, now); } /** Returns whether this registry has an application for the given application id */ @@ -80,7 +80,7 @@ public final class ApplicationMapper { } public List<Application> listApplications(ApplicationId applicationId) { - return requestHandlers.get(applicationId).getAllApplications(); + return requestHandlers.get(applicationId).applications(); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationReindexing.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationReindexing.java index f4e1918e6e3..5ecc7e6c8aa 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationReindexing.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationReindexing.java @@ -11,10 +11,6 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; -import static java.util.Objects.requireNonNull; -import static java.util.stream.Collectors.toMap; -import static java.util.stream.Collectors.toUnmodifiableMap; - /** * Pending reindexing: convergence to the stored config generation allows reindexing to start. * Ready reindexing: reindexing may start after this timestamp. diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationSet.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationVersions.java index 5650c2e7e15..71ec91e758f 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationSet.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationVersions.java @@ -16,14 +16,14 @@ import java.util.Optional; * * @author vegard */ -public final class ApplicationSet { +public final class ApplicationVersions { private final Version latestVersion; private final ApplicationId applicationId; private final long generation; private final HashMap<Version, Application> applications = new HashMap<>(); - private ApplicationSet(List<Application> applications) { + private ApplicationVersions(List<Application> applications) { if (applications.isEmpty()) throw new IllegalArgumentException("application list cannot be empty"); Application firstApp = applications.get(0); @@ -44,12 +44,12 @@ public final class ApplicationSet { latestVersion = this.applications.keySet().stream().max(Version::compareTo).get(); } - public static ApplicationSet fromList(List<Application> applications) { - return new ApplicationSet(applications); + public static ApplicationVersions fromList(List<Application> applications) { + return new ApplicationVersions(applications); } // For testing - public static ApplicationSet from(Application application) { + public static ApplicationVersions from(Application application) { return fromList(List.of(application)); } @@ -86,7 +86,7 @@ public final class ApplicationSet { public ApplicationId getId() { return applicationId; } - public Collection<String> getAllHosts() { + public Collection<String> allHosts() { return applications.values().stream() .flatMap(app -> app.getModel().getHosts().stream() .map(HostInfo::getHostname)) @@ -97,15 +97,15 @@ public final class ApplicationSet { applications.values().forEach(app -> app.updateHostMetrics(app.getModel().getHosts().size())); } - public long getApplicationGeneration() { + public long applicationGeneration() { return generation; } - List<Application> getAllApplications() { + List<Application> applications() { return new ArrayList<>(applications.values()); } - public List<Version> getAllVersions(ApplicationId applicationId) { + public List<Version> versions(ApplicationId applicationId) { return applications.values().stream() .filter(application -> application.getId().equals(applicationId)) .map(Application::getVespaVersion) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigInstanceBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigInstanceBuilder.java index d99d9a7e017..920e1862efa 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigInstanceBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigInstanceBuilder.java @@ -53,9 +53,8 @@ class ConfigInstanceBuilder { Field innerField = builder.getClass().getDeclaredField(node.getName()); innerField.setAccessible(true); Object innerFieldVal = innerField.get(builder); - if (innerFieldVal instanceof List) { + if (innerFieldVal instanceof List<?> innerList) { // inner array? Check that list elems are ConfigBuilder - List<?> innerList = (List<?>) innerFieldVal; for (Object b : innerList) { if (b instanceof ConfigBuilder) { applyDef((ConfigBuilder) b, (InnerCNode) node); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigNotConvergedException.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigNotConvergedException.java index 88cddb93d9d..4fe8dc0866c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigNotConvergedException.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigNotConvergedException.java @@ -10,8 +10,4 @@ public class ConfigNotConvergedException extends RuntimeException { super(t); } - public ConfigNotConvergedException(String message) { - super(message); - } - } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/FileDistributionStatus.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/FileDistributionStatus.java index c80faa2375a..ef34d62cef6 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/FileDistributionStatus.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/FileDistributionStatus.java @@ -100,16 +100,9 @@ public class FileDistributionStatus extends AbstractComponent { int countFinished = 0; for (HostStatus hostStatus : hostStatuses) { switch (hostStatus.status) { - case IN_PROGRESS: - countInProgress++; - break; - case FINISHED: - countFinished++; - break; - case UNKNOWN: - countUnknown++; - break; - default: + case IN_PROGRESS -> countInProgress++; + case FINISHED -> countFinished++; + case UNKNOWN -> countUnknown++; } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java index cddcb0f316d..538534d0040 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java @@ -27,7 +27,9 @@ import com.yahoo.vespa.curator.CompletionTimeoutException; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.curator.transaction.CuratorTransaction; +import com.yahoo.vespa.flags.BooleanFlag; import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.ListFlag; import com.yahoo.vespa.flags.PermanentFlags; import org.apache.curator.framework.CuratorFramework; @@ -41,6 +43,7 @@ import java.util.Collection; import java.util.LinkedHashSet; import java.util.List; import java.util.Optional; +import java.util.OptionalLong; import java.util.Set; import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; @@ -76,6 +79,8 @@ public class TenantApplications implements RequestHandler, HostValidator { private final TenantFileSystemDirs tenantFileSystemDirs; private final String serverId; private final ListFlag<String> incompatibleVersions; + private final BooleanFlag writeApplicationDataAsJson; + private final BooleanFlag readApplicationDataAsJson; public TenantApplications(TenantName tenant, Curator curator, StripedExecutor<TenantName> zkWatcherExecutor, ExecutorService zkCacheExecutor, Metrics metrics, ConfigActivationListener configActivationListener, @@ -97,6 +102,8 @@ public class TenantApplications implements RequestHandler, HostValidator { this.clock = clock; this.serverId = configserverConfig.serverId(); this.incompatibleVersions = PermanentFlags.INCOMPATIBLE_VERSIONS.bindTo(flagSource); + this.writeApplicationDataAsJson = Flags.WRITE_APPLICATION_DATA_AS_JSON.bindTo(flagSource); + this.readApplicationDataAsJson = Flags.READ_APPLICATION_DATA_AS_JSON.bindTo(flagSource); } /** The curator backed ZK storage of this. */ @@ -123,6 +130,14 @@ public class TenantApplications implements RequestHandler, HostValidator { return database().activeSessionOf(id); } + /** + * Returns application data for the given application. + * Returns Optional.empty if application not found or no application data exists. + */ + public Optional<ApplicationData> applicationData(ApplicationId id) { + return database().applicationData(id, readApplicationDataAsJson.value()); + } + public boolean sessionExistsInFileSystem(long sessionId) { return Files.exists(Paths.get(tenantFileSystemDirs.sessionsPath().getAbsolutePath(), String.valueOf(sessionId))); } @@ -131,17 +146,34 @@ public class TenantApplications implements RequestHandler, HostValidator { * Returns a transaction which writes the given session id as the currently active for the given application. * * @param applicationId An {@link ApplicationId} that represents an active application. - * @param sessionId Id of the session containing the application package for this id. + * @param sessionId session id belonging to the application package for this application id. + */ + public Transaction createWriteActiveTransaction(Transaction transaction, ApplicationId applicationId, long sessionId) { + return database().createWriteActiveTransaction(transaction, applicationId, sessionId, writeApplicationDataAsJson.value()); + } + + /** + * Returns a transaction which writes the given session id as the last deployed for the given application. + * + * @param applicationId An {@link ApplicationId} that represents an active application. + * @param sessionId session id belonging to the application package for this application id. */ - public Transaction createPutTransaction(ApplicationId applicationId, long sessionId) { - return database().createPutTransaction(applicationId, sessionId); + public Transaction createWritePrepareTransaction(Transaction transaction, + ApplicationId applicationId, + long sessionId, + Optional<Long> activeSessionId) { + return database().createWritePrepareTransaction(transaction, + applicationId, + sessionId, + activeSessionId.map(OptionalLong::of).orElseGet(OptionalLong::empty), + writeApplicationDataAsJson.value()); } /** * Creates a node for the given application, marking its existence. */ public void createApplication(ApplicationId id) { - database().createApplication(id); + database().createApplication(id, writeApplicationDataAsJson.value()); } /** @@ -215,29 +247,29 @@ public class TenantApplications implements RequestHandler, HostValidator { return application.resolveConfig(req, responseFactory); } - private void notifyConfigActivationListeners(ApplicationSet applicationSet) { - List<Application> applications = applicationSet.getAllApplications(); + private void notifyConfigActivationListeners(ApplicationVersions applicationVersions) { + List<Application> applications = applicationVersions.applications(); if (applications.isEmpty()) throw new IllegalArgumentException("application set cannot be empty"); - hostRegistry.update(applications.get(0).getId(), applicationSet.getAllHosts()); - configActivationListener.configActivated(applicationSet); + hostRegistry.update(applications.get(0).getId(), applicationVersions.allHosts()); + configActivationListener.configActivated(applicationVersions); } /** * Activates the config of the given app. Notifies listeners * - * @param applicationSet the {@link ApplicationSet} to be activated + * @param applicationVersions the {@link ApplicationVersions} to be activated */ - public void activateApplication(ApplicationSet applicationSet, long activeSessionId) { - ApplicationId id = applicationSet.getId(); + public void activateApplication(ApplicationVersions applicationVersions, long activeSessionId) { + ApplicationId id = applicationVersions.getId(); try (@SuppressWarnings("unused") Lock lock = lock(id)) { if ( ! exists(id)) return; // Application was deleted before activation. - if (applicationSet.getApplicationGeneration() != activeSessionId) + if (applicationVersions.applicationGeneration() != activeSessionId) return; // Application activated a new session before we got here. - setActiveApp(applicationSet); - notifyConfigActivationListeners(applicationSet); + setActiveApp(applicationVersions); + notifyConfigActivationListeners(applicationVersions); } } @@ -281,13 +313,13 @@ public class TenantApplications implements RequestHandler, HostValidator { configActivationListener.applicationRemoved(applicationId); } - private void setActiveApp(ApplicationSet applicationSet) { - ApplicationId applicationId = applicationSet.getId(); - Collection<String> hostsForApp = applicationSet.getAllHosts(); + private void setActiveApp(ApplicationVersions applicationVersions) { + ApplicationId applicationId = applicationVersions.getId(); + Collection<String> hostsForApp = applicationVersions.allHosts(); hostRegistry.update(applicationId, hostsForApp); - applicationSet.updateHostMetrics(); + applicationVersions.updateHostMetrics(); tenantMetricUpdater.setApplications(applicationMapper.numApplications()); - applicationMapper.register(applicationId, applicationSet); + applicationMapper.register(applicationId, applicationVersions); } @Override diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java index 0e45d42efcf..328bd143d81 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java @@ -19,7 +19,7 @@ import com.yahoo.container.jdisc.secretstore.SecretStore; import com.yahoo.vespa.config.server.ServerCache; import com.yahoo.vespa.config.server.application.Application; import com.yahoo.vespa.config.server.application.ApplicationCuratorDatabase; -import com.yahoo.vespa.config.server.application.ApplicationSet; +import com.yahoo.vespa.config.server.application.ApplicationVersions; import com.yahoo.vespa.config.server.deploy.ModelContextImpl; import com.yahoo.vespa.config.server.monitoring.MetricUpdater; import com.yahoo.vespa.config.server.monitoring.Metrics; @@ -51,7 +51,7 @@ public class ActivatedModelsBuilder extends ModelsBuilder<Application> { private final TenantName tenant; private final long applicationGeneration; private final SessionZooKeeperClient zkClient; - private final Optional<ApplicationSet> currentActiveApplicationSet; + private final Optional<ApplicationVersions> activeApplicationVersions; private final ConfigDefinitionRepo configDefinitionRepo; private final Metrics metrics; private final Curator curator; @@ -62,7 +62,7 @@ public class ActivatedModelsBuilder extends ModelsBuilder<Application> { public ActivatedModelsBuilder(TenantName tenant, long applicationGeneration, SessionZooKeeperClient zkClient, - Optional<ApplicationSet> currentActiveApplicationSet, + Optional<ApplicationVersions> activeApplicationVersions, ExecutorService executor, Curator curator, Metrics metrics, @@ -77,7 +77,7 @@ public class ActivatedModelsBuilder extends ModelsBuilder<Application> { this.tenant = tenant; this.applicationGeneration = applicationGeneration; this.zkClient = zkClient; - this.currentActiveApplicationSet = currentActiveApplicationSet; + this.activeApplicationVersions = activeApplicationVersions; this.configDefinitionRepo = configDefinitionRepo; this.metrics = metrics; this.curator = curator; @@ -122,8 +122,8 @@ public class ActivatedModelsBuilder extends ModelsBuilder<Application> { } private Optional<Model> modelOf(Version version) { - if (currentActiveApplicationSet.isEmpty()) return Optional.empty(); - return currentActiveApplicationSet.get().get(version).map(Application::getModel); + if (activeApplicationVersions.isEmpty()) return Optional.empty(); + return activeApplicationVersions.get().get(version).map(Application::getModel); } private static <T> Optional<T> getForVersionOrLatest(Map<Version, T> map, Version version) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java index a89ba88bfbe..af611b131f6 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java @@ -29,14 +29,13 @@ import com.yahoo.config.provision.Zone; import com.yahoo.container.jdisc.secretstore.SecretStore; import com.yahoo.vespa.config.server.application.Application; import com.yahoo.vespa.config.server.application.ApplicationCuratorDatabase; -import com.yahoo.vespa.config.server.application.ApplicationSet; +import com.yahoo.vespa.config.server.application.ApplicationVersions; import com.yahoo.vespa.config.server.deploy.ModelContextImpl; import com.yahoo.vespa.config.server.host.HostValidator; import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; import com.yahoo.vespa.config.server.session.PrepareParams; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.flags.FlagSource; -import com.yahoo.yolean.Exceptions; import java.io.File; import java.io.IOException; @@ -67,7 +66,7 @@ public class PreparedModelsBuilder extends ModelsBuilder<PreparedModelsBuilder.P private final HostValidator hostValidator; private final PrepareParams params; private final FileRegistry fileRegistry; - private final Optional<ApplicationSet> currentActiveApplicationSet; + private final Optional<ApplicationVersions> activeApplicationVersions; private final Curator curator; private final ExecutorService executor; @@ -84,7 +83,7 @@ public class PreparedModelsBuilder extends ModelsBuilder<PreparedModelsBuilder.P HostValidator hostValidator, DeployLogger deployLogger, PrepareParams params, - Optional<ApplicationSet> currentActiveApplicationSet, + Optional<ApplicationVersions> activeApplicationVersions, ConfigserverConfig configserverConfig, Zone zone) { super(modelFactoryRegistry, configserverConfig, zone, hostProvisionerProvider, deployLogger); @@ -97,7 +96,7 @@ public class PreparedModelsBuilder extends ModelsBuilder<PreparedModelsBuilder.P this.hostValidator = hostValidator; this.curator = curator; this.params = params; - this.currentActiveApplicationSet = currentActiveApplicationSet; + this.activeApplicationVersions = activeApplicationVersions; this.executor = executor; } @@ -149,8 +148,8 @@ public class PreparedModelsBuilder extends ModelsBuilder<PreparedModelsBuilder.P } private Optional<Model> modelOf(Version version) { - if (currentActiveApplicationSet.isEmpty()) return Optional.empty(); - return currentActiveApplicationSet.get().get(version).map(Application::getModel); + if (activeApplicationVersions.isEmpty()) return Optional.empty(); + return activeApplicationVersions.get().get(version).map(Application::getModel); } private HostProvisioner createHostProvisioner(ApplicationPackage applicationPackage, Provisioned provisioned) { @@ -213,7 +212,7 @@ public class PreparedModelsBuilder extends ModelsBuilder<PreparedModelsBuilder.P zone(), Set.copyOf(containerEndpoints), params.isBootstrap(), - currentActiveApplicationSet.isEmpty(), + activeApplicationVersions.isEmpty(), LegacyFlags.from(applicationPackage, flagSource), endpointCertificateSecrets, params.athenzDomain(), diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java index d26a22284c0..78a8cda7c34 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java @@ -31,7 +31,7 @@ import com.yahoo.vespa.config.server.ConfigActivationListener; import com.yahoo.vespa.config.server.GetConfigContext; import com.yahoo.vespa.config.server.RequestHandler; import com.yahoo.vespa.config.server.SuperModelRequestHandler; -import com.yahoo.vespa.config.server.application.ApplicationSet; +import com.yahoo.vespa.config.server.application.ApplicationVersions; import com.yahoo.vespa.config.server.filedistribution.FileServer; import com.yahoo.vespa.config.server.host.HostRegistry; import com.yahoo.vespa.config.server.monitoring.MetricUpdater; @@ -44,6 +44,7 @@ import com.yahoo.vespa.filedistribution.FileDownloader; import com.yahoo.vespa.filedistribution.FileReceiver; import com.yahoo.vespa.filedistribution.FileReferenceData; import com.yahoo.vespa.filedistribution.FileReferenceDownload; + import java.nio.ByteBuffer; import java.time.Duration; import java.util.Arrays; @@ -61,12 +62,13 @@ import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; -import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; import java.util.stream.Stream; import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType; +import static java.util.logging.Level.FINE; +import static java.util.logging.Level.WARNING; /** * An RPC server class that handles the config protocol RPC method "getConfigV3". @@ -77,8 +79,6 @@ import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType // TODO: Split business logic out of this public class RpcServer implements Runnable, ConfigActivationListener, TenantListener { - static final String getConfigMethodName = "getConfigV3"; - private static final int TRACELEVEL = 6; static final int TRACELEVEL_DEBUG = 9; private static final String THREADPOOL_NAME = "rpcserver worker pool"; @@ -158,9 +158,6 @@ public class RpcServer implements Runnable, ConfigActivationListener, TenantList * Uses the template pattern to call methods in classes that extend RpcServer. */ private void getConfigV3(Request req) { - if (log.isLoggable(Level.FINEST)) { - log.log(Level.FINEST, getConfigMethodName); - } req.detach(); rpcAuthorizer.authorizeConfigRequest(req) .thenRun(() -> addToRequestQueue(JRTServerConfigRequestV3.createFromRequest(req))); @@ -184,7 +181,7 @@ public class RpcServer implements Runnable, ConfigActivationListener, TenantList @Override public void run() { - log.log(Level.FINE, "Rpc server will listen on port " + spec.port()); + log.log(FINE, "Rpc server will listen on port " + spec.port()); try { Acceptor acceptor = supervisor.listen(spec); isRunning = true; @@ -260,25 +257,23 @@ public class RpcServer implements Runnable, ConfigActivationListener, TenantList * This method should be called when config is activated in the server. */ @Override - public void configActivated(ApplicationSet applicationSet) { - ApplicationId applicationId = applicationSet.getId(); + public void configActivated(ApplicationVersions applicationVersions) { + ApplicationId applicationId = applicationVersions.getId(); ApplicationState state = getState(applicationId); - state.setActiveGeneration(applicationSet.getApplicationGeneration()); - reloadSuperModel(applicationSet); + state.setActiveGeneration(applicationVersions.applicationGeneration()); + reloadSuperModel(applicationVersions); configActivated(applicationId); } - private void reloadSuperModel(ApplicationSet applicationSet) { - superModelRequestHandler.activateConfig(applicationSet); + private void reloadSuperModel(ApplicationVersions applicationVersions) { + superModelRequestHandler.activateConfig(applicationVersions); configActivated(ApplicationId.global()); } void configActivated(ApplicationId applicationId) { List<DelayedConfigResponses.DelayedConfigResponse> responses = delayedConfigResponses.drainQueue(applicationId); String logPre = TenantRepository.logPre(applicationId); - if (log.isLoggable(Level.FINE)) { - log.log(Level.FINE, logPre + "Start of configActivated: " + responses.size() + " requests on delayed requests queue"); - } + log.log(FINE, () -> logPre + "Start of configActivated: " + responses.size() + " requests on delayed requests queue"); int responsesSent = 0; CompletionService<Boolean> completionService = new ExecutorCompletionService<>(executorService); while (!responses.isEmpty()) { @@ -287,15 +282,13 @@ public class RpcServer implements Runnable, ConfigActivationListener, TenantList // Doing cancel here deals with the case where the timer is already running or has not run, so // there is no need for any extra check. if (delayedConfigResponse.cancel()) { - if (log.isLoggable(Level.FINE)) { - logRequestDebug(Level.FINE, logPre + "Timer cancelled for ", delayedConfigResponse.request); - } + log.log(FINE, () -> logPre + "Timer cancelled for " + delayedConfigResponse.request); // Do not wait for this request if we were unable to execute if (addToRequestQueue(delayedConfigResponse.request, false, completionService)) { responsesSent++; } } else { - log.log(Level.FINE, () -> logPre + "Timer already cancelled or finished or never scheduled"); + log.log(FINE, () -> logPre + "Timer already cancelled or finished or never scheduled"); } } @@ -306,15 +299,6 @@ public class RpcServer implements Runnable, ConfigActivationListener, TenantList Thread.currentThread().interrupt(); } } - - if (log.isLoggable(Level.FINE)) - log.log(Level.FINE, logPre + "Finished activating " + responsesSent + " requests"); - } - - private void logRequestDebug(Level level, String message, JRTServerConfigRequest request) { - if (log.isLoggable(level)) { - log.log(level, message + request.getShortDescription()); - } } @Override @@ -325,9 +309,7 @@ public class RpcServer implements Runnable, ConfigActivationListener, TenantList } public void respond(JRTServerConfigRequest request) { - if (log.isLoggable(Level.FINE)) { - log.log(Level.FINE, "Trace at request return:\n" + request.getRequestTrace().toString()); - } + log.log(FINE, () -> "Trace when responding:\n" + request.getRequestTrace().toString()); request.getRequest().returnRequest(); } @@ -344,7 +326,7 @@ public class RpcServer implements Runnable, ConfigActivationListener, TenantList if (GetConfigProcessor.logDebug(trace)) { String message = "Did not find tenant for host '" + hostname + "', using " + TenantName.defaultName() + ". Hosts in host registry: " + hostRegistry.getAllHosts(); - log.log(Level.FINE, () -> message); + log.log(FINE, () -> message); trace.trace(6, message); } return Optional.empty(); @@ -368,7 +350,6 @@ public class RpcServer implements Runnable, ConfigActivationListener, TenantList public Boolean addToRequestQueue(JRTServerConfigRequest request, boolean forceResponse, CompletionService<Boolean> completionService) { // It's no longer delayed if we get here request.setDelayedResponse(false); - //ConfigDebug.logDebug(log, System.currentTimeMillis(), request.getConfigKey(), "RpcServer.addToRequestQueue()"); try { final GetConfigProcessor task = new GetConfigProcessor(this, request, forceResponse); if (completionService == null) { @@ -405,7 +386,7 @@ public class RpcServer implements Runnable, ConfigActivationListener, TenantList "'. Request from host '" + request.getClientHostName() + "'"; metrics.incUnknownHostRequests(); trace.trace(TRACELEVEL, msg); - log.log(Level.WARNING, msg); + log.log(WARNING, msg); return GetConfigContext.empty(); } RequestHandler handler = requestHandler.get(); @@ -430,7 +411,7 @@ public class RpcServer implements Runnable, ConfigActivationListener, TenantList @Override public void onTenantDelete(TenantName tenant) { - log.log(Level.FINE, () -> TenantRepository.logPre(tenant) + + log.log(FINE, () -> TenantRepository.logPre(tenant) + "Tenant deleted, removing request handler and cleaning host registry"); tenants.remove(tenant); } @@ -500,7 +481,8 @@ public class RpcServer implements Runnable, ConfigActivationListener, TenantList Request request = createMetaRequest(fileData); invokeRpcIfValidConnection(request); if (request.isError()) { - log.warning("Failed delivering meta for reference '" + fileData.fileReference().value() + "' with file '" + fileData.filename() + "' to " + + log.log(WARNING, () -> "Failed delivering meta for reference '" + fileData.fileReference().value() + + "' with file '" + fileData.filename() + "' to " + target.toString() + " with error: '" + request.errorMessage() + "'."); return 1; } else { @@ -519,6 +501,7 @@ public class RpcServer implements Runnable, ConfigActivationListener, TenantList request.parameters().add(new StringValue(fileData.type().name())); request.parameters().add(new Int64Value(fileData.size())); // Only add parameter if not gzip, this is default and old clients will not handle the extra parameter + // TODO Always add parameter in Vespa 9 if (fileData.compressionType() != CompressionType.gzip) request.parameters().add(new StringValue(fileData.compressionType().name())); return request; @@ -584,7 +567,6 @@ public class RpcServer implements Runnable, ConfigActivationListener, TenantList acceptedCompressionTypes = Arrays.stream(request.parameters().get(2).asStringArray()) .map(CompressionType::valueOf) .collect(Collectors.toSet()); - log.log(Level.FINE, "acceptedCompressionTypes=" + acceptedCompressionTypes); fileServer.serveFile(reference, downloadFromOtherSourceIfNotFound, acceptedCompressionTypes, request, receiver); }); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java index 12c61272f20..aa6d33fbda8 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.config.server.session; import com.yahoo.config.provision.TenantName; -import com.yahoo.vespa.config.server.application.ApplicationSet; +import com.yahoo.vespa.config.server.application.ApplicationVersions; import java.util.Objects; import java.util.Optional; @@ -15,7 +15,7 @@ import java.util.Optional; */ public class RemoteSession extends Session { - private final Optional<ApplicationSet> applicationSet; + private final Optional<ApplicationVersions> applicationVersions; /** * Creates a session. This involves loading the application, validating it and distributing it. @@ -36,17 +36,17 @@ public class RemoteSession extends Session { * @param zooKeeperClient a SessionZooKeeperClient instance * @param applicationSet current application set for this session */ - RemoteSession(TenantName tenant, long sessionId, SessionZooKeeperClient zooKeeperClient, Optional<ApplicationSet> applicationSet) { + RemoteSession(TenantName tenant, long sessionId, SessionZooKeeperClient zooKeeperClient, Optional<ApplicationVersions> applicationSet) { super(tenant, sessionId, zooKeeperClient); - this.applicationSet = applicationSet; + this.applicationVersions = applicationSet; } @Override - public Optional<ApplicationSet> applicationSet() { return applicationSet; } + public Optional<ApplicationVersions> applicationVersions() { return applicationVersions; } - public synchronized RemoteSession activated(ApplicationSet applicationSet) { - Objects.requireNonNull(applicationSet, "applicationSet cannot be null"); - return new RemoteSession(tenant, sessionId, sessionZooKeeperClient, Optional.of(applicationSet)); + public synchronized RemoteSession activated(ApplicationVersions applicationVersions) { + Objects.requireNonNull(applicationVersions, "applicationVersions cannot be null"); + return new RemoteSession(tenant, sessionId, sessionZooKeeperClient, Optional.of(applicationVersions)); } public synchronized RemoteSession deactivated() { @@ -55,7 +55,7 @@ public class RemoteSession extends Session { @Override public String toString() { - return super.toString() + ",application set=" + applicationSet; + return super.toString() + ",application set=" + applicationVersions; } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java index eb359f9ffc6..f354b5238b2 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java @@ -17,7 +17,7 @@ import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.TenantName; import com.yahoo.path.Path; import com.yahoo.transaction.Transaction; -import com.yahoo.vespa.config.server.application.ApplicationSet; +import com.yahoo.vespa.config.server.application.ApplicationVersions; import com.yahoo.vespa.config.server.tenant.TenantRepository; import java.security.cert.X509Certificate; import java.time.Instant; @@ -184,7 +184,7 @@ public abstract class Session implements Comparable<Session> { return getApplicationPackage().getFile(relativePath); } - Optional<ApplicationSet> applicationSet() { return Optional.empty(); } + Optional<ApplicationVersions> applicationVersions() { return Optional.empty(); } private void markSessionEdited() { setStatus(Session.Status.NEW); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java index 6c0120eb337..aeff97169f4 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java @@ -34,7 +34,7 @@ import com.yahoo.net.HostName; import com.yahoo.path.Path; import com.yahoo.vespa.config.server.ConfigServerSpec; import com.yahoo.vespa.config.server.TimeoutBudget; -import com.yahoo.vespa.config.server.application.ApplicationSet; +import com.yahoo.vespa.config.server.application.ApplicationVersions; import com.yahoo.vespa.config.server.configchange.ConfigChangeActions; import com.yahoo.vespa.config.server.deploy.ZooKeeperDeployer; import com.yahoo.vespa.config.server.filedistribution.FileDistributionFactory; @@ -125,14 +125,14 @@ public class SessionPreparer { * @param hostValidator host validator * @param logger for storing logs returned in response to client. * @param params parameters controlling behaviour of prepare. - * @param activeApplicationSet set of currently active applications. + * @param activeApplicationVersions active application versions. * @return the config change actions that must be done to handle the activation of the models prepared. */ public PrepareResult prepare(HostValidator hostValidator, DeployLogger logger, PrepareParams params, - Optional<ApplicationSet> activeApplicationSet, Instant now, File serverDbSessionDir, + Optional<ApplicationVersions> activeApplicationVersions, Instant now, File serverDbSessionDir, ApplicationPackage applicationPackage, SessionZooKeeperClient sessionZooKeeperClient) { ApplicationId applicationId = params.getApplicationId(); - Preparation preparation = new Preparation(hostValidator, logger, params, activeApplicationSet, + Preparation preparation = new Preparation(hostValidator, logger, params, activeApplicationVersions, TenantRepository.getTenantPath(applicationId.tenant()), serverDbSessionDir, applicationPackage, sessionZooKeeperClient); preparation.preprocess(); @@ -184,7 +184,7 @@ public class SessionPreparer { private final FileRegistry fileRegistry; Preparation(HostValidator hostValidator, DeployLogger logger, PrepareParams params, - Optional<ApplicationSet> currentActiveApplicationSet, Path tenantPath, + Optional<ApplicationVersions> activeApplicationVersions, Path tenantPath, File serverDbSessionDir, ApplicationPackage applicationPackage, SessionZooKeeperClient sessionZooKeeperClient) { this.logger = logger; @@ -217,7 +217,7 @@ public class SessionPreparer { hostValidator, logger, params, - currentActiveApplicationSet, + activeApplicationVersions, configserverConfig, zone); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java index eb8f78036ef..44a656a1579 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java @@ -22,7 +22,7 @@ import com.yahoo.transaction.NestedTransaction; import com.yahoo.transaction.Transaction; import com.yahoo.vespa.config.server.ConfigServerDB; import com.yahoo.vespa.config.server.TimeoutBudget; -import com.yahoo.vespa.config.server.application.ApplicationSet; +import com.yahoo.vespa.config.server.application.ApplicationVersions; import com.yahoo.vespa.config.server.application.TenantApplications; import com.yahoo.vespa.config.server.configchange.ConfigChangeActions; import com.yahoo.vespa.config.server.deploy.TenantFileSystemDirs; @@ -39,6 +39,7 @@ import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.vespa.config.server.zookeeper.SessionCounter; import com.yahoo.vespa.config.server.zookeeper.ZKApplication; import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.curator.transaction.CuratorTransaction; import com.yahoo.vespa.flags.BooleanFlag; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.flags.Flags; @@ -239,16 +240,24 @@ public class SessionRepository { throw new UnknownVespaVersionException("Vespa version '" + version + "' not known by this config server"); }); - applicationRepo.createApplication(params.getApplicationId()); // TODO jvenstad: This is wrong, but it has to be done now, since preparation can change the application ID of a session :( - logger.log(Level.FINE, "Created application " + params.getApplicationId()); + ApplicationId applicationId = params.getApplicationId(); + applicationRepo.createApplication(applicationId); // TODO jvenstad: This is wrong, but it has to be done now, since preparation can change the application ID of a session :( + logger.log(Level.FINE, "Created application " + applicationId); long sessionId = session.getSessionId(); SessionZooKeeperClient sessionZooKeeperClient = createSessionZooKeeperClient(sessionId); Optional<CompletionWaiter> waiter = params.isDryRun() ? Optional.empty() : Optional.of(sessionZooKeeperClient.createPrepareWaiter()); - Optional<ApplicationSet> activeApplicationSet = getActiveApplicationSet(params.getApplicationId()); + Optional<ApplicationVersions> activeApplicationVersions = activeApplicationVersions(applicationId); + try (var transaction = new CuratorTransaction(curator)) { + applicationRepo.createWritePrepareTransaction(transaction, + applicationId, + sessionId, + getActiveSessionId(applicationId)) + .commit(); + } ConfigChangeActions actions = sessionPreparer.prepare(applicationRepo, logger, params, - activeApplicationSet, now, getSessionAppDir(sessionId), + activeApplicationVersions, now, getSessionAppDir(sessionId), session.getApplicationPackage(), sessionZooKeeperClient) .getConfigChangeActions(); setPrepared(session); @@ -277,6 +286,7 @@ public class SessionRepository { timeoutBudget, deployLogger, created); + applicationRepo.createApplication(applicationId); write(existingSession, session, applicationId, created); return session; } @@ -293,9 +303,10 @@ public class SessionRepository { ApplicationId applicationId, TimeoutBudget timeoutBudget, DeployLogger deployLogger) { - applicationRepo.createApplication(applicationId); - return createSessionFromApplication(applicationDirectory, applicationId, false, timeoutBudget, + LocalSession session = createSessionFromApplication(applicationDirectory, applicationId, false, timeoutBudget, deployLogger, clock.instant()); + applicationRepo.createApplication(applicationId); + return session; } /** @@ -479,20 +490,20 @@ public class SessionRepository { notifyCompletion(waiter); } - public ApplicationSet ensureApplicationLoaded(RemoteSession session) { - if (session.applicationSet().isPresent()) { - return session.applicationSet().get(); + public ApplicationVersions ensureApplicationLoaded(RemoteSession session) { + if (session.applicationVersions().isPresent()) { + return session.applicationVersions().get(); } Optional<Long> activeSessionId = getActiveSessionId(session.getApplicationId()); - Optional<ApplicationSet> previousApplicationSet = activeSessionId.filter(session::isNewerThan) - .flatMap(this::getApplicationSet); - ApplicationSet applicationSet = loadApplication(session, previousApplicationSet); - RemoteSession activated = session.activated(applicationSet); + Optional<ApplicationVersions> previousActiveApplicationVersions = activeSessionId.filter(session::isNewerThan) + .flatMap(this::activeApplicationVersions); + ApplicationVersions applicationVersions = loadApplication(session, previousActiveApplicationVersions); + RemoteSession activated = session.activated(applicationVersions); long sessionId = activated.getSessionId(); remoteSessionCache.put(sessionId, activated); updateSessionStateWatcher(sessionId); - return applicationSet; + return applicationVersions; } void confirmUpload(Session session) { @@ -526,7 +537,7 @@ public class SessionRepository { } } - private ApplicationSet loadApplication(Session session, Optional<ApplicationSet> previousApplicationSet) { + private ApplicationVersions loadApplication(Session session, Optional<ApplicationVersions> previousApplicationSet) { log.log(Level.FINE, () -> "Loading application for " + session); SessionZooKeeperClient sessionZooKeeperClient = createSessionZooKeeperClient(session.getSessionId()); ActivatedModelsBuilder builder = new ActivatedModelsBuilder(session.getTenantName(), @@ -543,12 +554,12 @@ public class SessionRepository { zone, modelFactoryRegistry, configDefinitionRepo); - return ApplicationSet.fromList(builder.buildModels(session.getApplicationId(), - session.getDockerImageRepository(), - session.getVespaVersion(), - sessionZooKeeperClient.loadApplicationPackage(), - new AllocatedHostsFromAllModels(), - clock.instant())); + return ApplicationVersions.fromList(builder.buildModels(session.getApplicationId(), + session.getDockerImageRepository(), + session.getVespaVersion(), + sessionZooKeeperClient.loadApplicationPackage(), + new AllocatedHostsFromAllModels(), + clock.instant())); } private void nodeChanged() { @@ -775,11 +786,11 @@ public class SessionRepository { } } - public Optional<ApplicationSet> getActiveApplicationSet(ApplicationId appId) { - return applicationRepo.activeSessionOf(appId).flatMap(this::getApplicationSet); + public Optional<ApplicationVersions> activeApplicationVersions(ApplicationId appId) { + return applicationRepo.activeSessionOf(appId).flatMap(this::activeApplicationVersions); } - private Optional<ApplicationSet> getApplicationSet(long sessionId) { + private Optional<ApplicationVersions> activeApplicationVersions(long sessionId) { try { return Optional.ofNullable(getRemoteSession(sessionId)).map(this::ensureApplicationLoaded); } catch (IllegalArgumentException e) { @@ -974,7 +985,7 @@ public class SessionRepository { public Transaction createActivateTransaction(Session session) { Transaction transaction = createSetStatusTransaction(session, Session.Status.ACTIVATE); - transaction.add(applicationRepo.createPutTransaction(session.getApplicationId(), session.getSessionId()).operations()); + transaction.add(applicationRepo.createWriteActiveTransaction(transaction, session.getApplicationId(), session.getSessionId()).operations()); return transaction; } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java index 950a21e5750..c355be5090a 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java @@ -326,7 +326,7 @@ public class ApplicationRepositoryTest { // Delete app and verify that it has been deleted from repos and no application set exists assertTrue(applicationRepository.delete(applicationId())); assertTrue(applicationRepository.getActiveSession(applicationId()).isEmpty()); - assertEquals(Optional.empty(), sessionRepository.getRemoteSession(sessionId).applicationSet()); + assertEquals(Optional.empty(), sessionRepository.getRemoteSession(sessionId).applicationVersions()); assertTrue(curator.exists(sessionNode)); assertEquals(Session.Status.DELETE.name(), Utf8.toString(curator.getData(sessionNode.append("sessionState")).get())); assertTrue(sessionFile.exists()); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelRequestHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelRequestHandlerTest.java index c3766ad9b83..1b4e9ad1231 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelRequestHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelRequestHandlerTest.java @@ -7,7 +7,7 @@ import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.config.server.application.Application; -import com.yahoo.vespa.config.server.application.ApplicationSet; +import com.yahoo.vespa.config.server.application.ApplicationVersions; import com.yahoo.vespa.config.server.monitoring.MetricUpdater; import com.yahoo.vespa.curator.mock.MockCurator; import com.yahoo.vespa.flags.InMemoryFlagSource; @@ -112,8 +112,8 @@ public class SuperModelRequestHandlerTest { assertTrue(controller.hasApplication(ApplicationId.global(), Optional.empty())); } - private ApplicationSet createApp(ApplicationId applicationId, long generation) throws IOException, SAXException { - return ApplicationSet.from( + private ApplicationVersions createApp(ApplicationId applicationId, long generation) throws IOException, SAXException { + return ApplicationVersions.from( new TestApplication( new VespaModel(FilesApplicationPackage.fromFile(testApp)), new ServerCache(), diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationCuratorDatabaseTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationCuratorDatabaseTest.java index cbdb462c35e..8d1b22c94c5 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationCuratorDatabaseTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationCuratorDatabaseTest.java @@ -3,13 +3,17 @@ package com.yahoo.vespa.config.server.application; import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.curator.transaction.CuratorTransaction; import org.junit.Test; import java.time.Instant; import java.util.Optional; +import java.util.OptionalLong; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; /** * @author jonmv @@ -35,4 +39,72 @@ public class ApplicationCuratorDatabaseTest { assertEquals(reindexing, db.readReindexingStatus(id).orElseThrow()); } + @Test + public void testReadingAndWritingApplicationData() { + ApplicationId id = ApplicationId.defaultId(); + MockCurator curator = new MockCurator(); + ApplicationCuratorDatabase db = new ApplicationCuratorDatabase(id.tenant(), curator); + + assertEquals(Optional.empty(), db.applicationData(id)); + + db.createApplication(id, false); + assertEquals(Optional.empty(), db.applicationData(id)); // still empty, as no data has been written to node + + db.createApplication(id, true); + try { + Optional<ApplicationData> applicationData = db.applicationData(id); + fail("Expected exception, got " + applicationData); + } catch (NumberFormatException e) { + // expected + } + + // Can be read as json, but no active session or last deployed session + Optional<ApplicationData> applicationData = db.applicationData(id, true); + assertTrue(applicationData.isPresent()); + assertEquals(id, applicationData.get().applicationId()); + assertFalse(applicationData.get().activeSession().isPresent()); + assertFalse(applicationData.get().lastDeployedSession().isPresent()); + + // Prepare session 2, no active session + try (var t = db.createWritePrepareTransaction(new CuratorTransaction(curator), id, 2, OptionalLong.empty(), false)) { + t.commit(); + } + // Activate session 2, last deployed session not present (not writing json) + try (var t = db.createWriteActiveTransaction(new CuratorTransaction(curator), id, 2, false)) { + t.commit(); + } + // Can be read as session id only + applicationData = db.applicationData(id, false); + assertTrue(applicationData.isPresent()); + assertEquals(id, applicationData.get().applicationId()); + assertTrue(applicationData.get().activeSession().isPresent()); + assertEquals(2, applicationData.get().activeSession().getAsLong()); + assertFalse(applicationData.get().lastDeployedSession().isPresent()); + + // Prepare session 3, last deployed session is still 2 + try (var t = db.createWritePrepareTransaction(new CuratorTransaction(curator), id, 3, OptionalLong.of(2), true)) { + t.commit(); + } + // Can be read as json, active session is still 2 and last deployed session is 3 + applicationData = db.applicationData(id, true); + assertTrue(applicationData.isPresent()); + assertEquals(id, applicationData.get().applicationId()); + assertTrue(applicationData.get().activeSession().isPresent()); + assertEquals(2, applicationData.get().activeSession().getAsLong()); + assertTrue(applicationData.get().lastDeployedSession().isPresent()); + assertEquals(3, applicationData.get().lastDeployedSession().getAsLong()); + + try (var t = db.createWriteActiveTransaction(new CuratorTransaction(curator), id, 3, true)) { + t.commit(); + } + // Can be read as json, active session and last deployed session present + applicationData = db.applicationData(id, true); + assertTrue(applicationData.isPresent()); + assertEquals(id, applicationData.get().applicationId()); + assertTrue(applicationData.get().activeSession().isPresent()); + assertEquals(3, applicationData.get().activeSession().getAsLong()); + assertTrue(applicationData.get().lastDeployedSession().isPresent()); + assertEquals(3, applicationData.get().lastDeployedSession().getAsLong()); + } + } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationMapperTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationMapperTest.java index 57af219c813..fb5d6537a19 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationMapperTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationMapperTest.java @@ -38,7 +38,7 @@ public class ApplicationMapperTest { @Test public void testGetForVersionReturnsCorrectVersion() { - applicationMapper.register(appId, ApplicationSet.fromList(applications)); + applicationMapper.register(appId, ApplicationVersions.fromList(applications)); assertEquals(applicationMapper.getForVersion(appId, Optional.of(vespaVersions.get(0)), Instant.now()), applications.get(0)); assertEquals(applicationMapper.getForVersion(appId, Optional.of(vespaVersions.get(1)), Instant.now()), applications.get(1)); assertEquals(applicationMapper.getForVersion(appId, Optional.of(vespaVersions.get(2)), Instant.now()), applications.get(2)); @@ -46,19 +46,19 @@ public class ApplicationMapperTest { @Test public void testGetForVersionReturnsLatestVersion() { - applicationMapper.register(appId, ApplicationSet.fromList(applications)); + applicationMapper.register(appId, ApplicationVersions.fromList(applications)); assertEquals(applicationMapper.getForVersion(appId, Optional.empty(), Instant.now()), applications.get(2)); } @Test (expected = VersionDoesNotExistException.class) public void testGetForVersionThrows() { - applicationMapper.register(appId, ApplicationSet.fromList(Arrays.asList(applications.get(0), applications.get(2)))); + applicationMapper.register(appId, ApplicationVersions.fromList(Arrays.asList(applications.get(0), applications.get(2)))); applicationMapper.getForVersion(appId, Optional.of(vespaVersions.get(1)), Instant.now()); } @Test (expected = NotFoundException.class) public void testGetForVersionThrows2() { - applicationMapper.register(appId, ApplicationSet.from(applications.get(0))); + applicationMapper.register(appId, ApplicationVersions.from(applications.get(0))); applicationMapper.getForVersion(new ApplicationId.Builder() .tenant("different") diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationSetTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationVersionsTest.java index 7629680b16f..d0a3bf4ec9b 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationSetTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationVersionsTest.java @@ -18,9 +18,9 @@ import static org.junit.Assert.assertEquals; /** * @author Vegard Sjonfjell */ -public class ApplicationSetTest { +public class ApplicationVersionsTest { - private ApplicationSet applicationSet; + private ApplicationVersions applicationVersions; private final List<Version> vespaVersions = new ArrayList<>(); private final List<Application> applications = new ArrayList<>(); @@ -32,29 +32,29 @@ public class ApplicationSetTest { @Test public void testGetForVersionOrLatestReturnsCorrectVersion() { - applicationSet = ApplicationSet.fromList(applications); - assertEquals(applicationSet.getForVersionOrLatest(Optional.of(vespaVersions.get(0)), Instant.now()), applications.get(0)); - assertEquals(applicationSet.getForVersionOrLatest(Optional.of(vespaVersions.get(1)), Instant.now()), applications.get(1)); - assertEquals(applicationSet.getForVersionOrLatest(Optional.of(vespaVersions.get(2)), Instant.now()), applications.get(2)); + applicationVersions = ApplicationVersions.fromList(applications); + assertEquals(applicationVersions.getForVersionOrLatest(Optional.of(vespaVersions.get(0)), Instant.now()), applications.get(0)); + assertEquals(applicationVersions.getForVersionOrLatest(Optional.of(vespaVersions.get(1)), Instant.now()), applications.get(1)); + assertEquals(applicationVersions.getForVersionOrLatest(Optional.of(vespaVersions.get(2)), Instant.now()), applications.get(2)); } @Test public void testGetForVersionOrLatestReturnsLatestVersion() { - applicationSet = ApplicationSet.fromList(applications); - assertEquals(applicationSet.getForVersionOrLatest(Optional.empty(), Instant.now()), applications.get(2)); + applicationVersions = ApplicationVersions.fromList(applications); + assertEquals(applicationVersions.getForVersionOrLatest(Optional.empty(), Instant.now()), applications.get(2)); } @Test (expected = VersionDoesNotExistException.class) public void testGetForVersionOrLatestThrows() { - applicationSet = ApplicationSet.fromList(Arrays.asList(applications.get(0), applications.get(2))); - applicationSet.getForVersionOrLatest(Optional.of(vespaVersions.get(1)), Instant.now()); + applicationVersions = ApplicationVersions.fromList(Arrays.asList(applications.get(0), applications.get(2))); + applicationVersions.getForVersionOrLatest(Optional.of(vespaVersions.get(1)), Instant.now()); } @Test public void testGetAllVersions() { - applicationSet = ApplicationSet.fromList(applications); + applicationVersions = ApplicationVersions.fromList(applications); assertEquals(List.of(Version.fromString("1.2.3"), Version.fromString("1.2.4"), Version.fromString("1.2.5")), - applicationSet.getAllVersions(ApplicationId.defaultId())); + applicationVersions.versions(ApplicationId.defaultId())); } private Application createApplication(Version version) { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/TenantApplicationsTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/TenantApplicationsTest.java index 2ad04fdd572..81544f4ed61 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/TenantApplicationsTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/TenantApplicationsTest.java @@ -26,6 +26,7 @@ import com.yahoo.vespa.config.server.tenant.TestTenantRepository; import com.yahoo.vespa.curator.CompletionTimeoutException; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.curator.transaction.CuratorTransaction; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.model.VespaModel; @@ -36,6 +37,7 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; import org.xml.sax.SAXException; + import java.io.File; import java.io.IOException; import java.time.Clock; @@ -120,11 +122,11 @@ public class TenantApplicationsTest { TenantApplications repo = createZKAppRepo(); ApplicationId myapp = createApplicationId("myapp"); repo.createApplication(myapp); - repo.createPutTransaction(myapp, 3).commit(); + writeActiveTransaction(repo, myapp, 3); String path = TenantRepository.getApplicationsPath(tenantName).append(myapp.serializedForm()).getAbsolute(); assertNotNull(curatorFramework.checkExists().forPath(path)); assertEquals("3", Utf8.toString(curatorFramework.getData().forPath(path))); - repo.createPutTransaction(myapp, 5).commit(); + writeActiveTransaction(repo, myapp, 5); assertNotNull(curatorFramework.checkExists().forPath(path)); assertEquals("5", Utf8.toString(curatorFramework.getData().forPath(path))); } @@ -136,26 +138,26 @@ public class TenantApplicationsTest { ApplicationId id2 = createApplicationId("myapp2"); repo.createApplication(id1); repo.createApplication(id2); - repo.createPutTransaction(id1, 1).commit(); - repo.createPutTransaction(id2, 1).commit(); + writeActiveTransaction(repo, id1, 1); + writeActiveTransaction(repo, id2, 1); assertEquals(2, repo.activeApplications().size()); - repo.createDeleteTransaction(id1).commit(); + deleteApplication(repo, id1); assertEquals(1, repo.activeApplications().size()); - repo.createDeleteTransaction(id2).commit(); + deleteApplication(repo, id2); assertEquals(0, repo.activeApplications().size()); } - private static ApplicationSet createSet(ApplicationId id, Version version) throws IOException, SAXException { + private static ApplicationVersions createApplicationVersions(ApplicationId id, Version version) throws IOException, SAXException { VespaModel model = new VespaModel(new NullConfigModelRegistry(), new DeployState.Builder().wantedNodeVespaVersion(version) .applicationPackage(FilesApplicationPackage.fromFile(new File("src/test/apps/app"))) .build()); - return ApplicationSet.from(new Application(model, - new ServerCache(), - 1, - Version.emptyVersion, - MetricUpdater.createTestUpdater(), - id)); + return ApplicationVersions.from(new Application(model, + new ServerCache(), + 1, + Version.emptyVersion, + MetricUpdater.createTestUpdater(), + id)); } @Test @@ -164,22 +166,22 @@ public class TenantApplicationsTest { TenantApplications applications = createZKAppRepo(flagSource); ApplicationId app1 = createApplicationId("myapp"); applications.createApplication(app1); - applications.createPutTransaction(app1, 1).commit(); + writeActiveTransaction(applications, app1, 1); Version deployedVersion0 = Version.fromString("6.1"); - applications.activateApplication(createSet(app1, deployedVersion0), 1); + applications.activateApplication(createApplicationVersions(app1, deployedVersion0), 1); assertTrue("Empty version is compatible", applications.compatibleWith(Optional.empty(), app1)); Version nodeVersion0 = Version.fromString("6.0"); assertTrue("Lower version is compatible", applications.compatibleWith(Optional.of(nodeVersion0), app1)); Version deployedVersion1 = Version.fromString("7.1"); - applications.activateApplication(createSet(app1, deployedVersion1), 1); + applications.activateApplication(createApplicationVersions(app1, deployedVersion1), 1); assertTrue("New major is compatible", applications.compatibleWith(Optional.of(nodeVersion0), app1)); flagSource.withListFlag(PermanentFlags.INCOMPATIBLE_VERSIONS.id(), List.of("8"), String.class); Version deployedVersion2 = Version.fromString("8.1"); - applications.activateApplication(createSet(app1, deployedVersion2), 1); + applications.activateApplication(createApplicationVersions(app1, deployedVersion2), 1); assertFalse("New major is incompatible", applications.compatibleWith(Optional.of(nodeVersion0), app1)); Version nodeVersion1 = Version.fromString("8.0"); @@ -191,7 +193,7 @@ public class TenantApplicationsTest { final AtomicInteger removed = new AtomicInteger(0); @Override - public void configActivated(ApplicationSet application) { + public void configActivated(ApplicationVersions application) { activated.incrementAndGet(); } @@ -203,19 +205,19 @@ public class TenantApplicationsTest { @Test public void testListConfigs() throws IOException, SAXException { - TenantApplications applications = createTenantApplications(TenantName.defaultName(), new MockCurator(), configserverConfig, new MockConfigActivationListener(), new InMemoryFlagSource()); + TenantApplications applications = createTenantApplications(TenantName.defaultName(), curator, configserverConfig, new MockConfigActivationListener(), new InMemoryFlagSource()); assertFalse(applications.hasApplication(ApplicationId.defaultId(), Optional.of(vespaVersion))); VespaModel model = new VespaModel(FilesApplicationPackage.fromFile(new File("src/test/apps/app"))); ApplicationId applicationId = ApplicationId.defaultId(); applications.createApplication(applicationId); - applications.createPutTransaction(applicationId, 1).commit(); - applications.activateApplication(ApplicationSet.from(new Application(model, - new ServerCache(), - 1, - vespaVersion, - MetricUpdater.createTestUpdater(), - applicationId)), + writeActiveTransaction(applications, applicationId, 1); + applications.activateApplication(ApplicationVersions.from(new Application(model, + new ServerCache(), + 1, + vespaVersion, + MetricUpdater.createTestUpdater(), + applicationId)), 1); Set<ConfigKey<?>> configNames = applications.listConfigs(applicationId, Optional.of(vespaVersion), false); assertTrue(configNames.contains(new ConfigKey<>("sentinel", "hosts", "cloud.config"))); @@ -325,4 +327,16 @@ public class TenantApplicationsTest { flagSource); } + private static void deleteApplication(TenantApplications repo, ApplicationId id1) { + try (var transaction = repo.createDeleteTransaction(id1)) { + transaction.commit(); + } + } + + private void writeActiveTransaction(TenantApplications repo, ApplicationId id1, int x) { + try (var transaction = new CuratorTransaction(curator)) { + repo.createWriteActiveTransaction(transaction, id1, x).commit(); + } + } + } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java index ba1d69c13dd..6a4099bc45a 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java @@ -66,7 +66,7 @@ public class HostHandlerTest { public void require_correct_tenant_and_application_for_hostname() throws Exception { ApplicationId applicationId = applicationId(); applicationRepository.deploy(testApp, new PrepareParams.Builder().applicationId(applicationId).build()); - String hostname = applicationRepository.getActiveApplicationSet(applicationId).get().getAllHosts().iterator().next(); + String hostname = applicationRepository.getActiveApplicationSet(applicationId).get().allHosts().iterator().next(); assertApplicationForHost(hostname, applicationId); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsHandlerTest.java index 76790e6264d..841867921da 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsHandlerTest.java @@ -15,6 +15,7 @@ import com.yahoo.vespa.config.server.application.TenantApplications; import com.yahoo.vespa.config.server.http.SessionHandlerTest; import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.vespa.config.server.tenant.TestTenantRepository; +import com.yahoo.vespa.curator.transaction.CuratorTransaction; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -38,6 +39,7 @@ public class ListApplicationsHandlerTest { private TenantApplications applicationRepo, applicationRepo2; private ListApplicationsHandler handler; + private TenantRepository tenantRepository; @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); @@ -48,7 +50,7 @@ public class ListApplicationsHandlerTest { .configServerDBDir(temporaryFolder.newFolder().getAbsolutePath()) .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) .build(); - TenantRepository tenantRepository = new TestTenantRepository.Builder() + tenantRepository = new TestTenantRepository.Builder() .withConfigserverConfig(configserverConfig) .build(); tenantRepository.addTenant(mytenant); @@ -67,12 +69,12 @@ public class ListApplicationsHandlerTest { "[]"); ApplicationId id1 = ApplicationId.from("mytenant", "foo", "quux"); applicationRepo.createApplication(id1); - applicationRepo.createPutTransaction(id1, 1).commit(); + writeActiveTransaction(applicationRepo, id1, 1); assertResponse(url, Response.Status.OK, "[\"" + url + "foo/environment/dev/region/us-east/instance/quux\"]"); ApplicationId id2 = ApplicationId.from("mytenant", "bali", "quux"); applicationRepo.createApplication(id2); - applicationRepo.createPutTransaction(id2, 1).commit(); + writeActiveTransaction(applicationRepo, id2, 1); assertResponse(url, Response.Status.OK, "[\"" + url + "bali/environment/dev/region/us-east/instance/quux\"," + "\"" + url + "foo/environment/dev/region/us-east/instance/quux\"]" @@ -98,10 +100,10 @@ public class ListApplicationsHandlerTest { public void require_that_listing_works_with_multiple_tenants() throws Exception { ApplicationId id1 = ApplicationId.from("mytenant", "foo", "quux"); applicationRepo.createApplication(id1); - applicationRepo.createPutTransaction(id1, 1).commit(); + writeActiveTransaction(applicationRepo, id1, 1); ApplicationId id2 = ApplicationId.from("foobar", "quux", "foo"); applicationRepo2.createApplication(id2); - applicationRepo2.createPutTransaction(id2, 1).commit(); + writeActiveTransaction(applicationRepo2, id2, 1); String url = "http://myhost:14000/application/v2/tenant/mytenant/application/"; assertResponse(url, Response.Status.OK, "[\"" + url + "foo/environment/dev/region/us-east/instance/quux\"]"); @@ -124,4 +126,11 @@ public class ListApplicationsHandlerTest { assertEquals(expectedStatus, response.getStatus()); assertEquals(expectedResponse, SessionHandlerTest.getRenderedString(response)); } + + private void writeActiveTransaction(TenantApplications repo, ApplicationId id1, int x) { + try (var transaction = new CuratorTransaction(tenantRepository.getCurator())) { + repo.createWriteActiveTransaction(transaction, id1, x).commit(); + } + } + } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java index 9190cbc0d8a..8db86aa4dec 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java @@ -25,7 +25,7 @@ import com.yahoo.vespa.config.protocol.Trace; import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.ServerCache; import com.yahoo.vespa.config.server.application.Application; -import com.yahoo.vespa.config.server.application.ApplicationSet; +import com.yahoo.vespa.config.server.application.ApplicationVersions; import com.yahoo.vespa.config.server.monitoring.MetricUpdater; import com.yahoo.vespa.config.server.session.PrepareParams; import com.yahoo.vespa.filedistribution.LazyFileReferenceData; @@ -142,7 +142,7 @@ public class RpcServerTest { new Version(1, 2, 3), MetricUpdater.createTestUpdater(), applicationId); - ApplicationSet appSet = ApplicationSet.from(app); + ApplicationVersions appSet = ApplicationVersions.from(app); tester.rpcServer().configActivated(appSet); ConfigKey<?> key = new ConfigKey<>(LbServicesConfig.class, "*"); JRTClientConfigRequest clientReq = createRequest(new RawConfig(key, LbServicesConfig.getDefMd5())); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionRepositoryTest.java index 07d3aac5a52..bb71cbd35d4 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionRepositoryTest.java @@ -19,7 +19,7 @@ import com.yahoo.io.reader.NamedReader; import com.yahoo.path.Path; import com.yahoo.text.Utf8; import com.yahoo.vespa.config.server.ApplicationRepository; -import com.yahoo.vespa.config.server.application.ApplicationSet; +import com.yahoo.vespa.config.server.application.ApplicationVersions; import com.yahoo.vespa.config.server.application.OrchestratorMock; import com.yahoo.vespa.config.server.filedistribution.MockFileDistributionFactory; import com.yahoo.vespa.config.server.http.InvalidApplicationException; @@ -115,11 +115,11 @@ public class SessionRepositoryTest { assertNotNull(sessionRepository.getLocalSession(secondSessionId)); assertNull(sessionRepository.getLocalSession(secondSessionId + 1)); - ApplicationSet applicationSet = sessionRepository.ensureApplicationLoaded(sessionRepository.getRemoteSession(firstSessionId)); - assertNotNull(applicationSet); - assertEquals(2, applicationSet.getApplicationGeneration()); - assertEquals(applicationId.application(), applicationSet.getForVersionOrLatest(Optional.empty(), Instant.now()).getId().application()); - assertNotNull(applicationSet.getForVersionOrLatest(Optional.empty(), Instant.now()).getModel()); + ApplicationVersions applicationVersions = sessionRepository.ensureApplicationLoaded(sessionRepository.getRemoteSession(firstSessionId)); + assertNotNull(applicationVersions); + assertEquals(2, applicationVersions.applicationGeneration()); + assertEquals(applicationId.application(), applicationVersions.getForVersionOrLatest(Optional.empty(), Instant.now()).getId().application()); + assertNotNull(applicationVersions.getForVersionOrLatest(Optional.empty(), Instant.now()).getModel()); LocalSession session = sessionRepository.getLocalSession(secondSessionId); Collection<NamedReader> a = session.applicationPackage.get().getSchemas(); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java index 9af1bbb875e..02ee3202475 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java @@ -17,7 +17,7 @@ import com.yahoo.vespa.config.server.MockSecretStore; import com.yahoo.vespa.config.server.ServerCache; import com.yahoo.vespa.config.server.TestConfigDefinitionRepo; import com.yahoo.vespa.config.server.application.Application; -import com.yahoo.vespa.config.server.application.ApplicationSet; +import com.yahoo.vespa.config.server.application.ApplicationVersions; import com.yahoo.vespa.config.server.application.TenantApplications; import com.yahoo.vespa.config.server.application.TenantApplicationsTest; import com.yahoo.vespa.config.server.filedistribution.FileDirectory; @@ -29,6 +29,7 @@ import com.yahoo.vespa.config.server.monitoring.Metrics; import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.curator.transaction.CuratorTransaction; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.model.VespaModel; @@ -106,13 +107,15 @@ public class TenantRepositoryTest { TenantApplications applicationRepo = tenantRepository.getTenant(tenant1).getApplicationRepo(); ApplicationId id = ApplicationId.from(tenant1, ApplicationName.defaultName(), InstanceName.defaultName()); applicationRepo.createApplication(id); - applicationRepo.createPutTransaction(id, 4).commit(); - applicationRepo.activateApplication(ApplicationSet.from(new Application(new VespaModel(MockApplicationPackage.createEmpty()), - new ServerCache(), - 4L, - new Version(1, 2, 3), - MetricUpdater.createTestUpdater(), - id)), + try (var transaction = new CuratorTransaction(curator)) { + applicationRepo.createWriteActiveTransaction(transaction, id, 4).commit(); + } + applicationRepo.activateApplication(ApplicationVersions.from(new Application(new VespaModel(MockApplicationPackage.createEmpty()), + new ServerCache(), + 4L, + new Version(1, 2, 3), + MetricUpdater.createTestUpdater(), + id)), 4); assertEquals(1, listener.activated.get()); } |