From c4ff400b725a9864f92a6b0c5545aa21a1100030 Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Fri, 15 Sep 2023 12:18:37 +0200 Subject: Always write application data as json Test of transition from old to new format, need to keep some fucntionality around this until Vespa 9 --- .../application/ApplicationCuratorDatabase.java | 54 ++++++++++++++-------- .../server/application/TenantApplications.java | 14 ++---- .../ApplicationCuratorDatabaseTest.java | 33 ++++++++----- 3 files changed, 61 insertions(+), 40 deletions(-) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationCuratorDatabase.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationCuratorDatabase.java index 68b1339a0a9..ea563ca6d0d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationCuratorDatabase.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationCuratorDatabase.java @@ -74,34 +74,56 @@ public class ApplicationCuratorDatabase { /** * Creates a node for the given application, marking its existence. */ - public void createApplication(ApplicationId id, boolean writeAsJson) { + public void createApplication(ApplicationId id) { if ( ! id.tenant().equals(tenant)) throw new IllegalArgumentException("Cannot write application id '" + id + "' for tenant '" + tenant + "'"); try (Lock lock = lock(id)) { if (curator.exists(applicationPath(id))) return; - if (writeAsJson) { - var applicationData = new ApplicationData(id, OptionalLong.empty(), OptionalLong.empty()); - curator.set(applicationPath(id), applicationData.toJson()); - } else { - curator.create(applicationPath(id)); - } + var applicationData = new ApplicationData(id, OptionalLong.empty(), OptionalLong.empty()); + curator.set(applicationPath(id), applicationData.toJson()); + modifyReindexing(id, ApplicationReindexing.empty(), UnaryOperator.identity()); + } + } + + /** + * Creates a node for the given application, marking its existence. + */ + // TODO: Remove in Vespa 9 + public void createApplicationInOldFormat(ApplicationId id) { + if (! id.tenant().equals(tenant)) + throw new IllegalArgumentException("Cannot write application id '" + id + "' for tenant '" + tenant + "'"); + + try (Lock lock = lock(id)) { + if (curator.exists(applicationPath(id))) return; + + curator.create(applicationPath(id)); modifyReindexing(id, ApplicationReindexing.empty(), UnaryOperator.identity()); } } + /** + * Returns a transaction which writes the given session id as the currently active for the given application. + * + * @param applicationId An {@link ApplicationId} that represents an active application. + * @param sessionId session id belonging to the application package for this application id. + */ + public Transaction createWriteActiveTransaction(Transaction transaction, ApplicationId applicationId, long sessionId) { + String path = applicationPath(applicationId).getAbsolute(); + return transaction.add(setData(path, new ApplicationData(applicationId, OptionalLong.of(sessionId), OptionalLong.of(sessionId)).toJson())); + } + /** * Returns a transaction which writes the given session id as the currently active for the given application. * * @param applicationId An {@link ApplicationId} that represents an active application. * @param sessionId session id belonging to the application package for this application id. */ - public Transaction createWriteActiveTransaction(Transaction transaction, ApplicationId applicationId, long sessionId, boolean writeAsJson) { + // TODO: Remove in Vespa 9 + public Transaction createWriteActiveTransactionInOldFormat(Transaction transaction, ApplicationId applicationId, long sessionId) { String path = applicationPath(applicationId).getAbsolute(); - return transaction.add(writeAsJson - ? setData(path, new ApplicationData(applicationId, OptionalLong.of(sessionId), OptionalLong.of(sessionId)).toJson()) - : setData(path, Utf8.toAsciiBytes(sessionId))); + return transaction.add(setData(path, Utf8.toAsciiBytes(sessionId))); } /** @@ -113,16 +135,10 @@ public class ApplicationCuratorDatabase { public Transaction createWritePrepareTransaction(Transaction transaction, ApplicationId applicationId, long sessionId, - OptionalLong activeSessionId, - boolean writeAsJson) { - + OptionalLong activeSessionId) { // Needs to read or be supplied current active session id, to avoid overwriting a newer session id. - String path = applicationPath(applicationId).getAbsolute(); - if (writeAsJson) - return transaction.add(setData(path, new ApplicationData(applicationId, activeSessionId, OptionalLong.of(sessionId)).toJson())); - else - return transaction; // Do nothing, as there is nothing to write in this case + return transaction.add(setData(path, new ApplicationData(applicationId, activeSessionId, OptionalLong.of(sessionId)).toJson())); } /** diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java index 1d1ed1042ee..296e31ce801 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java @@ -13,8 +13,8 @@ import com.yahoo.transaction.Transaction; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.GetConfigRequest; import com.yahoo.vespa.config.protocol.ConfigResponse; -import com.yahoo.vespa.config.server.NotFoundException; import com.yahoo.vespa.config.server.ConfigActivationListener; +import com.yahoo.vespa.config.server.NotFoundException; import com.yahoo.vespa.config.server.RequestHandler; import com.yahoo.vespa.config.server.deploy.TenantFileSystemDirs; import com.yahoo.vespa.config.server.host.HostRegistry; @@ -27,13 +27,12 @@ import com.yahoo.vespa.curator.CompletionTimeoutException; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.curator.transaction.CuratorTransaction; -import com.yahoo.vespa.flags.BooleanFlag; import com.yahoo.vespa.flags.FlagSource; -import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.ListFlag; import com.yahoo.vespa.flags.PermanentFlags; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; + import java.nio.file.Files; import java.nio.file.Paths; import java.time.Clock; @@ -79,7 +78,6 @@ public class TenantApplications implements RequestHandler, HostValidator { private final TenantFileSystemDirs tenantFileSystemDirs; private final String serverId; private final ListFlag incompatibleVersions; - private final BooleanFlag writeApplicationDataAsJson; public TenantApplications(TenantName tenant, Curator curator, StripedExecutor zkWatcherExecutor, ExecutorService zkCacheExecutor, Metrics metrics, ConfigActivationListener configActivationListener, @@ -101,7 +99,6 @@ public class TenantApplications implements RequestHandler, HostValidator { this.clock = clock; this.serverId = configserverConfig.serverId(); this.incompatibleVersions = PermanentFlags.INCOMPATIBLE_VERSIONS.bindTo(flagSource); - this.writeApplicationDataAsJson = Flags.WRITE_APPLICATION_DATA_AS_JSON.bindTo(flagSource); } /** The curator backed ZK storage of this. */ @@ -147,7 +144,7 @@ public class TenantApplications implements RequestHandler, HostValidator { * @param sessionId session id belonging to the application package for this application id. */ public Transaction createWriteActiveTransaction(Transaction transaction, ApplicationId applicationId, long sessionId) { - return database().createWriteActiveTransaction(transaction, applicationId, sessionId, writeApplicationDataAsJson.value()); + return database().createWriteActiveTransaction(transaction, applicationId, sessionId); } /** @@ -163,15 +160,14 @@ public class TenantApplications implements RequestHandler, HostValidator { return database().createWritePrepareTransaction(transaction, applicationId, sessionId, - activeSessionId.map(OptionalLong::of).orElseGet(OptionalLong::empty), - writeApplicationDataAsJson.value()); + activeSessionId.map(OptionalLong::of).orElseGet(OptionalLong::empty)); } /** * Creates a node for the given application, marking its existence. */ public void createApplication(ApplicationId id) { - database().createApplication(id, writeApplicationDataAsJson.value()); + database().createApplication(id); } /** diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationCuratorDatabaseTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationCuratorDatabaseTest.java index 2f0ac236170..06a160bc8ff 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationCuratorDatabaseTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationCuratorDatabaseTest.java @@ -47,11 +47,11 @@ public class ApplicationCuratorDatabaseTest { assertEquals(Optional.empty(), db.applicationData(id)); - db.createApplication(id, false); + db.createApplicationInOldFormat(id); assertEquals(Optional.empty(), db.applicationData(id)); // still empty, as no data has been written to node deleteApplication(db, id); - db.createApplication(id, true); + db.createApplication(id); // Can be read as json, but no active session or last deployed session Optional applicationData = db.applicationData(id); assertTrue(applicationData.isPresent()); @@ -60,9 +60,9 @@ public class ApplicationCuratorDatabaseTest { assertFalse(applicationData.get().lastDeployedSession().isPresent()); // Prepare session 2, no active session - prepareSession(db, id, 2, OptionalLong.empty(), false); + prepareSessionOldFormat(db, id, 2, OptionalLong.empty()); // Activate session 2, last deployed session not present (not writing json) - activateSession(db, id, 2, false); + activateSessionOldFormat(db, id, 2); // Can be read as session id only applicationData = db.applicationData(id); assertTrue(applicationData.isPresent()); @@ -79,7 +79,7 @@ public class ApplicationCuratorDatabaseTest { assertFalse(applicationData.get().lastDeployedSession().isPresent()); // Prepare session 3, last deployed session is still 2 - prepareSession(db, id, 3, OptionalLong.of(2), true); + prepareSession(db, id, 3, OptionalLong.of(2)); // Can be read as json, active session is still 2 and last deployed session is 3 applicationData = db.applicationData(id); assertTrue(applicationData.isPresent()); @@ -89,7 +89,7 @@ public class ApplicationCuratorDatabaseTest { assertTrue(applicationData.get().lastDeployedSession().isPresent()); assertEquals(3, applicationData.get().lastDeployedSession().get().longValue()); - activateSession(db, id, 3, true); + activateSession(db, id, 3); // Can be read as json, active session and last deployed session present applicationData = db.applicationData(id); assertTrue(applicationData.isPresent()); @@ -100,7 +100,7 @@ public class ApplicationCuratorDatabaseTest { assertEquals(3, applicationData.get().lastDeployedSession().get().longValue()); // createApplication should not overwrite the node if it already exists - db.createApplication(id, true); + db.createApplication(id); // Can be read as json, active session and last deployed session present applicationData = db.applicationData(id); assertTrue(applicationData.isPresent()); @@ -118,18 +118,27 @@ public class ApplicationCuratorDatabaseTest { } } - private void prepareSession(ApplicationCuratorDatabase db, ApplicationId applicationId, long sessionId, OptionalLong activesSessionId, boolean writeAsJson) { + private void prepareSession(ApplicationCuratorDatabase db, ApplicationId applicationId, long sessionId, OptionalLong activesSessionId) { try (var t = db.createWritePrepareTransaction(new CuratorTransaction(curator), applicationId, sessionId, - activesSessionId, - writeAsJson)) { + activesSessionId)) { t.commit(); } } - private void activateSession(ApplicationCuratorDatabase db, ApplicationId applicationId, long sessionId, boolean writeAsJson) { - try (var t = db.createWriteActiveTransaction(new CuratorTransaction(curator), applicationId, sessionId, writeAsJson)) { + private void prepareSessionOldFormat(ApplicationCuratorDatabase db, ApplicationId applicationId, long sessionId, OptionalLong activesSessionId) { + return; // Nothing to do, just return + } + + private void activateSession(ApplicationCuratorDatabase db, ApplicationId applicationId, long sessionId) { + try (var t = db.createWriteActiveTransaction(new CuratorTransaction(curator), applicationId, sessionId)) { + t.commit(); + } + } + + private void activateSessionOldFormat(ApplicationCuratorDatabase db, ApplicationId applicationId, long sessionId) { + try (var t = db.createWriteActiveTransactionInOldFormat(new CuratorTransaction(curator), applicationId, sessionId)) { t.commit(); } } -- cgit v1.2.3