Write tenant metadata if feature flag is true

We want metadata to be able to find unused tenants, there is no way of
tracking this today without tracking all sessions for a tenant and we allow
sessions some lifetime, so this makes it impossible to find out in systems
where there are a lot of short-lived deployments (test systems).

6 files changed, 149 insertions, 15 deletions

diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
index e49bb21acba..c7a98623e8d 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
@@ -25,6 +25,7 @@ import com.yahoo.io.IOUtils;
 import com.yahoo.jdisc.Metric;
 import com.yahoo.path.Path;
 import com.yahoo.slime.Slime;
+import com.yahoo.text.Utf8;
 import com.yahoo.transaction.NestedTransaction;
 import com.yahoo.transaction.Transaction;
 import com.yahoo.vespa.config.server.application.Application;
@@ -60,10 +61,17 @@ import com.yahoo.vespa.config.server.tenant.ApplicationRolesStore;
 import com.yahoo.vespa.config.server.tenant.ContainerEndpointsCache;
 import com.yahoo.vespa.config.server.tenant.EndpointCertificateMetadataStore;
 import com.yahoo.vespa.config.server.tenant.Tenant;
+import com.yahoo.vespa.config.server.tenant.TenantMetaData;
 import com.yahoo.vespa.config.server.tenant.TenantRepository;
 import com.yahoo.vespa.curator.Curator;
 import com.yahoo.vespa.curator.Lock;
+import com.yahoo.vespa.curator.transaction.CuratorOperations;
+import com.yahoo.vespa.curator.transaction.CuratorTransaction;
 import com.yahoo.vespa.defaults.Defaults;
+import com.yahoo.vespa.flags.BooleanFlag;
+import com.yahoo.vespa.flags.FlagSource;
+import com.yahoo.vespa.flags.Flags;
+import com.yahoo.vespa.flags.InMemoryFlagSource;
 import com.yahoo.vespa.orchestrator.Orchestrator;
 
 import java.io.File;
@@ -92,9 +100,9 @@ import static com.yahoo.config.model.api.container.ContainerServiceType.CLUSTERC
 import static com.yahoo.config.model.api.container.ContainerServiceType.CONTAINER;
 import static com.yahoo.config.model.api.container.ContainerServiceType.LOGSERVER_CONTAINER;
 import static com.yahoo.vespa.config.server.filedistribution.FileDistributionUtil.fileReferenceExistsOnDisk;
-import static com.yahoo.vespa.curator.Curator.CompletionWaiter;
 import static com.yahoo.vespa.config.server.filedistribution.FileDistributionUtil.getFileReferencesOnDisk;
 import static com.yahoo.vespa.config.server.tenant.TenantRepository.HOSTED_VESPA_TENANT;
+import static com.yahoo.vespa.curator.Curator.CompletionWaiter;
 import static com.yahoo.yolean.Exceptions.uncheck;
 import static java.nio.file.Files.readAttributes;
 
@@ -122,6 +130,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
     private final LogRetriever logRetriever;
     private final TesterClient testerClient;
     private final Metric metric;
+    private final BooleanFlag useTenantMetaData;
 
     @Inject
     public ApplicationRepository(TenantRepository tenantRepository,
@@ -132,7 +141,8 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
                                  ConfigserverConfig configserverConfig,
                                  Orchestrator orchestrator,
                                  TesterClient testerClient,
-                                 Metric metric) {
+                                 Metric metric,
+                                 FlagSource flagSource) {
         this(tenantRepository,
              hostProvisionerProvider.getHostProvisioner(),
              infraDeployerProvider.getInfraDeployer(),
@@ -144,7 +154,8 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
              new FileDistributionStatus(),
              Clock.systemUTC(),
              testerClient,
-             metric);
+             metric,
+             flagSource);
     }
 
     // For testing
@@ -159,7 +170,8 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
              new LogRetriever(),
              clock,
              new TesterClient(),
-             new NullMetric());
+             new NullMetric(),
+             new InMemoryFlagSource());
     }
 
     // For testing
@@ -170,7 +182,8 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
                                  LogRetriever logRetriever,
                                  Clock clock,
                                  TesterClient testerClient,
-                                 Metric metric) {
+                                 Metric metric,
+                                 FlagSource flagSource) {
         this(tenantRepository,
              Optional.of(hostProvisioner),
              Optional.empty(),
@@ -182,7 +195,8 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
              new FileDistributionStatus(),
              clock,
              testerClient,
-             metric);
+             metric,
+             flagSource);
     }
 
     private ApplicationRepository(TenantRepository tenantRepository,
@@ -196,7 +210,8 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
                                   FileDistributionStatus fileDistributionStatus,
                                   Clock clock,
                                   TesterClient testerClient,
-                                  Metric metric) {
+                                  Metric metric,
+                                  FlagSource flagSource) {
         this.tenantRepository = tenantRepository;
         this.hostProvisioner = hostProvisioner;
         this.infraDeployer = infraDeployer;
@@ -209,6 +224,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
         this.clock = clock;
         this.testerClient = testerClient;
         this.metric = metric;
+        this.useTenantMetaData = Flags.USE_TENANT_META_DATA.bindTo(flagSource);
     }
 
     public Metric metric() {
@@ -348,16 +364,36 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
     }
 
     public Transaction deactivateCurrentActivateNew(Session active, LocalSession prepared, boolean ignoreStaleSessionFailure) {
-        SessionRepository sessionRepository = tenantRepository.getTenant(prepared.getTenantName()).getSessionRepository();
-        Transaction transaction = sessionRepository.createActivateTransaction(prepared);
+        Tenant tenant = tenantRepository.getTenant(prepared.getTenantName());
+        Transaction transaction = tenant.getSessionRepository().createActivateTransaction(prepared);
         if (active != null) {
             checkIfActiveHasChanged(prepared, active, ignoreStaleSessionFailure);
             checkIfActiveIsNewerThanSessionToBeActivated(prepared.getSessionId(), active.getSessionId());
             transaction.add(active.createDeactivateTransaction().operations());
         }
+
+        if (useTenantMetaData.value())
+            transaction.add(writeTenantMetaData(createMetaData(tenant), prepared.getApplicationId()).operations());
+
         return transaction;
     }
 
+    private String createMetaData(Tenant tenant) {
+        return new TenantMetaData(tenant.getSessionRepository().clock().instant()).asJsonString();
+    }
+
+    TenantMetaData getTenantMetaData(ApplicationId applicationId) {
+        Tenant tenant = getTenant(applicationId);
+        Optional<byte[]> data = tenantRepository.getCurator().getData(TenantRepository.getTenantPath(tenant.getName()));
+        return data.map(bytes -> TenantMetaData.fromJsonString(Utf8.toString(bytes))).orElse(new TenantMetaData(tenant.getCreatedTime()));
+    }
+
+    private Transaction writeTenantMetaData(String jsonString, ApplicationId applicationId) {
+        return new CuratorTransaction(tenantRepository.getCurator())
+                .add(CuratorOperations.setData(TenantRepository.getTenantPath(applicationId.tenant()).getAbsolute(),
+                                               Utf8.toBytes(jsonString)));
+    }
+
     static void checkIfActiveHasChanged(LocalSession session, Session currentActiveSession, boolean ignoreStaleSessionFailure) {
         long activeSessionAtCreate = session.getActiveSessionAtCreate();
         log.log(Level.FINE, currentActiveSession.logPre() + "active session id at create time=" + activeSessionAtCreate);
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java
index 1964750495b..0d093f83ed0 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java
@@ -691,6 +691,8 @@ public class SessionRepository {
         return curator.lock(lockPath(sessionId), Duration.ofMinutes(1)); // These locks shouldn't be held for very long.
     }
 
+    public Clock clock() { return clock; }
+
     private Path lockPath(long sessionId) {
         return locksPath.append(String.valueOf(sessionId));
     }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantMetaData.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantMetaData.java
new file mode 100644
index 00000000000..6a91c757c3b
--- /dev/null
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantMetaData.java
@@ -0,0 +1,65 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.config.server.tenant;
+
+import com.yahoo.slime.Cursor;
+import com.yahoo.slime.Inspector;
+import com.yahoo.slime.JsonDecoder;
+import com.yahoo.slime.JsonFormat;
+import com.yahoo.slime.Slime;
+import com.yahoo.text.Utf8;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.time.Instant;
+
+/**
+ * Metadata for a tenant. At the moment only stores last deploy time, to be used by TenantsMaintainer
+ * to GC unused tenants
+ *
+ * @author hmusum
+ */
+public class TenantMetaData {
+
+    private final Instant lastDeployTimestamp;
+
+    public TenantMetaData(Instant instant) {
+        this.lastDeployTimestamp = instant;
+    }
+
+    public Instant lastDeployTimestamp() {
+        return lastDeployTimestamp;
+    }
+
+    Slime getSlime() {
+        Slime slime = new Slime();
+        Cursor meta = slime.setObject();
+        meta.setLong("lastDeployTimestamp", lastDeployTimestamp.toEpochMilli());
+        return slime;
+    }
+
+    public String asJsonString() {
+        Slime slime = getSlime();
+        ByteArrayOutputStream baos = new ByteArrayOutputStream();
+        try {
+            new JsonFormat(false).encode(baos, slime);
+            return baos.toString(StandardCharsets.UTF_8);
+        } catch (IOException e) {
+            throw new RuntimeException("Unable to encode metadata", e);
+        }
+    }
+
+    public static TenantMetaData fromJsonString(String jsonString) {
+        try {
+            Slime data = new Slime();
+            new JsonDecoder().decode(data, Utf8.toBytes(jsonString));
+            Inspector root = data.get();
+            Inspector lastDeployTimestamp = root.field("lastDeployTimestamp");
+
+            return new TenantMetaData(Instant.ofEpochMilli(lastDeployTimestamp.asLong()));
+        } catch (Exception e) {
+            throw new IllegalArgumentException("Error parsing json metadata", e);
+        }
+    }
+
+}
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java
index f85911eac01..bf9048841de 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java
@@ -152,7 +152,8 @@ public class ApplicationRepositoryTest {
                                                           new MockLogRetriever(),
                                                           clock,
                                                           new MockTesterClient(),
-                                                          new NullMetric());
+                                                          new NullMetric(),
+                                                          flagSource);
         timeoutBudget = new TimeoutBudget(clock, Duration.ofSeconds(60));
     }
 
@@ -166,6 +167,29 @@ public class ApplicationRepositoryTest {
         LocalSession session = tenant.getSessionRepository().getLocalSession(tenant.getApplicationRepo()
                                                                                .requireActiveSessionOf(applicationId()));
         session.getAllocatedHosts();
+
+        assertEquals(Instant.EPOCH, applicationRepository.getTenantMetaData(applicationId()).lastDeployTimestamp());
+    }
+
+    @Test
+    public void prepareAndActivateWithTenantMetaData() throws IOException {
+        FlagSource flagSource = new InMemoryFlagSource().withBooleanFlag(Flags.USE_TENANT_META_DATA.id(), true);
+        setup(flagSource);
+
+        Duration duration = Duration.ofHours(1);
+        clock.advance(duration);
+        Instant deployTime = clock.instant();
+        PrepareResult result = prepareAndActivate(testApp);
+        assertTrue(result.configChangeActions().getRefeedActions().isEmpty());
+        assertTrue(result.configChangeActions().getRestartActions().isEmpty());
+
+        Tenant tenant = applicationRepository.getTenant(applicationId());
+        LocalSession session = tenant.getSessionRepository().getLocalSession(tenant.getApplicationRepo()
+                                                                                     .requireActiveSessionOf(applicationId()));
+        session.getAllocatedHosts();
+
+        assertEquals(deployTime.toEpochMilli(),
+                     applicationRepository.getTenantMetaData(applicationId()).lastDeployTimestamp().toEpochMilli());
     }
 
     @Test
@@ -434,7 +458,8 @@ public class ApplicationRepositoryTest {
                                                           new MockLogRetriever(),
                                                           new ManualClock(),
                                                           new MockTesterClient(),
-                                                          actual);
+                                                          actual,
+                                                          new InMemoryFlagSource());
         deployApp(testAppLogServerWithContainer);
         Map<String, ?> context = Map.of("applicationId", "test1.testapp.default",
                                         "tenantName", "test1",
@@ -684,7 +709,8 @@ public class ApplicationRepositoryTest {
                                          new MockLogRetriever(),
                                          clock,
                                          new MockTesterClient(),
-                                         new NullMetric());
+                                         new NullMetric(),
+                                         new InMemoryFlagSource());
     }
 
     private PrepareResult prepareAndActivate(File application) {
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java
index a574d8f3b60..2b7c1c3a6dc 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java
@@ -43,6 +43,7 @@ import com.yahoo.vespa.config.server.tenant.Tenant;
 import com.yahoo.vespa.config.server.tenant.TenantRepository;
 import com.yahoo.vespa.curator.Curator;
 import com.yahoo.vespa.curator.mock.MockCurator;
+import com.yahoo.vespa.flags.InMemoryFlagSource;
 import com.yahoo.vespa.model.VespaModel;
 import com.yahoo.vespa.model.VespaModelFactory;
 
@@ -133,7 +134,8 @@ public class DeployTester {
                                                           new LogRetriever(),
                                                           clock,
                                                           new MockTesterClient(),
-                                                          new NullMetric());
+                                                          new NullMetric(),
+                                                          new InMemoryFlagSource());
     }
 
     public Tenant tenant() {
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java
index b6a96d680ec..9f8eb95a348 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java
@@ -32,6 +32,7 @@ import com.yahoo.vespa.config.server.provision.HostProvisionerProvider;
 import com.yahoo.vespa.config.server.session.PrepareParams;
 import com.yahoo.vespa.config.server.tenant.Tenant;
 import com.yahoo.vespa.config.server.tenant.TenantRepository;
+import com.yahoo.vespa.flags.InMemoryFlagSource;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
@@ -98,7 +99,8 @@ public class ApplicationHandlerTest {
                                                           logRetriever,
                                                           Clock.systemUTC(),
                                                           testerClient,
-                                                          metric);
+                                                          metric,
+                                                          new InMemoryFlagSource());
     }
 
     @After
@@ -207,7 +209,8 @@ public class ApplicationHandlerTest {
                                                                                 configserverConfig,
                                                                                 orchestrator,
                                                                                 testerClient,
-                                                                                metric);
+                                                                                metric,
+                                                                                new InMemoryFlagSource());
         ApplicationHandler mockHandler = createApplicationHandler(applicationRepository);
         when(mockHttpProxy.get(any(), eq(host), eq(CLUSTERCONTROLLER_CONTAINER.serviceName),eq("clustercontroller-status/v1/clusterName1")))
                 .thenReturn(new StaticResponse(200, "text/html", "<html>...</html>"));