Add maintainer for deleting unused tenants

10 files changed, 197 insertions, 73 deletions

diff --git a/configdefinitions/src/vespa/configserver.def b/configdefinitions/src/vespa/configserver.def
index 77f45b104b2..5e8578e76f3 100644
--- a/configdefinitions/src/vespa/configserver.def
+++ b/configdefinitions/src/vespa/configserver.def
@@ -48,3 +48,7 @@ loadBalancerAddress string default=""
 
 # Node admin
 nodeAdminInContainer bool default=true
+
+# Maintainers
+# TODO: Default set to a high value (1 year) => maintainer will not run, change when maintainer verified out in prod
+tenantsMaintainerIntervalMinutes int default=525600
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
index 4f8d7818316..28718ee7154 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
@@ -58,6 +58,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import java.util.Set;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
@@ -65,6 +66,7 @@ import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 import java.util.logging.Level;
 import java.util.logging.Logger;
+import java.util.stream.Collectors;
 
 /**
  * The API for managing applications.
@@ -364,6 +366,32 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
         return session.getSessionId();
     }
 
+
+    // ---------------- Tenant operations ----------------------------------------------------------------
+
+    public Set<TenantName> removeUnusedTenants() {
+        Set<TenantName> tenantsToBeDeleted = tenantRepository.getAllTenantNames().stream()
+                .filter(tenantName -> activeApplications(tenantName).isEmpty())
+                .filter(tenantName -> !tenantName.equals(TenantName.defaultName())) // Not allowed to remove 'default' tenant
+                .collect(Collectors.toSet());
+        tenantsToBeDeleted.forEach(tenantRepository::deleteTenant);
+        return tenantsToBeDeleted;
+    }
+
+    public void deleteTenant(TenantName tenantName) {
+        List<ApplicationId> activeApplications = activeApplications(tenantName);
+        if (activeApplications.isEmpty())
+            tenantRepository.deleteTenant(tenantName);
+        else
+            throw new IllegalArgumentException("Cannot delete tenant '" + tenantName + "', it has active applications: " + activeApplications);
+    }
+
+    private List<ApplicationId> activeApplications(TenantName tenantName) {
+        return tenantRepository.getTenant(tenantName).getApplicationRepo().listApplications();
+    }
+
+    // ---------------- Misc operations ----------------------------------------------------------------
+
     public Tenant verifyTenantAndApplication(ApplicationId applicationId) {
         TenantName tenantName = applicationId.tenant();
         if (!tenantRepository.checkThatTenantExists(tenantName)) {
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantHandler.java
index 3857fea9d14..c8e9da1265b 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantHandler.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantHandler.java
@@ -1,18 +1,15 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.config.server.http.v2;
 
-import java.util.List;
 import com.google.inject.Inject;
 
-import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.TenantName;
 import com.yahoo.container.jdisc.HttpRequest;
 import com.yahoo.container.jdisc.HttpResponse;
 import com.yahoo.jdisc.application.BindingMatch;
-import com.yahoo.vespa.config.server.tenant.Tenant;
+import com.yahoo.vespa.config.server.ApplicationRepository;
 import com.yahoo.vespa.config.server.tenant.TenantRepository;
 import com.yahoo.yolean.Exceptions;
-import com.yahoo.vespa.config.server.application.TenantApplications;
 import com.yahoo.vespa.config.server.http.BadRequestException;
 import com.yahoo.vespa.config.server.http.HttpHandler;
 import com.yahoo.vespa.config.server.http.InternalServerException;
@@ -27,11 +24,13 @@ public class TenantHandler extends HttpHandler {
 
     private static final String TENANT_NAME_REGEXP = "[\\w-]+";
     private final TenantRepository tenantRepository;
+    private final ApplicationRepository applicationRepository;
 
     @Inject
-    public TenantHandler(HttpHandler.Context ctx, TenantRepository tenantRepository) {
+    public TenantHandler(Context ctx, TenantRepository tenantRepository, ApplicationRepository applicationRepository) {
         super(ctx);
         this.tenantRepository = tenantRepository;
+        this.applicationRepository = applicationRepository;
     }
 
     @Override
@@ -62,22 +61,7 @@ public class TenantHandler extends HttpHandler {
     protected HttpResponse handleDELETE(HttpRequest request) {
         final TenantName tenantName = getTenantNameFromRequest(request);
         Utils.checkThatTenantExists(tenantRepository, tenantName);
-        // TODO: Move logic to ApplicationRepository
-        Tenant tenant = tenantRepository.getTenant(tenantName);
-        TenantApplications applicationRepo = tenant.getApplicationRepo();
-        final List<ApplicationId> activeApplications = applicationRepo.listApplications();
-        if (activeApplications.isEmpty()) {
-            try {
-                tenantRepository.deleteTenant(tenantName);
-            } catch (IllegalArgumentException e) {
-                throw e;
-            } catch (Exception e) {
-                throw new InternalServerException(Exceptions.toMessageString(e));
-            }
-        } else {
-            throw new BadRequestException("Cannot delete tenant '" + tenantName + "', as it has active applications: " +
-                    activeApplications);
-        }
+        applicationRepository.deleteTenant(tenantName);
         return new TenantDeleteResponse(tenantName);
     }
 
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java
new file mode 100644
index 00000000000..c8b3bc824a8
--- /dev/null
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java
@@ -0,0 +1,29 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.config.server.maintenance;
+
+import com.yahoo.cloud.config.ConfigserverConfig;
+import com.yahoo.component.AbstractComponent;
+import com.yahoo.vespa.config.server.ApplicationRepository;
+import com.yahoo.vespa.curator.Curator;
+
+import java.time.Duration;
+
+public class ConfigServerMaintenance extends AbstractComponent {
+
+    private final TenantsMaintainer tenantsMaintainer;
+
+    @SuppressWarnings("unused") // instantiated by Dependency Injection
+    public ConfigServerMaintenance(ConfigserverConfig configserverConfig,
+                                   ApplicationRepository applicationRepository,
+                                   Curator curator) {
+        tenantsMaintainer = new TenantsMaintainer(applicationRepository,
+                                                  curator,
+                                                  Duration.ofMinutes(configserverConfig.tenantsMaintainerIntervalMinutes()));
+    }
+
+    @Override
+    public void deconstruct() {
+        tenantsMaintainer.deconstruct();
+    }
+
+}
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/Maintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/Maintainer.java
new file mode 100644
index 00000000000..ce0811184a3
--- /dev/null
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/Maintainer.java
@@ -0,0 +1,73 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.config.server.maintenance;
+
+import com.google.common.util.concurrent.UncheckedTimeoutException;
+import com.yahoo.component.AbstractComponent;
+import com.yahoo.concurrent.DaemonThreadFactory;
+import com.yahoo.path.Path;
+import com.yahoo.vespa.config.server.ApplicationRepository;
+import com.yahoo.vespa.curator.Curator;
+import com.yahoo.vespa.curator.Lock;
+
+import java.time.Duration;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+public abstract class Maintainer extends AbstractComponent implements Runnable {
+
+    protected static final Logger log = Logger.getLogger(Maintainer.class.getName());
+    private static final Path root = Path.fromString("/configserver/v1/");
+    private static final com.yahoo.path.Path lockRoot = root.append("locks");
+
+    private final Duration maintenanceInterval;
+    private final ScheduledExecutorService service;
+    protected final ApplicationRepository applicationRepository;
+    protected final Curator curator;
+
+    Maintainer(ApplicationRepository applicationRepository, Curator curator, Duration interval) {
+        this.applicationRepository = applicationRepository;
+        this.curator = curator;
+        this.maintenanceInterval = interval;
+        service = new ScheduledThreadPoolExecutor(1, new DaemonThreadFactory(name()));
+        service.scheduleAtFixedRate(this, interval.toMillis(), interval.toMillis(), TimeUnit.MILLISECONDS);
+    }
+
+    @Override
+    @SuppressWarnings("try")
+    public void run() {
+        try {
+            Path path = lockRoot.append(name());
+            try (Lock lock = new Lock(path.toString(), curator)) {
+                maintain();
+            }
+        } catch (UncheckedTimeoutException e) {
+            // another config server instance is running this job at the moment; ok
+        } catch (Throwable t) {
+            log.log(Level.WARNING, this + " failed. Will retry in " + maintenanceInterval.toMinutes() + " minutes", t);
+        }
+    }
+
+    @Override
+    public void deconstruct() {
+        this.service.shutdown();
+    }
+
+    /**
+     * Called once each time this maintenance job should run
+     */
+    protected abstract void maintain();
+
+    public String name() { return this.getClass().getSimpleName(); }
+
+    /**
+     * Returns the name of this
+     */
+    @Override
+    public final String toString() {
+        return name();
+    }
+
+}
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainer.java
new file mode 100644
index 00000000000..e06bf530486
--- /dev/null
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainer.java
@@ -0,0 +1,19 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.config.server.maintenance;
+
+import com.yahoo.vespa.config.server.ApplicationRepository;
+import com.yahoo.vespa.curator.Curator;
+
+import java.time.Duration;
+
+public class TenantsMaintainer extends Maintainer {
+
+    public TenantsMaintainer(ApplicationRepository applicationRepository, Curator curator, Duration interval) {
+        super(applicationRepository, curator, interval);
+    }
+
+    @Override
+    protected void maintain() {
+        applicationRepository.removeUnusedTenants();
+    }
+}
diff --git a/configserver/src/main/resources/configserver-app/services.xml b/configserver/src/main/resources/configserver-app/services.xml
index a9e67738d96..b984ce60702 100644
--- a/configserver/src/main/resources/configserver-app/services.xml
+++ b/configserver/src/main/resources/configserver-app/services.xml
@@ -40,6 +40,7 @@
     <component id="com.yahoo.vespa.config.server.application.ApplicationConvergenceChecker" bundle="configserver" />
     <component id="com.yahoo.vespa.config.server.application.HttpProxy" bundle="configserver" />
     <component id="com.yahoo.vespa.config.server.filedistribution.FileServer" bundle="configserver" />
+    <component id="com.yahoo.vespa.config.server.maintenance.ConfigServerMaintenance" bundle="configserver" />
 
     <component id="com.yahoo.vespa.serviceview.ConfigServerLocation" bundle="configserver" />
 
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java
index 90f0b5ee4e5..17cbe41fde5 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java
@@ -27,6 +27,7 @@ import java.time.Clock;
 import java.time.Duration;
 import java.time.Instant;
 
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
@@ -76,18 +77,26 @@ public class ApplicationRepositoryTest {
 
     @Test
     public void createAndPrepareAndActivate() throws IOException {
-        PrepareResult result = createAndPrepareAndActivateApp();
+        PrepareResult result = deployApp();
         assertTrue(result.configChangeActions().getRefeedActions().isEmpty());
         assertTrue(result.configChangeActions().getRestartActions().isEmpty());
     }
 
+    @Test
+    public void deleteUnusedTenants() throws IOException {
+        deployApp();
+        assertTrue(applicationRepository.removeUnusedTenants().isEmpty());
+        applicationRepository.remove(applicationId());
+        assertEquals(tenantName, applicationRepository.removeUnusedTenants().iterator().next());
+    }
+
     private PrepareResult prepareAndActivateApp(File application) throws IOException {
         FilesApplicationPackage appDir = FilesApplicationPackage.fromFile(application);
         long sessionId = applicationRepository.createSession(applicationId(), timeoutBudget, appDir.getAppDir());
         return applicationRepository.prepareAndActivate(tenant, sessionId, prepareParams(), false, false, Instant.now());
     }
 
-    private PrepareResult createAndPrepareAndActivateApp() throws IOException {
+    private PrepareResult deployApp() throws IOException {
         File file = CompressedApplicationInputStreamTest.createTarFile();
         return applicationRepository.deploy(CompressedApplicationInputStream.createFromCompressedStream(
                                                     new FileInputStream(file), ApplicationApiHandler.APPLICATION_X_GZIP),
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TenantHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TenantHandlerTest.java
index 0963e2ea024..35b22d19d6a 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TenantHandlerTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TenantHandlerTest.java
@@ -4,6 +4,7 @@ package com.yahoo.vespa.config.server.http.v2;
 import static org.hamcrest.CoreMatchers.is;
 import static org.junit.Assert.*;
 
+import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.time.Clock;
 
@@ -11,7 +12,14 @@ import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.ApplicationName;
 import com.yahoo.config.provision.InstanceName;
 import com.yahoo.config.provision.TenantName;
+import com.yahoo.vespa.config.server.ApplicationRepository;
+import com.yahoo.vespa.config.server.TestComponentRegistry;
+import com.yahoo.vespa.config.server.http.SessionHandlerTest;
+import com.yahoo.vespa.config.server.http.SessionResponse;
 import com.yahoo.vespa.config.server.tenant.Tenant;
+import com.yahoo.vespa.config.server.tenant.TenantRepository;
+import com.yahoo.vespa.curator.mock.MockCurator;
+import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 
@@ -20,14 +28,23 @@ import com.yahoo.jdisc.http.HttpRequest.Method;
 import com.yahoo.vespa.config.server.http.BadRequestException;
 import com.yahoo.vespa.config.server.http.NotFoundException;
 
-public class TenantHandlerTest extends TenantTest {
+public class TenantHandlerTest {
 
+    private TenantRepository tenantRepository;
     private TenantHandler handler;
     private final TenantName a = TenantName.from("a");
 
     @Before
     public void setup() {
-        handler = new TenantHandler(TenantHandler.testOnlyContext(), tenantRepository);
+        tenantRepository = new TenantRepository(new TestComponentRegistry.Builder().curator(new MockCurator()).build());
+        ApplicationRepository applicationRepository =
+                new ApplicationRepository(tenantRepository, new SessionHandlerTest.MockProvisioner(), Clock.systemUTC());
+        handler = new TenantHandler(TenantHandler.testOnlyContext(), tenantRepository, applicationRepository);
+    }
+
+    @After
+    public void closeTenantRepo() {
+        tenantRepository.close();
     }
 
     @Test
@@ -96,8 +113,8 @@ public class TenantHandlerTest extends TenantTest {
         try {
             handler.handleDELETE(HttpRequest.createTestRequest("http://deploy.example.yahoo.com:80/application/v2/tenant/" + a, Method.DELETE));
             fail();
-        } catch (BadRequestException e) {
-            assertThat(e.getMessage(), is("Cannot delete tenant 'a', as it has active applications: [a.foo]"));
+        } catch (IllegalArgumentException e) {
+            assertThat(e.getMessage(), is("Cannot delete tenant 'a', it has active applications: [a.foo]"));
         }
     }
 
@@ -115,4 +132,10 @@ public class TenantHandlerTest extends TenantTest {
         return (TenantCreateResponse) handler.handlePUT(testRequest);
     }
 
+    private void assertResponseEquals(SessionResponse response, String payload) throws IOException {
+        ByteArrayOutputStream baos = new ByteArrayOutputStream();
+        response.render(baos);
+        assertEquals(baos.toString("UTF-8"), payload);
+    }
+
 }
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TenantTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TenantTest.java
deleted file mode 100644
index 7814266f815..00000000000
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TenantTest.java
+++ /dev/null
@@ -1,46 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.config.server.http.v2;
-
-import static org.junit.Assert.assertEquals;
-
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.util.concurrent.Executor;
-
-import com.yahoo.vespa.config.server.*;
-import com.yahoo.vespa.config.server.http.SessionResponse;
-import com.yahoo.vespa.config.server.tenant.TenantRepository;
-import org.junit.After;
-import org.junit.Before;
-
-/**
- * Supertype for tests in the multi tenant application API
- * 
- * @author Vegard Havdal
- *
- */
-public class TenantTest extends TestWithCurator {
-
-    protected TenantRepository tenantRepository;
-
-    @Before
-    public void setupTenants() {
-        tenantRepository = createTenants();
-    }
-
-    @After
-    public void closeTenants() {
-        tenantRepository.close();
-    }
-
-    private TenantRepository createTenants() {
-        return new TenantRepository(new TestComponentRegistry.Builder().curator(curator).build());
-    }
-
-    void assertResponseEquals(SessionResponse response, String payload) throws IOException {
-        ByteArrayOutputStream baos = new ByteArrayOutputStream();
-        response.render(baos);
-        assertEquals(baos.toString("UTF-8"), payload);
-    }
-
-}