diff options
author | Harald Musum <musum@oath.com> | 2018-05-24 15:38:57 +0200 |
---|---|---|
committer | Harald Musum <musum@oath.com> | 2018-05-24 15:38:57 +0200 |
commit | a3f9833302739e654ccf985d0379b28420b6e18c (patch) | |
tree | e3f5fa35477e3797eab092e13396797a626e1ddf | |
parent | 7a5eb504b4ce7b6bdcd0e0e613bcb2287fd358f1 (diff) |
Add zookeeper data maintainer (removes unused file distribution data)
7 files changed, 102 insertions, 16 deletions
diff --git a/configdefinitions/src/vespa/configserver.def b/configdefinitions/src/vespa/configserver.def index 7a1ddeb8f66..5e81526dc53 100644 --- a/configdefinitions/src/vespa/configserver.def +++ b/configdefinitions/src/vespa/configserver.def @@ -52,5 +52,6 @@ ztsUrl string default="" nodeAdminInContainer bool default=true # Maintainers +maintainerIntervalMinutes int default=30 # TODO: Default set to a high value (1 year) => maintainer will not run, change when maintainer verified out in prod tenantsMaintainerIntervalMinutes int default=525600 diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java index 3e8e5ac11df..e87d88c5ed0 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java @@ -11,20 +11,27 @@ import java.time.Duration; public class ConfigServerMaintenance extends AbstractComponent { + private static final Duration intervalInCd = Duration.ofMinutes(5); + private final TenantsMaintainer tenantsMaintainer; + private final ZooKeeperDataMaintainer zooKeeperDataMaintainer; @SuppressWarnings("unused") // instantiated by Dependency Injection public ConfigServerMaintenance(ConfigserverConfig configserverConfig, ApplicationRepository applicationRepository, Curator curator) { - Duration interval = configserverConfig.system().equals(SystemName.cd.name()) ? Duration.ofMinutes(5) : - Duration.ofMinutes(configserverConfig.tenantsMaintainerIntervalMinutes()); - tenantsMaintainer = new TenantsMaintainer(applicationRepository, curator, interval); + boolean isCd = configserverConfig.system().equals(SystemName.cd.name()); + Duration defaultInterval = isCd ? intervalInCd : Duration.ofMinutes(configserverConfig.maintainerIntervalMinutes()); + Duration tenantsMaintainerInterval = isCd ? intervalInCd : Duration.ofMinutes(configserverConfig.tenantsMaintainerIntervalMinutes()); + + tenantsMaintainer = new TenantsMaintainer(applicationRepository, curator, tenantsMaintainerInterval); + zooKeeperDataMaintainer = new ZooKeeperDataMaintainer(applicationRepository, curator, defaultInterval); } @Override public void deconstruct() { tenantsMaintainer.deconstruct(); + zooKeeperDataMaintainer.deconstruct(); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainer.java index e06bf530486..36306dbdde8 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainer.java @@ -8,7 +8,7 @@ import java.time.Duration; public class TenantsMaintainer extends Maintainer { - public TenantsMaintainer(ApplicationRepository applicationRepository, Curator curator, Duration interval) { + TenantsMaintainer(ApplicationRepository applicationRepository, Curator curator, Duration interval) { super(applicationRepository, curator, interval); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ZooKeeperDataMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ZooKeeperDataMaintainer.java new file mode 100644 index 00000000000..852768b6937 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ZooKeeperDataMaintainer.java @@ -0,0 +1,23 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + package com.yahoo.vespa.config.server.maintenance; + +import com.yahoo.path.Path; +import com.yahoo.vespa.config.server.ApplicationRepository; +import com.yahoo.vespa.curator.Curator; + +import java.time.Duration; + +/** + * Removes unused zookeeper data (for now only data used by old file distribution code is removed) + */ +public class ZooKeeperDataMaintainer extends Maintainer { + + ZooKeeperDataMaintainer(ApplicationRepository applicationRepository, Curator curator, Duration interval) { + super(applicationRepository, curator, interval); + } + + @Override + protected void maintain() { + curator.delete(Path.fromString("/vespa/filedistribution")); + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/MaintainerTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/MaintainerTester.java new file mode 100644 index 00000000000..b92feffbb55 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/MaintainerTester.java @@ -0,0 +1,32 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.maintenance; + +import com.yahoo.vespa.config.server.ApplicationRepository; +import com.yahoo.vespa.config.server.GlobalComponentRegistry; +import com.yahoo.vespa.config.server.TestComponentRegistry; +import com.yahoo.vespa.config.server.http.SessionHandlerTest; +import com.yahoo.vespa.config.server.tenant.TenantRepository; +import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.curator.mock.MockCurator; + +import java.time.Clock; + +class MaintainerTester { + + private final Curator curator; + private final TenantRepository tenantRepository; + private final ApplicationRepository applicationRepository; + + MaintainerTester() { + curator = new MockCurator(); + GlobalComponentRegistry componentRegistry = new TestComponentRegistry.Builder().curator(curator).build(); + tenantRepository = new TenantRepository(componentRegistry, false); + applicationRepository = new ApplicationRepository(tenantRepository, new SessionHandlerTest.MockProvisioner(), Clock.systemUTC()); + } + + Curator curator() { return curator; } + TenantRepository tenantRepository() { return tenantRepository; } + + ApplicationRepository applicationRepository() { return applicationRepository;} + +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainerTest.java index 9d2bf832a63..80d9f808bfc 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/TenantsMaintainerTest.java @@ -6,17 +6,11 @@ import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.TenantName; import com.yahoo.vespa.config.server.ApplicationRepository; -import com.yahoo.vespa.config.server.GlobalComponentRegistry; -import com.yahoo.vespa.config.server.TestComponentRegistry; -import com.yahoo.vespa.config.server.http.SessionHandlerTest; import com.yahoo.vespa.config.server.session.PrepareParams; import com.yahoo.vespa.config.server.tenant.TenantRepository; -import com.yahoo.vespa.curator.Curator; -import com.yahoo.vespa.curator.mock.MockCurator; import org.junit.Test; import java.io.File; -import java.time.Clock; import java.time.Duration; import static org.junit.Assert.assertNotNull; @@ -26,11 +20,10 @@ public class TenantsMaintainerTest { @Test public void deleteTenantWithNoApplications() { - Curator curator = new MockCurator(); - GlobalComponentRegistry componentRegistry = new TestComponentRegistry.Builder().curator(curator).build(); - TenantRepository tenantRepository = new TenantRepository(componentRegistry, false); - ApplicationRepository applicationRepository = - new ApplicationRepository(tenantRepository, new SessionHandlerTest.MockProvisioner(), Clock.systemUTC()); + MaintainerTester tester = new MaintainerTester(); + TenantRepository tenantRepository = tester.tenantRepository(); + ApplicationRepository applicationRepository = tester.applicationRepository(); + TenantName shouldBeDeleted = TenantName.from("to-be-deleted"); TenantName shouldNotBeDeleted = TenantName.from("should-not-be-deleted"); @@ -43,7 +36,7 @@ public class TenantsMaintainerTest { assertNotNull(tenantRepository.getTenant(shouldBeDeleted)); assertNotNull(tenantRepository.getTenant(shouldNotBeDeleted)); - new TenantsMaintainer(applicationRepository, curator, Duration.ofDays(1)).run(); + new TenantsMaintainer(applicationRepository, tester.curator(), Duration.ofDays(1)).run(); // One tenant should now have been deleted assertNull(tenantRepository.getTenant(shouldBeDeleted)); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/ZooKeeperDataMaintainerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/ZooKeeperDataMaintainerTest.java new file mode 100644 index 00000000000..1c886adde73 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/ZooKeeperDataMaintainerTest.java @@ -0,0 +1,30 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.maintenance; + +import com.yahoo.path.Path; +import com.yahoo.vespa.curator.Curator; +import org.junit.Test; + +import java.time.Duration; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class ZooKeeperDataMaintainerTest { + + @Test + public void deleteOldData() { + MaintainerTester tester = new MaintainerTester(); + Curator curator = tester.curator(); + + curator.create(Path.fromString("/foo")); + curator.create(Path.fromString("/vespa/bar")); + curator.create(Path.fromString("/vespa/filedistribution")); + + new ZooKeeperDataMaintainer(tester.applicationRepository(), curator, Duration.ofDays(1)).run(); + + assertTrue(curator.exists(Path.fromString("/foo"))); + assertTrue(curator.exists(Path.fromString("/vespa"))); + assertFalse(curator.exists(Path.fromString("/vespa/filedistribution"))); + } +} |