diff options
author | Jon Bratseth <bratseth@oath.com> | 2018-10-18 13:50:42 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2018-10-18 13:50:42 +0200 |
commit | a10b26a6111b71e9ce15e4411fcf1b2459ca89b3 (patch) | |
tree | 4208121a88796b02a9553fd640802a251f9dde96 /configserver | |
parent | f63c1ed7f2f0196be3ccd593cf90104996e8a110 (diff) | |
parent | aa7233fc50ee01653fbf84bf0057c81fb387c4ad (diff) |
Merge pull request #7349 from vespa-engine/musum/limit-number-of-config-models-build-when-adding-node
Musum/limit number of config models build when adding node
Diffstat (limited to 'configserver')
7 files changed, 160 insertions, 117 deletions
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java index 27343e71a87..24309569c9d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java @@ -148,7 +148,7 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> { return allApplicationVersions; // load old model versions - versions = versionsToBuild(versions, wantedNodeVespaVersion, allocatedHosts.get()); + versions = versionsToBuild(versions, wantedNodeVespaVersion, latest.getMajor(), allocatedHosts.get()); // TODO: We use the allocated hosts from the newest version when building older model versions. // This is correct except for the case where an old model specifies a cluster which the new version // does not. In that case we really want to extend the set of allocated hosts to include those of that @@ -169,16 +169,14 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> { return allApplicationVersions; } - private Set<Version> versionsToBuild(Set<Version> versions, com.yahoo.component.Version wantedVersion, AllocatedHosts allocatedHosts) { + private Set<Version> versionsToBuild(Set<Version> versions, com.yahoo.component.Version wantedVersion, int majorVersion, AllocatedHosts allocatedHosts) { if (configserverConfig.buildMinimalSetOfConfigModels()) versions = keepThoseUsedOn(allocatedHosts, versions); // Make sure we build wanted version if we are building models for this major version and we are on hosted vespa // If not on hosted vespa, we do not want to try to build this version, since we have only one version (the latest) - // Also handle the case where there are no allocated hosts in the zone, so versions is empty - Version wanted = Version.fromIntValues(wantedVersion.getMajor(), wantedVersion.getMinor(), wantedVersion.getMicro()); - if (hosted && (versions.isEmpty() || wantedVersion.getMajor() == findLatest(versions).getMajor())) - versions.add(wanted); + if (hosted && wantedVersion.getMajor() == majorVersion) + versions.add(Version.fromIntValues(wantedVersion.getMajor(), wantedVersion.getMinor(), wantedVersion.getMicro())); return versions; } @@ -198,13 +196,13 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> { /** Returns the subset of the given versions which are in use on these hosts */ private Set<Version> keepThoseUsedOn(AllocatedHosts hosts, Set<Version> versions) { - return versions.stream().filter(version -> mayBeUsedOn(hosts, version)).collect(Collectors.toSet()); + return versions.stream().filter(version -> isUsedOn(hosts, version)).collect(Collectors.toSet()); } - private boolean mayBeUsedOn(AllocatedHosts hosts, Version version) { + private boolean isUsedOn(AllocatedHosts hosts, Version version) { com.yahoo.component.Version v = new com.yahoo.component.Version(version.toString()); return hosts.getHosts().stream() - .anyMatch(host -> ! host.version().isPresent() || host.version().get().equals(v)); + .anyMatch(host -> host.version().isPresent() && host.version().get().equals(v)); } protected abstract MODELRESULT buildModelVersion(ModelFactory modelFactory, ApplicationPackage applicationPackage, diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java index 2ae8917e905..ef25effbc64 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java @@ -280,7 +280,7 @@ public class ApplicationRepositoryTest { .configDefinitionsDir(Files.createTempDir().getAbsolutePath()) .sessionLifetime(60)); DeployTester tester = new DeployTester(configserverConfig, clock); - tester.deployApp("src/test/apps/app", "myapp", Instant.now()); // session 2 (numbering starts at 2) + tester.deployApp("src/test/apps/app", clock.instant()); // session 2 (numbering starts at 2) clock.advance(Duration.ofSeconds(10)); Optional<Deployment> deployment2 = tester.redeployFromLocalActive(); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java index 293b8e1e8eb..205b69d1a1e 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java @@ -7,7 +7,6 @@ import com.yahoo.config.model.provision.Host; import com.yahoo.config.model.provision.Hosts; import com.yahoo.config.model.provision.InMemoryProvisioner; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.Version; @@ -57,7 +56,7 @@ public class ConfigServerBootstrapTest { ConfigserverConfig configserverConfig = createConfigserverConfig(temporaryFolder); InMemoryProvisioner provisioner = new InMemoryProvisioner(true, "host0", "host1", "host3"); DeployTester tester = new DeployTester(configserverConfig, provisioner); - tester.deployApp("src/test/apps/hosted/", "myApp", "4.5.6", Instant.now()); + tester.deployApp("src/test/apps/hosted/"); File versionFile = temporaryFolder.newFile(); VersionState versionState = new VersionState(versionFile); @@ -66,12 +65,7 @@ public class ConfigServerBootstrapTest { RpcServer rpcServer = createRpcServer(configserverConfig); VipStatus vipStatus = new VipStatus(); // Take a host away so that there are too few for the application, to verify we can still bootstrap - ClusterSpec contentCluster = ClusterSpec.from(ClusterSpec.Type.content, - ClusterSpec.Id.from("music"), - ClusterSpec.Group.from(0), - new com.yahoo.component.Version(4, 5, 6), - false); - provisioner.allocations().get(contentCluster).remove(0); + provisioner.allocations().values().iterator().next().remove(0); ConfigServerBootstrap bootstrap = new ConfigServerBootstrap(tester.applicationRepository(), rpcServer, versionState, createStateMonitor(), vipStatus); assertFalse(vipStatus.isInRotation()); waitUntil(rpcServer::isRunning, "failed waiting for Rpc server running"); @@ -88,7 +82,7 @@ public class ConfigServerBootstrapTest { public void testBootstrapWhenRedeploymentFails() throws Exception { ConfigserverConfig configserverConfig = createConfigserverConfig(temporaryFolder); DeployTester tester = new DeployTester(configserverConfig); - tester.deployApp("src/test/apps/hosted/", "myApp", "4.5.6", Instant.now()); + tester.deployApp("src/test/apps/hosted/"); File versionFile = temporaryFolder.newFile(); VersionState versionState = new VersionState(versionFile); @@ -123,14 +117,16 @@ public class ConfigServerBootstrapTest { @Test public void testBootstrapNonHostedOneConfigModel() throws Exception { ConfigserverConfig configserverConfig = createConfigserverConfigNonHosted(temporaryFolder); - List<ModelFactory> modelFactories = Collections.singletonList(DeployTester.createModelFactory(Version.fromString("1.2.3"))); - List<Host> hosts = Arrays.asList(createHost("host1", "1.2.3"), createHost("host2", "1.2.3"), createHost("host3", "1.2.3")); + String vespaVersion = "1.2.3"; + List<ModelFactory> modelFactories = Collections.singletonList(DeployTester.createModelFactory(Version.fromString(vespaVersion))); + List<Host> hosts = createHosts(vespaVersion); InMemoryProvisioner provisioner = new InMemoryProvisioner(new Hosts(hosts), true); Curator curator = new MockCurator(); DeployTester tester = new DeployTester(modelFactories, configserverConfig, Clock.systemUTC(), new Zone(Environment.dev, RegionName.defaultName()), provisioner, curator); - ApplicationId app = tester.deployApp("src/test/apps/app/", "myApp", "1.2.3", Instant.now()); + tester.deployApp("src/test/apps/app/", vespaVersion, Instant.now()); + ApplicationId applicationId = tester.applicationId(); File versionFile = temporaryFolder.newFile(); VersionState versionState = new VersionState(versionFile); @@ -139,7 +135,7 @@ public class ConfigServerBootstrapTest { // Ugly hack, but I see no other way of doing it: // Manipulate application version in zookeeper so that it is an older version than the model we know, which is // the case when upgrading on non-hosted installations - curator.set(Path.fromString("/config/v2/tenants/" + app.tenant().value() + "/sessions/2/version"), Utf8.toBytes("1.2.2")); + curator.set(Path.fromString("/config/v2/tenants/" + applicationId.tenant().value() + "/sessions/2/version"), Utf8.toBytes("1.2.2")); RpcServer rpcServer = createRpcServer(configserverConfig); VipStatus vipStatus = new VipStatus(); @@ -189,6 +185,10 @@ public class ConfigServerBootstrapTest { .sleepTimeWhenRedeployingFails(0)); /* seconds */ } + private List<Host> createHosts(String vespaVersion) { + return Arrays.asList(createHost("host1", vespaVersion), createHost("host2", vespaVersion), createHost("host3", vespaVersion)); + } + private Host createHost(String hostname, String version) { return new Host(hostname, Collections.emptyList(), Optional.empty(), Optional.of(com.yahoo.component.Version.fromString(version))); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java index b70947de0a6..0d2654d693e 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java @@ -15,12 +15,10 @@ import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.provision.InMemoryProvisioner; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.HostSpec; -import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.ProvisionLogger; import com.yahoo.config.provision.Provisioner; @@ -31,6 +29,7 @@ import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.TestComponentRegistry; import com.yahoo.vespa.config.server.TimeoutBudget; +import com.yahoo.vespa.config.server.http.v2.PrepareResult; import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry; import com.yahoo.vespa.config.server.monitoring.Metrics; import com.yahoo.vespa.config.server.session.LocalSession; @@ -60,13 +59,12 @@ import java.util.Optional; public class DeployTester { private static final TenantName tenantName = TenantName.from("deploytester"); + private static final ApplicationId applicationId = ApplicationId.from(tenantName.value(), "myApp", "default"); private final Clock clock; private final TenantRepository tenantRepository; private final ApplicationRepository applicationRepository; - private ApplicationId id; - public DeployTester() { this(Collections.singletonList(createModelFactory(Clock.systemUTC()))); } @@ -148,30 +146,33 @@ public class DeployTester { /** Create a model factory which always fails validation */ public static ModelFactory createFailingModelFactory(Version version) { return new FailingModelFactory(version); } - + + /** * Do the initial "deploy" with the existing API-less code as the deploy API doesn't support first deploys yet. */ - public ApplicationId deployApp(String applicationPath, String appName, Instant now) { - return deployApp(applicationPath, appName, null, now); + public PrepareResult deployApp(String applicationPath) { + return deployApp(applicationPath, null, Instant.now()); } /** * Do the initial "deploy" with the existing API-less code as the deploy API doesn't support first deploys yet. */ - public ApplicationId deployApp(String applicationPath, String appName, String vespaVersion, Instant now) { - Tenant tenant = tenant(); - TimeoutBudget timeoutBudget = new TimeoutBudget(clock, Duration.ofSeconds(60)); - ApplicationId id = ApplicationId.from(tenant.getName(), ApplicationName.from(appName), InstanceName.defaultName()); - PrepareParams.Builder paramsBuilder = new PrepareParams.Builder().applicationId(id); + public PrepareResult deployApp(String applicationPath, Instant now) { + return deployApp(applicationPath, null, now); + } + + /** + * Do the initial "deploy" with the existing API-less code as the deploy API doesn't support first deploys yet. + */ + public PrepareResult deployApp(String applicationPath, String vespaVersion, Instant now) { + PrepareParams.Builder paramsBuilder = new PrepareParams.Builder() + .applicationId(applicationId) + .timeoutBudget(new TimeoutBudget(clock, Duration.ofSeconds(60))); if (vespaVersion != null) paramsBuilder.vespaVersion(vespaVersion); - long sessionId = applicationRepository.createSession(id, timeoutBudget, new File(applicationPath)); - applicationRepository.prepare(tenant, sessionId, paramsBuilder.build(), now); - applicationRepository.activate(tenant, sessionId, timeoutBudget, false, false); - this.id = id; - return id; + return applicationRepository.deploy(new File(applicationPath), paramsBuilder.build(), false, false, now); } public AllocatedHosts getAllocatedHostsOf(ApplicationId applicationId) { @@ -181,10 +182,10 @@ public class DeployTester { return session.getAllocatedHosts(); } - public ApplicationId applicationId() { return id; } + public ApplicationId applicationId() { return applicationId; } public Optional<com.yahoo.config.provision.Deployment> redeployFromLocalActive() { - return redeployFromLocalActive(id); + return applicationRepository.deployFromLocalActive(applicationId); } public Optional<com.yahoo.config.provision.Deployment> redeployFromLocalActive(ApplicationId id) { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java index 02059513dff..e2a2bcd830e 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java @@ -1,9 +1,15 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.deploy; +import com.google.common.collect.ImmutableSet; import com.google.common.io.Files; import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.config.model.api.ConfigChangeAction; +import com.yahoo.config.model.api.ModelContext; +import com.yahoo.config.model.api.ModelCreateResult; import com.yahoo.config.model.api.ModelFactory; +import com.yahoo.config.model.api.ServiceInfo; +import com.yahoo.config.model.api.ValidationParameters; import com.yahoo.config.model.provision.Host; import com.yahoo.config.model.provision.Hosts; import com.yahoo.config.model.provision.InMemoryProvisioner; @@ -13,8 +19,11 @@ import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.Version; import com.yahoo.config.provision.Zone; import com.yahoo.test.ManualClock; -import static com.yahoo.vespa.config.server.deploy.DeployTester.CountingModelFactory; +import com.yahoo.vespa.config.server.configchange.MockRestartAction; +import com.yahoo.vespa.config.server.configchange.RestartActions; +import com.yahoo.vespa.config.server.http.v2.PrepareResult; +import com.yahoo.vespa.config.server.model.TestModelFactory; import com.yahoo.vespa.config.server.session.LocalSession; import org.junit.Test; @@ -24,13 +33,18 @@ import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.IntStream; +import static com.yahoo.vespa.config.server.deploy.DeployTester.CountingModelFactory; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -43,9 +57,9 @@ public class HostedDeployTest { public void testRedeployWithVersion() { CountingModelFactory modelFactory = DeployTester.createModelFactory(Version.fromString("4.5.6"), Clock.systemUTC()); DeployTester tester = new DeployTester(Collections.singletonList(modelFactory), createConfigserverConfig()); - tester.deployApp("src/test/apps/hosted/", "myApp", "4.5.6", Instant.now()); + tester.deployApp("src/test/apps/hosted/", "4.5.6", Instant.now()); - Optional<com.yahoo.config.provision.Deployment> deployment = tester.redeployFromLocalActive(); + Optional<com.yahoo.config.provision.Deployment> deployment = tester.redeployFromLocalActive(tester.applicationId()); assertTrue(deployment.isPresent()); deployment.get().prepare(); deployment.get().activate(); @@ -55,7 +69,8 @@ public class HostedDeployTest { @Test public void testRedeploy() { DeployTester tester = new DeployTester(createConfigserverConfig()); - ApplicationId appId = tester.deployApp("src/test/apps/hosted/", "myApp", Instant.now()); + ApplicationId appId = tester.applicationId(); + tester.deployApp("src/test/apps/hosted/"); LocalSession s1 = tester.applicationRepository().getActiveSession(appId); System.out.println("First session: " + s1.getSessionId()); assertFalse(tester.applicationRepository().getActiveSession(appId).getMetaData().isInternalRedeploy()); @@ -72,54 +87,77 @@ public class HostedDeployTest { @Test public void testDeployMultipleVersions() { ManualClock clock = new ManualClock("2016-10-09T00:00:00"); - List<ModelFactory> modelFactories = new ArrayList<>(); - modelFactories.add(DeployTester.createModelFactory(Version.fromString("6.1.0"), clock)); - modelFactories.add(DeployTester.createModelFactory(Version.fromString("6.2.0"), clock)); - modelFactories.add(DeployTester.createModelFactory(Version.fromString("7.0.0"), clock)); + List<ModelFactory> modelFactories = + Arrays.asList(DeployTester.createModelFactory(Version.fromString("6.1.0"), clock), + DeployTester.createModelFactory(Version.fromString("6.2.0"), clock), + DeployTester.createModelFactory(Version.fromString("7.0.0"), clock)); DeployTester tester = new DeployTester(modelFactories, createConfigserverConfig(), clock, Zone.defaultZone()); - ApplicationId app = tester.deployApp("src/test/apps/hosted/", "myApp", "6.2.0", Instant.now()); - assertEquals(3, tester.getAllocatedHostsOf(app).getHosts().size()); + tester.deployApp("src/test/apps/hosted/", "6.2.0", Instant.now()); + assertEquals(3, tester.getAllocatedHostsOf(tester.applicationId()).getHosts().size()); } /** Test that only the minimal set of models are created (model versions used on hosts, the wanted version and the latest version) */ @Test public void testCreateOnlyNeededModelVersions() { - List<Host> hosts = new ArrayList<>(); - hosts.add(createHost("host1", "6.0.0")); - hosts.add(createHost("host2", "6.0.2")); - hosts.add(createHost("host3", "7.1.0")); + List<Host> hosts = Arrays.asList(createHost("host1", "6.0.0"), + createHost("host2", "6.2.0"), + createHost("host3")); //Use a host with no version as well InMemoryProvisioner provisioner = new InMemoryProvisioner(new Hosts(hosts), true); - ManualClock clock = new ManualClock("2016-10-09T00:00:00"); - CountingModelFactory factory600 = DeployTester.createModelFactory(Version.fromString("6.0.0"), clock); - CountingModelFactory factory610 = DeployTester.createModelFactory(Version.fromString("6.1.0"), clock); - CountingModelFactory factory620 = DeployTester.createModelFactory(Version.fromString("6.2.0"), clock); - CountingModelFactory factory700 = DeployTester.createModelFactory(Version.fromString("7.0.0"), clock); - CountingModelFactory factory710 = DeployTester.createModelFactory(Version.fromString("7.1.0"), clock); - CountingModelFactory factory720 = DeployTester.createModelFactory(Version.fromString("7.2.0"), clock); - List<ModelFactory> modelFactories = new ArrayList<>(); - modelFactories.add(factory600); - modelFactories.add(factory610); - modelFactories.add(factory620); - modelFactories.add(factory700); - modelFactories.add(factory710); - modelFactories.add(factory720); + CountingModelFactory factory600 = DeployTester.createModelFactory(Version.fromString("6.0.0")); + CountingModelFactory factory610 = DeployTester.createModelFactory(Version.fromString("6.1.0")); + CountingModelFactory factory620 = DeployTester.createModelFactory(Version.fromString("6.2.0")); + CountingModelFactory factory700 = DeployTester.createModelFactory(Version.fromString("7.0.0")); + CountingModelFactory factory710 = DeployTester.createModelFactory(Version.fromString("7.1.0")); + CountingModelFactory factory720 = DeployTester.createModelFactory(Version.fromString("7.2.0")); + List<ModelFactory> modelFactories = Arrays.asList(factory600, + factory610, + factory620, + factory700, + factory710, + factory720); - DeployTester tester = new DeployTester(modelFactories, createConfigserverConfig(), - clock, new Zone(Environment.dev, RegionName.defaultName()), provisioner); + DeployTester tester = new DeployTester(modelFactories, createConfigserverConfig(), Clock.systemUTC(), provisioner); // Deploy with version that does not exist on hosts, the model for this version should also be created - ApplicationId app = tester.deployApp("src/test/apps/hosted/", "myApp", "7.0.0", Instant.now()); - assertEquals(3, tester.getAllocatedHostsOf(app).getHosts().size()); + tester.deployApp("src/test/apps/hosted/", "7.0.0", Instant.now()); + assertEquals(3, tester.getAllocatedHostsOf(tester.applicationId()).getHosts().size()); // Check >0 not ==0 as the session watcher thread is running and will redeploy models in the background assertTrue(factory600.creationCount() > 0); assertFalse(factory610.creationCount() > 0); assertTrue(factory620.creationCount() > 0); assertTrue(factory700.creationCount() > 0); - assertTrue(factory710.creationCount() > 0); + assertFalse(factory710.creationCount() > 0); assertTrue("Newest is always included", factory720.creationCount() > 0); } + + /** Test that only the minimal set of models are created (the wanted version and the latest version per major, since nodes are without version) */ + @Test + public void testCreateOnlyNeededModelVersionsNewNodes() { + List<Host> hosts = Arrays.asList(createHost("host1"), createHost("host2"), createHost("host3")); + InMemoryProvisioner provisioner = new InMemoryProvisioner(new Hosts(hosts), true); + + CountingModelFactory factory600 = DeployTester.createModelFactory(Version.fromString("6.0.0")); + CountingModelFactory factory610 = DeployTester.createModelFactory(Version.fromString("6.1.0")); + CountingModelFactory factory700 = DeployTester.createModelFactory(Version.fromString("7.0.0")); + CountingModelFactory factory710 = DeployTester.createModelFactory(Version.fromString("7.1.0")); + CountingModelFactory factory720 = DeployTester.createModelFactory(Version.fromString("7.2.0")); + List<ModelFactory> modelFactories = Arrays.asList(factory600, factory610, factory700, factory710, factory720); + + DeployTester tester = new DeployTester(modelFactories, createConfigserverConfig(), Clock.systemUTC(), provisioner); + // Deploy with version that does not exist on hosts, the model for this version should also be created + tester.deployApp("src/test/apps/hosted/", "7.0.0", Instant.now()); + assertEquals(3, tester.getAllocatedHostsOf(tester.applicationId()).getHosts().size()); + + // Check >0 not ==0 as the session watcher thread is running and will redeploy models in the background + assertFalse(factory600.creationCount() > 0); // latest on major version 6 + assertTrue("Newest per major version is always included", factory610.creationCount() > 0); + assertTrue(factory700.creationCount() > 0); + assertFalse(factory710.creationCount() > 0); + assertTrue("Newest per major version is always included", factory720.creationCount() > 0); + } + /** * Test that deploying an application works when there are no allocated hosts in the system * (the bootstrap a new zone case, so deploying the routing app since that is the first deployment @@ -133,13 +171,12 @@ public class HostedDeployTest { CountingModelFactory factory700 = DeployTester.createModelFactory(Version.fromString("7.0.0"), clock); CountingModelFactory factory720 = DeployTester.createModelFactory(Version.fromString("7.2.0"), clock); - List<ModelFactory> modelFactories = new ArrayList<>(); - modelFactories.add(factory700); - modelFactories.add(factory720); + List<ModelFactory> modelFactories = Arrays.asList(factory700, factory720); DeployTester tester = new DeployTester(modelFactories, createConfigserverConfig(), clock, new Zone(Environment.dev, RegionName.defaultName()), provisioner); - tester.deployApp("src/test/apps/hosted-routing-app/", "myApp", "7.2.0", Instant.now()); + tester.deployApp("src/test/apps/hosted-routing-app/", "7.2.0", Instant.now()); + assertFalse(factory700.creationCount() > 0); assertTrue("Newest is always included", factory720.creationCount() > 0); } @@ -156,14 +193,15 @@ public class HostedDeployTest { DeployTester tester = new DeployTester(modelFactories, createConfigserverConfig(), Clock.systemUTC(), new Zone(Environment.prod, RegionName.defaultName()), provisioner); + ApplicationId applicationId = tester.applicationId(); // Deploy with oldest version - ApplicationId app = tester.deployApp("src/test/apps/hosted/", "myApp", "6.0.0", Instant.now()); - assertEquals(3, tester.getAllocatedHostsOf(app).getHosts().size()); + tester.deployApp("src/test/apps/hosted/", "6.0.0", Instant.now()); + assertEquals(3, tester.getAllocatedHostsOf(applicationId).getHosts().size()); // Deploy with version that does not exist on hosts and with app package that has no write access control, // validation of access control should not be done, since the app is already deployed in prod - app = tester.deployApp("src/test/apps/hosted-no-write-access-control", "myApp", "6.1.0", Instant.now()); - assertEquals(3, tester.getAllocatedHostsOf(app).getHosts().size()); + tester.deployApp("src/test/apps/hosted-no-write-access-control", "6.1.0", Instant.now()); + assertEquals(3, tester.getAllocatedHostsOf(applicationId).getHosts().size()); } @Test @@ -174,7 +212,7 @@ public class HostedDeployTest { modelFactories.add(DeployTester.createModelFactory(clock)); modelFactories.add(DeployTester.createFailingModelFactory(Version.fromIntValues(1, 0, 0))); // older than default DeployTester tester = new DeployTester(modelFactories, createConfigserverConfig()); - tester.deployApp("src/test/apps/validationOverride/", "myApp", clock.instant()); + tester.deployApp("src/test/apps/validationOverride/", clock.instant()); // Redeployment from local active works { @@ -206,6 +244,28 @@ public class HostedDeployTest { } } + @Test + public void testThatConfigChangeActionsAreCollectedFromAllModels() { + List<Host> hosts = Arrays.asList(createHost("host1", "6.1.0"), + createHost("host2", "6.2.0"), + createHost("host3", "6.2.0")); + InMemoryProvisioner provisioner = new InMemoryProvisioner(new Hosts(hosts), true); + List<ServiceInfo> services = Collections.singletonList( + new ServiceInfo("serviceName", "serviceType", null, new HashMap<>(), "configId", "hostName")); + + List<ModelFactory> modelFactories = Arrays.asList( + new ConfigChangeActionsModelFactory(Version.fromIntValues(6, 1, 0), new MockRestartAction("change", services)), + new ConfigChangeActionsModelFactory(Version.fromIntValues(6, 2, 0), new MockRestartAction("other change", services))); + + DeployTester tester = new DeployTester(modelFactories, createConfigserverConfig(), Clock.systemUTC(), provisioner); + PrepareResult prepareResult = tester.deployApp("src/test/apps/hosted/", "6.2.0", Instant.now()); + + assertEquals(3, tester.getAllocatedHostsOf(tester.applicationId()).getHosts().size()); + List<RestartActions.Entry> actions = prepareResult.configChangeActions().getRestartActions().getEntries(); + assertThat(actions.size(), is(1)); + assertThat(actions.get(0).getMessages(), equalTo(ImmutableSet.of("change", "other change"))); + } + private static ConfigserverConfig createConfigserverConfig() { return new ConfigserverConfig(new ConfigserverConfig.Builder() .configServerDBDir(Files.createTempDir().getAbsolutePath()) @@ -222,4 +282,20 @@ public class HostedDeployTest { return new Host(hostname, Collections.emptyList(), Optional.empty(), Optional.empty()); } + private static class ConfigChangeActionsModelFactory extends TestModelFactory { + + private final ConfigChangeAction action; + + ConfigChangeActionsModelFactory(Version vespaVersion, ConfigChangeAction action) { + super(vespaVersion); + this.action = action; + } + + @Override + public ModelCreateResult createAndValidateModel(ModelContext modelContext, ValidationParameters validationParameters) { + ModelCreateResult result = super.createAndValidateModel(modelContext, validationParameters); + return new ModelCreateResult(result.getModel(), Arrays.asList(action)); + } + } + } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/RedeployTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/RedeployTest.java index c6f4df74049..d3380157fb4 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/RedeployTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/RedeployTest.java @@ -28,7 +28,7 @@ public class RedeployTest { @Test public void testRedeploy() { DeployTester tester = new DeployTester(); - tester.deployApp("src/test/apps/app", "myapp", Instant.now()); + tester.deployApp("src/test/apps/app"); Optional<com.yahoo.config.provision.Deployment> deployment = tester.redeployFromLocalActive(); assertTrue(deployment.isPresent()); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java index a221a496e0c..07e61048611 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java @@ -1,12 +1,9 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.session; -import com.google.common.collect.ImmutableSet; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.config.model.api.ConfigChangeAction; import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.api.ModelCreateResult; -import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.model.api.ValidationParameters; import com.yahoo.config.model.application.provider.*; import com.yahoo.config.provision.ApplicationName; @@ -23,8 +20,6 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.config.server.application.MemoryTenantApplications; import com.yahoo.vespa.config.server.application.PermanentApplicationPackage; import com.yahoo.vespa.config.server.configchange.ConfigChangeActions; -import com.yahoo.vespa.config.server.configchange.MockRestartAction; -import com.yahoo.vespa.config.server.configchange.RestartActions; import com.yahoo.vespa.config.server.deploy.DeployHandlerLogger; import com.yahoo.vespa.config.server.host.HostRegistry; import com.yahoo.vespa.config.server.http.InvalidApplicationException; @@ -45,7 +40,6 @@ import java.io.IOException; import java.time.Instant; import java.util.*; -import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.Matchers.contains; import static org.junit.Assert.*; @@ -60,7 +54,6 @@ public class SessionPreparerTest { private static final File testApp = new File("src/test/apps/app"); private static final File invalidTestApp = new File("src/test/apps/illegalApp"); private static final Version version123 = Version.fromIntValues(1, 2, 3); - private static final Version version124 = Version.fromIntValues(1, 2, 4); private static final Version version321 = Version.fromIntValues(3, 2, 1); private static final Version version323 = Version.fromIntValues(3, 2, 3); @@ -186,18 +179,6 @@ public class SessionPreparerTest { assertThat(zkc.readApplicationId(), is(origId)); } - @Test - public void require_that_config_change_actions_are_collected_from_all_models() throws IOException { - ServiceInfo service = new ServiceInfo("serviceName", "serviceType", null, new HashMap<>(), "configId", "hostName"); - ModelFactoryRegistry modelFactoryRegistry = new ModelFactoryRegistry(Arrays.asList( - new ConfigChangeActionsModelFactory(version123, new MockRestartAction("change", Arrays.asList(service))), - new ConfigChangeActionsModelFactory(version124, new MockRestartAction("other change", Arrays.asList(service))))); - preparer = createPreparer(modelFactoryRegistry, HostProvisionerProvider.empty()); - List<RestartActions.Entry> actions = prepare(testApp).getRestartActions().getEntries(); - assertThat(actions.size(), is(1)); - assertThat(actions.get(0).getMessages(), equalTo(ImmutableSet.of("change", "other change"))); - } - private Set<Rotation> readRotationsFromZK(ApplicationId applicationId) { return new Rotations(curator, tenantPath).readRotationsFromZooKeeper(applicationId); } @@ -278,17 +259,4 @@ public class SessionPreparerTest { ApplicationName.from(applicationName), InstanceName.defaultName()); } - private static class ConfigChangeActionsModelFactory extends TestModelFactory { - private final ConfigChangeAction action; - ConfigChangeActionsModelFactory(Version vespaVersion, ConfigChangeAction action) { - super(vespaVersion); - this.action = action; - } - - @Override - public ModelCreateResult createAndValidateModel(ModelContext modelContext, ValidationParameters validationParameters) { - ModelCreateResult result = super.createAndValidateModel(modelContext, validationParameters); - return new ModelCreateResult(result.getModel(), Arrays.asList(action)); - } - } } |