diff options
Diffstat (limited to 'configserver')
21 files changed, 161 insertions, 73 deletions
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 1bfe14566f7..94d758237de 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -270,10 +270,12 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye */ public boolean delete(ApplicationId applicationId) { // TODO: Use deleteApplication() in all zones - if ( ! configserverConfig.hostedVespa() || SystemName.from(configserverConfig.system()) == SystemName.cd) { - return deleteApplication(applicationId); - } else { + if (configserverConfig.deleteApplicationLegacy() || + (configserverConfig.hostedVespa() && SystemName.from(configserverConfig.system()) == SystemName.main + && !Arrays.asList("corp-us-east-1", "aws-us-east-1a").contains(configserverConfig.region()))) { return deleteApplicationLegacy(applicationId); + } else { + return deleteApplication(applicationId); } } @@ -648,6 +650,9 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye Set<ApplicationId> applicationsNotRedeployed = listApplications(); do { applicationsNotRedeployed = redeployApplications(applicationsNotRedeployed); + if ( ! applicationsNotRedeployed.isEmpty()) { + Thread.sleep(Duration.ofSeconds(30).toMillis()); + } } while ( ! applicationsNotRedeployed.isEmpty() && Instant.now().isBefore(end)); if ( ! applicationsNotRedeployed.isEmpty()) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 28dc0cc8414..914d6963ff0 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -124,6 +124,7 @@ public class ModelContextImpl implements ModelContext { private final Zone zone; private final Set<Rotation> rotations; private final boolean isBootstrap; + private final boolean isFirstTimeDeployment; public Properties(ApplicationId applicationId, boolean multitenant, @@ -134,7 +135,8 @@ public class ModelContextImpl implements ModelContext { boolean hostedVespa, Zone zone, Set<Rotation> rotations, - boolean isBootstrap) { + boolean isBootstrap, + boolean isFirstTimeDeployment) { this.applicationId = applicationId; this.multitenant = multitenant; this.configServerSpecs = configServerSpecs; @@ -145,6 +147,7 @@ public class ModelContextImpl implements ModelContext { this.zone = zone; this.rotations = rotations; this.isBootstrap = isBootstrap; + this.isFirstTimeDeployment = isFirstTimeDeployment; } @Override @@ -181,6 +184,8 @@ public class ModelContextImpl implements ModelContext { @Override public boolean isBootstrap() { return isBootstrap; } + @Override + public boolean isFirstTimeDeployment() { return isFirstTimeDeployment; } } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java index 314b6d78ae3..2955d948d69 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java @@ -116,7 +116,8 @@ public class ActivatedModelsBuilder extends ModelsBuilder<Application> { configserverConfig.hostedVespa(), zone(), new Rotations(curator, TenantRepository.getTenantPath(tenant)).readRotationsFromZooKeeper(applicationId), - false); // We may be bootstrapping, but we only know and care during prepare + false, // We may be bootstrapping, but we only know and care during prepare + false); // Always false, assume no one uses it when activating } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java index 969a2d49586..090e7aac494 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java @@ -8,6 +8,7 @@ import com.yahoo.config.model.api.ModelFactory; import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationLockException; +import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.OutOfCapacityException; import com.yahoo.config.provision.Version; import com.yahoo.config.provision.Zone; @@ -19,6 +20,7 @@ import com.yahoo.vespa.config.server.provision.StaticProvisioner; import java.time.Instant; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.Comparator; import java.util.List; @@ -140,7 +142,10 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> { List<MODELRESULT> allApplicationVersions = new ArrayList<>(); allApplicationVersions.add(latestModelVersion); - versions = keepThoseUsedOn(allocatedHosts.get(), versions); + // TODO: Enable for all zones + // Note: Cannot be enabled for prod zones yet, due to an issue with how AccessControlValidator works + if (Arrays.asList(Environment.dev, Environment.test, Environment.staging).contains(zone().environment())) + versions = keepThoseUsedOn(allocatedHosts.get(), versions); // Make sure we build wanted version if we are building models for this major version Version wantedVersion = Version.fromIntValues(wantedNodeVespaVersion.getMajor(), wantedNodeVespaVersion.getMinor(), wantedNodeVespaVersion.getMicro()); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java index 0d9346101e9..ba093ca9db1 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java @@ -10,6 +10,8 @@ import com.yahoo.config.model.api.Model; import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.api.ModelCreateResult; import com.yahoo.config.model.api.ModelFactory; +import com.yahoo.config.model.api.ValidationParameters; +import com.yahoo.config.model.api.ValidationParameters.IgnoreValidationErrors; import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.AllocatedHosts; @@ -104,7 +106,9 @@ public class PreparedModelsBuilder extends ModelsBuilder<PreparedModelsBuilder.P wantedNodeVespaVersion); log.log(LogLevel.DEBUG, "Create and validate model " + modelVersion + " for " + applicationId); - ModelCreateResult result = modelFactory.createAndValidateModel(modelContext, params.ignoreValidationErrors()); + ValidationParameters validationParameters = + new ValidationParameters(params.ignoreValidationErrors() ? IgnoreValidationErrors.TRUE : IgnoreValidationErrors.FALSE); + ModelCreateResult result = modelFactory.createAndValidateModel(modelContext, validationParameters); validateModelHosts(context.getHostValidator(), applicationId, result.getModel()); log.log(LogLevel.DEBUG, "Done building model " + modelVersion + " for " + applicationId); return new PreparedModelsBuilder.PreparedModelResult(modelVersion, result.getModel(), fileDistributionProvider, result.getConfigChangeActions()); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java index 0d9f8ce64b1..6a240806004 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java @@ -158,7 +158,8 @@ public class SessionPreparer { configserverConfig.hostedVespa(), zone, rotationsSet, - params.isBootstrap()); + params.isBootstrap(), + ! currentActiveApplicationSet.isPresent()); this.preparedModelsBuilder = new PreparedModelsBuilder(modelFactoryRegistry, permanentApplicationPackage, configDefinitionRepo, diff --git a/configserver/src/test/apps/hosted-no-write-access-control/searchdefinitions/music.sd b/configserver/src/test/apps/hosted-no-write-access-control/searchdefinitions/music.sd new file mode 100644 index 00000000000..78d58b27d4a --- /dev/null +++ b/configserver/src/test/apps/hosted-no-write-access-control/searchdefinitions/music.sd @@ -0,0 +1,10 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +search music { + document music { + field title type string { + indexing: index | summary + # index-to: default + } + } +} + diff --git a/configserver/src/test/apps/hosted-no-write-access-control/services.xml b/configserver/src/test/apps/hosted-no-write-access-control/services.xml new file mode 100644 index 00000000000..c2257ab34f7 --- /dev/null +++ b/configserver/src/test/apps/hosted-no-write-access-control/services.xml @@ -0,0 +1,25 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<services version="1.0"> + + <admin version="3.0"> + <nodes count='1'/> + </admin> + + <jdisc version="1.0"> + <http> + <server id="foo" port="4080" /> + </http> + <search/> + <nodes count='1'/> + </jdisc> + + <content id="music" version="1.0"> + <redundancy>1</redundancy> + <documents> + <document type="music" mode="index" /> + </documents> + <nodes count="2" groups="2"/> + </content> + +</services> diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java index f9c99b323af..dbb02e367a7 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java @@ -2,10 +2,8 @@ package com.yahoo.vespa.config.server; import com.yahoo.cloud.config.ConfigserverConfig; -import com.yahoo.config.model.api.HostProvisioner; import com.yahoo.config.model.provision.InMemoryProvisioner; import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.Version; import com.yahoo.container.handler.VipStatus; import com.yahoo.container.jdisc.config.HealthMonitorConfig; import com.yahoo.container.jdisc.state.StateMonitor; @@ -41,8 +39,8 @@ public class ConfigServerBootstrapTest { public void testBootstrap() throws Exception { ConfigserverConfig configserverConfig = createConfigserverConfig(temporaryFolder); InMemoryProvisioner provisioner = new InMemoryProvisioner(true, "host0", "host1", "host3"); - DeployTester tester = new DeployTester("src/test/apps/hosted/", configserverConfig, provisioner); - tester.deployApp("myApp", "4.5.6", Instant.now()); + DeployTester tester = new DeployTester(configserverConfig, provisioner); + tester.deployApp("src/test/apps/hosted/", "myApp", "4.5.6", Instant.now()); File versionFile = temporaryFolder.newFile(); VersionState versionState = new VersionState(versionFile); @@ -72,8 +70,8 @@ public class ConfigServerBootstrapTest { @Test public void testBootstrapWhenRedeploymentFails() throws Exception { ConfigserverConfig configserverConfig = createConfigserverConfig(temporaryFolder); - DeployTester tester = new DeployTester("src/test/apps/hosted/", configserverConfig); - tester.deployApp("myApp", "4.5.6", Instant.now()); + DeployTester tester = new DeployTester(configserverConfig); + tester.deployApp("src/test/apps/hosted/", "myApp", "4.5.6", Instant.now()); File versionFile = temporaryFolder.newFile(); VersionState versionState = new VersionState(versionFile); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java index ce53dc3f2fb..28fc179770a 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java @@ -51,6 +51,7 @@ public class ModelContextImplTest { false, Zone.defaultZone(), rotations, + false, false), Optional.empty(), new Version(6), @@ -67,5 +68,6 @@ public class ModelContextImplTest { assertTrue(context.properties().zone() instanceof Zone); assertFalse(context.properties().hostedVespa()); assertThat(context.properties().rotations(), equalTo(rotations)); + assertThat(context.properties().isFirstTimeDeployment(), equalTo(false)); } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ModelFactoryRegistryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ModelFactoryRegistryTest.java index 761d78ee7e0..65cbe65f0d9 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ModelFactoryRegistryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ModelFactoryRegistryTest.java @@ -6,6 +6,7 @@ import com.yahoo.config.model.api.Model; import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.api.ModelCreateResult; import com.yahoo.config.model.api.ModelFactory; +import com.yahoo.config.model.api.ValidationParameters; import com.yahoo.config.provision.Version; import com.yahoo.vespa.config.server.http.UnknownVespaVersionException; import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry; @@ -88,7 +89,7 @@ public class ModelFactoryRegistryTest { } @Override - public ModelCreateResult createAndValidateModel(ModelContext modelContext, boolean ignoreValidationErrors) { + public ModelCreateResult createAndValidateModel(ModelContext modelContext, ValidationParameters validationParameters) { return null; } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java index b15356a172e..8685065d001 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java @@ -10,6 +10,7 @@ import com.yahoo.config.model.api.Model; import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.api.ModelCreateResult; import com.yahoo.config.model.api.ModelFactory; +import com.yahoo.config.model.api.ValidationParameters; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.provision.InMemoryProvisioner; import com.yahoo.config.model.test.MockApplicationPackage; @@ -62,58 +63,56 @@ public class DeployTester { private final Clock clock; private final TenantRepository tenantRepository; - private final File testApp; private final ApplicationRepository applicationRepository; private ApplicationId id; - public DeployTester(String appPath) { - this(appPath, Collections.singletonList(createModelFactory(Clock.systemUTC()))); + public DeployTester() { + this(Collections.singletonList(createModelFactory(Clock.systemUTC()))); } - public DeployTester(String appPath, List<ModelFactory> modelFactories) { - this(appPath, modelFactories, + public DeployTester(List<ModelFactory> modelFactories) { + this(modelFactories, new ConfigserverConfig(new ConfigserverConfig.Builder() .configServerDBDir(Files.createTempDir().getAbsolutePath()) .configDefinitionsDir(Files.createTempDir().getAbsolutePath())), Clock.systemUTC()); } - public DeployTester(String appPath, ConfigserverConfig configserverConfig) { - this(appPath, Collections.singletonList(createModelFactory(Clock.systemUTC())), configserverConfig, Clock.systemUTC()); + public DeployTester(ConfigserverConfig configserverConfig) { + this(Collections.singletonList(createModelFactory(Clock.systemUTC())), configserverConfig, Clock.systemUTC()); } - public DeployTester(String appPath, ConfigserverConfig configserverConfig, HostProvisioner provisioner) { - this(appPath, Collections.singletonList(createModelFactory(Clock.systemUTC())), configserverConfig, Clock.systemUTC(), provisioner); + public DeployTester(ConfigserverConfig configserverConfig, HostProvisioner provisioner) { + this(Collections.singletonList(createModelFactory(Clock.systemUTC())), configserverConfig, Clock.systemUTC(), provisioner); } - public DeployTester(String appPath, ConfigserverConfig configserverConfig, Clock clock) { - this(appPath, Collections.singletonList(createModelFactory(clock)), configserverConfig, clock); + public DeployTester(ConfigserverConfig configserverConfig, Clock clock) { + this(Collections.singletonList(createModelFactory(clock)), configserverConfig, clock); } - public DeployTester(String appPath, List<ModelFactory> modelFactories, ConfigserverConfig configserverConfig) { - this(appPath, modelFactories, configserverConfig, Clock.systemUTC()); + public DeployTester(List<ModelFactory> modelFactories, ConfigserverConfig configserverConfig) { + this(modelFactories, configserverConfig, Clock.systemUTC()); } - public DeployTester(String appPath, List<ModelFactory> modelFactories, ConfigserverConfig configserverConfig, Clock clock) { - this(appPath, modelFactories, configserverConfig, clock, Zone.defaultZone()); + public DeployTester(List<ModelFactory> modelFactories, ConfigserverConfig configserverConfig, Clock clock) { + this(modelFactories, configserverConfig, clock, Zone.defaultZone()); } - public DeployTester(String appPath, List<ModelFactory> modelFactories, ConfigserverConfig configserverConfig, Clock clock, HostProvisioner provisioner) { - this(appPath, modelFactories, configserverConfig, clock, Zone.defaultZone(), provisioner); + public DeployTester(List<ModelFactory> modelFactories, ConfigserverConfig configserverConfig, Clock clock, HostProvisioner provisioner) { + this(modelFactories, configserverConfig, clock, Zone.defaultZone(), provisioner); } - public DeployTester(String appPath, List<ModelFactory> modelFactories, ConfigserverConfig configserverConfig, Clock clock, Zone zone) { - this(appPath, modelFactories, configserverConfig, clock, zone, createProvisioner()); + public DeployTester(List<ModelFactory> modelFactories, ConfigserverConfig configserverConfig, Clock clock, Zone zone) { + this(modelFactories, configserverConfig, clock, zone, createProvisioner()); } - public DeployTester(String appPath, List<ModelFactory> modelFactories, ConfigserverConfig configserverConfig, Clock clock, Zone zone, HostProvisioner provisioner) { + public DeployTester(List<ModelFactory> modelFactories, ConfigserverConfig configserverConfig, Clock clock, Zone zone, HostProvisioner provisioner) { this.clock = clock; TestComponentRegistry componentRegistry = createComponentRegistry(new MockCurator(), Metrics.createTestMetrics(), modelFactories, configserverConfig, clock, zone, provisioner); try { - this.testApp = new File(appPath); this.tenantRepository = new TenantRepository(componentRegistry); tenantRepository.addTenant(tenantName); } @@ -133,6 +132,11 @@ public class DeployTester { } /** Create a model factory for a particular version */ + public static CountingModelFactory createModelFactory(Version version) { + return new CountingModelFactory(version, Clock.systemUTC()); + } + + /** Create a model factory for a particular version */ public static CountingModelFactory createModelFactory(Version version, Clock clock) { return new CountingModelFactory(version, clock); } @@ -143,14 +147,14 @@ public class DeployTester { /** * Do the initial "deploy" with the existing API-less code as the deploy API doesn't support first deploys yet. */ - public ApplicationId deployApp(String appName, Instant now) { - return deployApp(appName, null, now); + public ApplicationId deployApp(String applicationPath, String appName, Instant now) { + return deployApp(applicationPath, appName, null, now); } /** * Do the initial "deploy" with the existing API-less code as the deploy API doesn't support first deploys yet. */ - public ApplicationId deployApp(String appName, String vespaVersion, Instant now) { + public ApplicationId deployApp(String applicationPath, String appName, String vespaVersion, Instant now) { Tenant tenant = tenant(); TimeoutBudget timeoutBudget = new TimeoutBudget(clock, Duration.ofSeconds(60)); ApplicationId id = ApplicationId.from(tenant.getName(), ApplicationName.from(appName), InstanceName.defaultName()); @@ -158,7 +162,7 @@ public class DeployTester { if (vespaVersion != null) paramsBuilder.vespaVersion(vespaVersion); - long sessionId = applicationRepository.createSession(id, timeoutBudget, testApp); + long sessionId = applicationRepository.createSession(id, timeoutBudget, new File(applicationPath)); applicationRepository.prepare(tenant, sessionId, paramsBuilder.build(), now); applicationRepository.activate(tenant, sessionId, timeoutBudget, false, false); this.id = id; @@ -256,7 +260,7 @@ public class DeployTester { try { Instant now = LocalDate.parse("2000-01-01", DateTimeFormatter.ISO_DATE).atStartOfDay().atZone(ZoneOffset.UTC).toInstant(); ApplicationPackage application = new MockApplicationPackage.Builder().withEmptyHosts().withEmptyServices().build(); - DeployState deployState = new DeployState.Builder().applicationPackage(application).now(now).build(true); + DeployState deployState = new DeployState.Builder().applicationPackage(application).now(now).build(); return new VespaModel(deployState); } catch (Exception e) { throw new RuntimeException(e); @@ -264,8 +268,8 @@ public class DeployTester { } @Override - public ModelCreateResult createAndValidateModel(ModelContext modelContext, boolean ignoreValidationErrors) { - if ( ! ignoreValidationErrors) + public ModelCreateResult createAndValidateModel(ModelContext modelContext, ValidationParameters validationParameters) { + if ( ! validationParameters.ignoreValidationErrors()) throw new IllegalArgumentException("Validation fails"); return new ModelCreateResult(createModel(modelContext), Collections.emptyList()); } @@ -300,8 +304,8 @@ public class DeployTester { } @Override - public ModelCreateResult createAndValidateModel(ModelContext modelContext, boolean ignoreValidationErrors) { - ModelCreateResult result = wrapped.createAndValidateModel(modelContext, ignoreValidationErrors); + public ModelCreateResult createAndValidateModel(ModelContext modelContext, ValidationParameters validationParameters) { + ModelCreateResult result = wrapped.createAndValidateModel(modelContext, validationParameters); creationCount++; return result; } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java index 6ee4720b6b0..a184a461ce1 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java @@ -22,9 +22,12 @@ import java.time.Clock; import java.time.Duration; import java.time.Instant; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.stream.Collectors; +import java.util.stream.IntStream; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -39,8 +42,8 @@ public class HostedDeployTest { @Test public void testRedeployWithVersion() { CountingModelFactory modelFactory = DeployTester.createModelFactory(Version.fromString("4.5.6"), Clock.systemUTC()); - DeployTester tester = new DeployTester("src/test/apps/hosted/", Collections.singletonList(modelFactory), createConfigserverConfig()); - tester.deployApp("myApp", "4.5.6", Instant.now()); + DeployTester tester = new DeployTester(Collections.singletonList(modelFactory), createConfigserverConfig()); + tester.deployApp("src/test/apps/hosted/", "myApp", "4.5.6", Instant.now()); Optional<com.yahoo.config.provision.Deployment> deployment = tester.redeployFromLocalActive(); assertTrue(deployment.isPresent()); @@ -51,8 +54,8 @@ public class HostedDeployTest { @Test public void testRedeploy() { - DeployTester tester = new DeployTester("src/test/apps/hosted/", createConfigserverConfig()); - ApplicationId appId = tester.deployApp("myApp", Instant.now()); + DeployTester tester = new DeployTester(createConfigserverConfig()); + ApplicationId appId = tester.deployApp("src/test/apps/hosted/", "myApp", Instant.now()); LocalSession s1 = tester.applicationRepository().getActiveSession(appId); System.out.println("First session: " + s1.getSessionId()); assertFalse(tester.applicationRepository().getActiveSession(appId).getMetaData().isInternalRedeploy()); @@ -73,10 +76,9 @@ public class HostedDeployTest { modelFactories.add(DeployTester.createModelFactory(Version.fromString("6.1.0"), clock)); modelFactories.add(DeployTester.createModelFactory(Version.fromString("6.2.0"), clock)); modelFactories.add(DeployTester.createModelFactory(Version.fromString("7.0.0"), clock)); - DeployTester tester = new DeployTester("src/test/apps/hosted/", modelFactories, createConfigserverConfig(), clock, Zone.defaultZone()); - ApplicationId app = tester.deployApp("myApp", "6.2.0", Instant.now()); + DeployTester tester = new DeployTester(modelFactories, createConfigserverConfig(), clock, Zone.defaultZone()); + ApplicationId app = tester.deployApp("src/test/apps/hosted/", "myApp", "6.2.0", Instant.now()); assertEquals(3, tester.getAllocatedHostsOf(app).getHosts().size()); - } /** Test that only the minimal set of models are created (model versions used on hosts, the wanted version and the latest version) */ @@ -103,10 +105,10 @@ public class HostedDeployTest { modelFactories.add(factory710); modelFactories.add(factory720); - DeployTester tester = new DeployTester("src/test/apps/hosted/", modelFactories, createConfigserverConfig(), + DeployTester tester = new DeployTester(modelFactories, createConfigserverConfig(), clock, new Zone(Environment.dev, RegionName.defaultName()), provisioner); // Deploy with version that does not exist on hosts, the model for this version should also be created - ApplicationId app = tester.deployApp("myApp", "7.0.0", Instant.now()); + ApplicationId app = tester.deployApp("src/test/apps/hosted/", "myApp", "7.0.0", Instant.now()); assertEquals(3, tester.getAllocatedHostsOf(app).getHosts().size()); // Check >0 not ==0 as the session watcher thread is running and will redeploy models in the background @@ -119,14 +121,37 @@ public class HostedDeployTest { } @Test + public void testAccessControlIsOnlyCheckedWhenNoProdDeploymentExists() { + // Provisioner does not reuse hosts, so need twice as many hosts as app requires + List<Host> hosts = IntStream.rangeClosed(1,6).mapToObj(i -> createHost("host" + i, "6.0.0")).collect(Collectors.toList()); + InMemoryProvisioner provisioner = new InMemoryProvisioner(new Hosts(hosts), true); + + CountingModelFactory factory600 = DeployTester.createModelFactory(Version.fromString("6.0.0")); + CountingModelFactory factory610 = DeployTester.createModelFactory(Version.fromString("6.1.0")); + CountingModelFactory factory620 = DeployTester.createModelFactory(Version.fromString("6.2.0")); + List<ModelFactory> modelFactories = Arrays.asList(factory600, factory610, factory620); + + DeployTester tester = new DeployTester(modelFactories, createConfigserverConfig(), + Clock.systemUTC(), new Zone(Environment.prod, RegionName.defaultName()), provisioner); + // Deploy with oldest version + ApplicationId app = tester.deployApp("src/test/apps/hosted/", "myApp", "6.0.0", Instant.now()); + assertEquals(3, tester.getAllocatedHostsOf(app).getHosts().size()); + + // Deploy with version that does not exist on hosts and with app package that has no write access control, + // validation of access control should not be done, since the app is already deployed in prod + app = tester.deployApp("src/test/apps/hosted-no-write-access-control", "myApp", "6.1.0", Instant.now()); + assertEquals(3, tester.getAllocatedHostsOf(app).getHosts().size()); + } + + @Test public void testRedeployAfterExpiredValidationOverride() { // Old version of model fails, but application disables loading old models until 2016-10-10, so deployment works ManualClock clock = new ManualClock("2016-10-09T00:00:00"); List<ModelFactory> modelFactories = new ArrayList<>(); modelFactories.add(DeployTester.createModelFactory(clock)); modelFactories.add(DeployTester.createFailingModelFactory(Version.fromIntValues(1, 0, 0))); // older than default - DeployTester tester = new DeployTester("src/test/apps/validationOverride/", modelFactories, createConfigserverConfig()); - tester.deployApp("myApp", clock.instant()); + DeployTester tester = new DeployTester(modelFactories, createConfigserverConfig()); + tester.deployApp("src/test/apps/validationOverride/", "myApp", clock.instant()); // Redeployment from local active works { @@ -149,7 +174,7 @@ public class HostedDeployTest { // However, redeployment from the outside fails after this date { try { - tester.deployApp("myApp", Instant.now()); + tester.deployApp("src/test/apps/validationOverride/", "myApp", Instant.now()); fail("Expected redeployment to fail"); } catch (Exception expected) { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/RedeployTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/RedeployTest.java index fd023a19617..9bc9a93e9fa 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/RedeployTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/RedeployTest.java @@ -34,8 +34,8 @@ public class RedeployTest { @Test public void testRedeploy() { - DeployTester tester = new DeployTester("src/test/apps/app"); - tester.deployApp("myapp", Instant.now()); + DeployTester tester = new DeployTester(); + tester.deployApp("src/test/apps/app", "myapp", Instant.now()); Optional<com.yahoo.config.provision.Deployment> deployment = tester.redeployFromLocalActive(); assertTrue(deployment.isPresent()); @@ -54,7 +54,7 @@ public class RedeployTest { List<ModelFactory> modelFactories = new ArrayList<>(); modelFactories.add(DeployTester.createModelFactory(Clock.systemUTC())); modelFactories.add(DeployTester.createFailingModelFactory(Version.fromIntValues(1, 0, 0))); - DeployTester tester = new DeployTester("ignored/app/path", modelFactories); + DeployTester tester = new DeployTester(modelFactories); ApplicationId id = ApplicationId.from(tester.tenant().getName(), ApplicationName.from("default"), InstanceName.from("default")); @@ -70,8 +70,8 @@ public class RedeployTest { .configDefinitionsDir(Files.createTempDir() .getAbsolutePath()) .sessionLifetime(60)); - DeployTester tester = new DeployTester("src/test/apps/app", configserverConfig, clock); - tester.deployApp("myapp", Instant.now()); // session 2 (numbering starts at 2) + DeployTester tester = new DeployTester(configserverConfig, clock); + tester.deployApp("src/test/apps/app", "myapp", Instant.now()); // session 2 (numbering starts at 2) clock.advance(Duration.ofSeconds(10)); Optional<com.yahoo.config.provision.Deployment> deployment2 = tester.redeployFromLocalActive(); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java index 8b89027e4a1..6fa36ba27bb 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java @@ -197,7 +197,7 @@ public class LbServicesProducerTest { } private Model createVespaModel(ApplicationPackage applicationPackage, DeployState.Builder deployStateBuilder) throws IOException, SAXException { - return new VespaModel(new NullConfigModelRegistry(), deployStateBuilder.applicationPackage(applicationPackage).build(true)); + return new VespaModel(new NullConfigModelRegistry(), deployStateBuilder.applicationPackage(applicationPackage).build()); } private void assertConfig(LbServicesConfig expected, LbServicesConfig actual) { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/model/RoutingProducerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/model/RoutingProducerTest.java index 8c48cfe4b99..8a6d772ff14 100755 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/model/RoutingProducerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/model/RoutingProducerTest.java @@ -102,6 +102,6 @@ public class RoutingProducerTest { } private Model createVespaModel(ApplicationPackage applicationPackage, DeployState.Builder deployStateBuilder) throws IOException, SAXException { - return new VespaModel(new NullConfigModelRegistry(), deployStateBuilder.applicationPackage(applicationPackage).build(true)); + return new VespaModel(new NullConfigModelRegistry(), deployStateBuilder.applicationPackage(applicationPackage).build()); } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/model/TestModelFactory.java b/configserver/src/test/java/com/yahoo/vespa/config/server/model/TestModelFactory.java index 48caba2baef..d736611cdcd 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/model/TestModelFactory.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/model/TestModelFactory.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.config.server.model; import com.yahoo.config.model.NullConfigModelRegistry; import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.api.ModelCreateResult; +import com.yahoo.config.model.api.ValidationParameters; import com.yahoo.config.provision.Version; import com.yahoo.vespa.model.VespaModelFactory; @@ -21,9 +22,9 @@ public class TestModelFactory extends VespaModelFactory { // Needed for testing (to get hold of ModelContext) @Override - public ModelCreateResult createAndValidateModel(ModelContext modelContext, boolean ignoreValidationErrors) { + public ModelCreateResult createAndValidateModel(ModelContext modelContext, ValidationParameters validationParameters) { this.modelContext = modelContext; - return super.createAndValidateModel(modelContext, ignoreValidationErrors); + return super.createAndValidateModel(modelContext, validationParameters); } @Override diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/provision/StaticProvisionerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/provision/StaticProvisionerTest.java index 07ba1925bf6..badcdf53b77 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/provision/StaticProvisionerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/provision/StaticProvisionerTest.java @@ -54,7 +54,7 @@ public class StaticProvisionerTest { .multitenant(true) .hostedVespa(true) .build()) - .build(true); + .build(); return new VespaModel(new NullConfigModelRegistry(), deployState); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/RemoteSessionTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/RemoteSessionTest.java index b57d2d1a1a1..459604fa333 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/RemoteSessionTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/RemoteSessionTest.java @@ -277,7 +277,7 @@ public class RemoteSessionTest { public Model loadModel() { try { ApplicationPackage application = new MockApplicationPackage.Builder().withEmptyHosts().withEmptyServices().withValidationOverrides(validationOverrides).build(); - DeployState deployState = new DeployState.Builder().applicationPackage(application).now(clock.instant()).build(true); + DeployState deployState = new DeployState.Builder().applicationPackage(application).now(clock.instant()).build(); return new VespaModel(deployState); } catch (Exception e) { throw new RuntimeException(e); @@ -285,7 +285,7 @@ public class RemoteSessionTest { } @Override - public ModelCreateResult createAndValidateModel(ModelContext modelContext, boolean ignoreValidationErrors) { + public ModelCreateResult createAndValidateModel(ModelContext modelContext, ValidationParameters validationParameters) { if (throwOnLoad) { throw new IllegalArgumentException("Foo"); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java index 6f26323b558..a221a496e0c 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java @@ -7,6 +7,7 @@ import com.yahoo.config.model.api.ConfigChangeAction; import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.api.ModelCreateResult; import com.yahoo.config.model.api.ServiceInfo; +import com.yahoo.config.model.api.ValidationParameters; import com.yahoo.config.model.application.provider.*; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.InstanceName; @@ -267,7 +268,7 @@ public class SessionPreparerTest { } @Override - public ModelCreateResult createAndValidateModel(ModelContext modelContext, boolean ignoreValidationErrors) { + public ModelCreateResult createAndValidateModel(ModelContext modelContext, ValidationParameters validationParameters) { throw exception; } } @@ -285,8 +286,8 @@ public class SessionPreparerTest { } @Override - public ModelCreateResult createAndValidateModel(ModelContext modelContext, boolean ignoreValidationErrors) { - ModelCreateResult result = super.createAndValidateModel(modelContext, ignoreValidationErrors); + public ModelCreateResult createAndValidateModel(ModelContext modelContext, ValidationParameters validationParameters) { + ModelCreateResult result = super.createAndValidateModel(modelContext, validationParameters); return new ModelCreateResult(result.getModel(), Arrays.asList(action)); } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java index 06908dbab51..22d5901a29a 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java @@ -55,7 +55,7 @@ public class ZKApplicationPackageTest { assertTrue(Pattern.compile(".*<slobroks>.*",Pattern.MULTILINE+Pattern.DOTALL).matcher(IOUtils.readAll(zkApp.getServices())).matches()); assertTrue(Pattern.compile(".*<alias>.*",Pattern.MULTILINE+Pattern.DOTALL).matcher(IOUtils.readAll(zkApp.getHosts())).matches()); assertTrue(Pattern.compile(".*<slobroks>.*",Pattern.MULTILINE+Pattern.DOTALL).matcher(IOUtils.readAll(zkApp.getFile(Path.fromString("services.xml")).createReader())).matches()); - DeployState deployState = new DeployState.Builder().applicationPackage(zkApp).build(true); + DeployState deployState = new DeployState.Builder().applicationPackage(zkApp).build(); assertEquals(deployState.getSearchDefinitions().size(), 5); assertEquals(zkApp.searchDefinitionContents().size(), 5); assertEquals(IOUtils.readAll(zkApp.getRankingExpression("foo.expression")), "foo()+1\n"); |