diff options
97 files changed, 1240 insertions, 371 deletions
diff --git a/application/src/main/java/com/yahoo/application/Application.java b/application/src/main/java/com/yahoo/application/Application.java index 5221f95f897..88140873b7b 100644 --- a/application/src/main/java/com/yahoo/application/Application.java +++ b/application/src/main/java/com/yahoo/application/Application.java @@ -112,7 +112,7 @@ public final class Application implements AutoCloseable { .applicationPackage(FilesApplicationPackage.fromFile(path.toFile(), /* Include source files */ true)) .deployLogger((level, s) -> { }) - .build(true); + .build(); return new VespaModel(new NullConfigModelRegistry(), deployState); } catch (IOException | SAXException e) { throw new IllegalArgumentException("Error creating application from '" + path + "'", e); diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/GenerateOsgiManifestMojo.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/GenerateOsgiManifestMojo.java index 8d19f112765..973854aa59a 100644 --- a/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/GenerateOsgiManifestMojo.java +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/GenerateOsgiManifestMojo.java @@ -65,6 +65,7 @@ public class GenerateOsgiManifestMojo extends AbstractMojo { @Parameter(alias = "Bundle-Version", defaultValue = "${project.version}") private String bundleVersion = null; + // TODO Vespa 7: default should be ${project.groupId}.${project.artifactId} @Parameter(alias = "Bundle-SymbolicName", defaultValue = "${project.artifactId}") private String bundleSymbolicName = null; diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index e23d6c5b968..c75174cd999 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -50,6 +50,7 @@ public interface ModelContext { Zone zone(); Set<Rotation> rotations(); boolean isBootstrap(); + boolean isFirstTimeDeployment(); } } diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelFactory.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelFactory.java index 572c470b6aa..6d78e7d7756 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelFactory.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelFactory.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.model.api; +import com.yahoo.config.model.api.ValidationParameters.IgnoreValidationErrors; import com.yahoo.config.provision.Version; /** @@ -17,8 +18,8 @@ public interface ModelFactory { /** * Creates an instance of a {@link Model}. The resulting instance will be used to serve config. No model - * validation will be done, calling this method assumes that {@link #createAndValidateModel} has already - * been called at some point for this model. + * validation will be done, calling this method assumes that{@link #createAndValidateModel(ModelContext, ValidationParameters)} + * has already been called at some point for this model. * * @param modelContext an instance of {@link ModelContext}, containing dependencies for creating a {@link Model}. * @return a {@link Model} instance. @@ -32,7 +33,27 @@ public interface ModelFactory { * @param modelContext an instance of {@link ModelContext}, containing dependencies for creating a {@link Model} * @param ignoreValidationErrors true if validation errors should not trigger exceptions * @return a {@link ModelCreateResult} instance. + * @deprecated use {@link #createAndValidateModel(ModelContext, ValidationParameters)} */ - ModelCreateResult createAndValidateModel(ModelContext modelContext, boolean ignoreValidationErrors); + @SuppressWarnings("DeprecatedIsStillUsed") + @Deprecated + // TODO: Remove when 6.259 is latest version in use + default ModelCreateResult createAndValidateModel(ModelContext modelContext, boolean ignoreValidationErrors) { + return createAndValidateModel(modelContext, new ValidationParameters(ignoreValidationErrors + ? IgnoreValidationErrors.FALSE + : IgnoreValidationErrors.TRUE)); + } + /** + * Creates an instance of a {@link Model}. The resulting instance will be used to serve config. Any validation + * of a {@link Model} and the {@link ModelContext} can be done in this method. + * + * @param modelContext an instance of {@link ModelContext}, containing dependencies for creating a {@link Model} + * @param validationParameters validation parameters + * @return a {@link ModelCreateResult} instance. + */ + // TODO: Remove default implementation when 6.259 is latest version in use + default ModelCreateResult createAndValidateModel(ModelContext modelContext, ValidationParameters validationParameters) { + return createAndValidateModel(modelContext, validationParameters.ignoreValidationErrors()); + } } diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ValidationParameters.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ValidationParameters.java new file mode 100644 index 00000000000..6e081d0d668 --- /dev/null +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ValidationParameters.java @@ -0,0 +1,50 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.api; + +/** + * @author hmusum + */ +public class ValidationParameters { + + public enum IgnoreValidationErrors {TRUE, FALSE} + + public enum FailOnIncompatibleChange {TRUE, FALSE} //Note: Default is FALSE + + public enum CheckRouting {TRUE, FALSE} + + private final IgnoreValidationErrors ignoreValidationErrors; + private final FailOnIncompatibleChange failOnIncompatibleChange; + private final CheckRouting checkRouting; + + public ValidationParameters() { + this(IgnoreValidationErrors.FALSE); + } + + public ValidationParameters(IgnoreValidationErrors ignoreValidationErrors) { + this(ignoreValidationErrors, FailOnIncompatibleChange.FALSE, CheckRouting.TRUE); + } + + public ValidationParameters(CheckRouting checkRouting) { + this(IgnoreValidationErrors.FALSE, FailOnIncompatibleChange.FALSE, checkRouting); + } + + public ValidationParameters(IgnoreValidationErrors ignoreValidationErrors, + FailOnIncompatibleChange failOnIncompatibleChange, + CheckRouting checkRouting) { + this.ignoreValidationErrors = ignoreValidationErrors; + this.failOnIncompatibleChange = failOnIncompatibleChange; + this.checkRouting = checkRouting; + } + + public boolean ignoreValidationErrors() { + return ignoreValidationErrors == IgnoreValidationErrors.TRUE; + } + + public boolean failOnIncompatibleChanges() { + return failOnIncompatibleChange == FailOnIncompatibleChange.TRUE; + } + + public boolean checkRouting() { + return checkRouting == CheckRouting.TRUE; + } +} diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployProperties.java index b259f6cf3fb..53c70399e94 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployProperties.java @@ -26,6 +26,7 @@ public class DeployProperties { private final boolean hostedVespa; private final Version vespaVersion; private final boolean isBootstrap; + private final boolean isFirstTimeDeployment; private DeployProperties(boolean multitenant, ApplicationId applicationId, @@ -35,7 +36,8 @@ public class DeployProperties { URI ztsUrl, String athenzDnsSuffix, Version vespaVersion, - boolean isBootstrap) { + boolean isBootstrap, + boolean isFirstTimeDeployment) { this.loadBalancerName = loadBalancerName; this.ztsUrl = ztsUrl; this.athenzDnsSuffix = athenzDnsSuffix; @@ -45,6 +47,7 @@ public class DeployProperties { this.serverSpecs.addAll(configServerSpecs); this.hostedVespa = hostedVespa; this.isBootstrap = isBootstrap; + this.isFirstTimeDeployment = isFirstTimeDeployment; } public boolean multitenant() { @@ -83,6 +86,9 @@ public class DeployProperties { /** Returns whether this deployment happens during bootstrap *prepare* (not set on activate) */ public boolean isBootstrap() { return isBootstrap; } + /** Returns whether this is the first deployment for this application (used during *prepare*, not set on activate) */ + public boolean isFirstTimeDeployment() { return isFirstTimeDeployment; } + public static class Builder { private ApplicationId applicationId = ApplicationId.defaultId(); @@ -94,6 +100,7 @@ public class DeployProperties { private boolean hostedVespa = false; private Version vespaVersion = Version.fromIntValues(1, 0, 0); private boolean isBootstrap = false; + private boolean isFirstTimeDeployment = false; public Builder applicationId(ApplicationId applicationId) { this.applicationId = applicationId; @@ -140,8 +147,14 @@ public class DeployProperties { return this; } + public Builder isFirstTimeDeployment(boolean isFirstTimeDeployment) { + this.isFirstTimeDeployment = isFirstTimeDeployment; + return this; + } + public DeployProperties build() { - return new DeployProperties(multitenant, applicationId, configServerSpecs, loadBalancerName, hostedVespa, ztsUrl, athenzDnsSuffix, vespaVersion, isBootstrap); + return new DeployProperties(multitenant, applicationId, configServerSpecs, loadBalancerName, hostedVespa, + ztsUrl, athenzDnsSuffix, vespaVersion, isBootstrap, isFirstTimeDeployment); } } diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java index 3dafb521dc8..8b0285ec2b4 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java @@ -10,6 +10,7 @@ import com.yahoo.config.application.api.UnparsedConfigDefinition; import com.yahoo.config.model.api.ConfigDefinitionRepo; import com.yahoo.config.model.api.HostProvisioner; import com.yahoo.config.model.api.Model; +import com.yahoo.config.model.api.ValidationParameters; import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.config.model.application.provider.MockFileRegistry; import com.yahoo.config.model.provision.HostsXmlProvisioner; @@ -70,11 +71,11 @@ public class DeployState implements ConfigDefinitionStore { private final HostProvisioner provisioner; public static DeployState createTestState() { - return new Builder().build(true); + return new Builder().build(); } public static DeployState createTestState(ApplicationPackage applicationPackage) { - return new Builder().applicationPackage(applicationPackage).build(true); + return new Builder().applicationPackage(applicationPackage).build(); } private DeployState(ApplicationPackage applicationPackage, SearchDocumentModel searchDocumentModel, RankProfileRegistry rankProfileRegistry, @@ -288,11 +289,15 @@ public class DeployState implements ConfigDefinitionStore { return this; } - public DeployState build(boolean validate) { + public DeployState build() { + return build(new ValidationParameters()); + } + + public DeployState build(ValidationParameters validationParameters) { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); QueryProfiles queryProfiles = new QueryProfilesBuilder().build(applicationPackage); SemanticRules semanticRules = new SemanticRuleBuilder().build(applicationPackage); - SearchDocumentModel searchDocumentModel = createSearchDocumentModel(rankProfileRegistry, logger, queryProfiles, validate); + SearchDocumentModel searchDocumentModel = createSearchDocumentModel(rankProfileRegistry, logger, queryProfiles, validationParameters); return new DeployState(applicationPackage, searchDocumentModel, rankProfileRegistry, fileRegistry, logger, hostProvisioner, properties, permanentApplicationPackage, configDefinitionRepo, previousModel, rotations, zone, queryProfiles, semanticRules, now, wantedNodeVespaVersion); @@ -301,7 +306,7 @@ public class DeployState implements ConfigDefinitionStore { private SearchDocumentModel createSearchDocumentModel(RankProfileRegistry rankProfileRegistry, DeployLogger logger, QueryProfiles queryProfiles, - boolean validate) { + ValidationParameters validationParameters) { Collection<NamedReader> readers = applicationPackage.getSearchDefinitions(); Map<String, String> names = new LinkedHashMap<>(); SearchBuilder builder = new SearchBuilder(applicationPackage, rankProfileRegistry, queryProfiles.getRegistry()); @@ -324,7 +329,7 @@ public class DeployState implements ConfigDefinitionStore { closeIgnoreException(reader.getReader()); } } - builder.build(validate, logger); + builder.build(! validationParameters.ignoreValidationErrors(), logger); return SearchDocumentModel.fromBuilderAndNames(builder, names); } diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java b/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java index e3d766e710f..29d151857cc 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java @@ -55,7 +55,7 @@ public class MockRoot extends AbstractConfigProducerRoot { } public MockRoot(String rootConfigId, ApplicationPackage applicationPackage) { - this(rootConfigId, new DeployState.Builder().applicationPackage(applicationPackage).build(true)); + this(rootConfigId, new DeployState.Builder().applicationPackage(applicationPackage).build()); } public MockRoot(String rootConfigId, DeployState deployState) { diff --git a/config-model/src/main/java/com/yahoo/config/model/test/TestDriver.java b/config-model/src/main/java/com/yahoo/config/model/test/TestDriver.java index 9fe0c92141f..b538468d0bc 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/TestDriver.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/TestDriver.java @@ -80,7 +80,7 @@ public class TestDriver { * @return a producer root capable of answering getConfig requests. */ public TestRoot buildModel(ApplicationPackage applicationPackage) { - return buildModel(new DeployState.Builder().applicationPackage(applicationPackage).build(true)); + return buildModel(new DeployState.Builder().applicationPackage(applicationPackage).build()); } /** diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java index 73fb532cfb4..f3e7a9623d1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java @@ -18,6 +18,7 @@ import com.yahoo.config.model.NullConfigModelRegistry; import com.yahoo.config.model.api.FileDistribution; import com.yahoo.config.model.api.HostInfo; import com.yahoo.config.model.api.Model; +import com.yahoo.config.model.api.ValidationParameters; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.config.model.producer.AbstractConfigProducerRoot; @@ -123,7 +124,7 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri * to instantiate config models */ public VespaModel(ApplicationPackage app, ConfigModelRegistry configModelRegistry) throws IOException, SAXException { - this(configModelRegistry, new DeployState.Builder().applicationPackage(app).build(true)); + this(configModelRegistry, new DeployState.Builder().applicationPackage(app).build()); } /** diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java index e88153342f9..75f70d03fcc 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java @@ -14,6 +14,7 @@ import com.yahoo.config.model.api.Model; import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.api.ModelCreateResult; import com.yahoo.config.model.api.ModelFactory; +import com.yahoo.config.model.api.ValidationParameters; import com.yahoo.config.model.application.provider.ApplicationPackageXmlFilesValidator; import com.yahoo.config.model.builder.xml.ConfigModelBuilder; import com.yahoo.config.model.deploy.DeployProperties; @@ -88,15 +89,15 @@ public class VespaModelFactory implements ModelFactory { @Override public Model createModel(ModelContext modelContext) { - return buildModel(createDeployState(modelContext, false)); + return buildModel(createDeployState(modelContext, new ValidationParameters(ValidationParameters.IgnoreValidationErrors.TRUE))); } @Override - public ModelCreateResult createAndValidateModel(ModelContext modelContext, boolean ignoreValidationErrors) { - validateXml(modelContext, ignoreValidationErrors); - DeployState deployState = createDeployState(modelContext, true); + public ModelCreateResult createAndValidateModel(ModelContext modelContext, ValidationParameters validationParameters) { + validateXml(modelContext, validationParameters.ignoreValidationErrors()); + DeployState deployState = createDeployState(modelContext, validationParameters); VespaModel model = buildModel(deployState); - List<ConfigChangeAction> changeActions = validateModel(model, deployState, ignoreValidationErrors); + List<ConfigChangeAction> changeActions = validateModel(model, deployState, validationParameters); return new ModelCreateResult(model, changeActions); } @@ -126,7 +127,7 @@ public class VespaModelFactory implements ModelFactory { } } - private DeployState createDeployState(ModelContext modelContext, boolean validate) { + private DeployState createDeployState(ModelContext modelContext, ValidationParameters validationParameters) { DeployState.Builder builder = new DeployState.Builder() .applicationPackage(modelContext.applicationPackage()) .deployLogger(modelContext.deployLogger()) @@ -140,7 +141,7 @@ public class VespaModelFactory implements ModelFactory { .now(clock.instant()) .wantedNodeVespaVersion(modelContext.wantedNodeVespaVersion()); modelContext.previousModel().ifPresent(builder::previousModel); - return builder.build(validate); + return builder.build(validationParameters); } private DeployProperties createDeployProperties(ModelContext.Properties properties) { @@ -154,6 +155,7 @@ public class VespaModelFactory implements ModelFactory { .hostedVespa(properties.hostedVespa()) .vespaVersion(getVersion()) .isBootstrap(properties.isBootstrap()) + .isFirstTimeDeployment(properties.isFirstTimeDeployment()) .build(); } @@ -172,11 +174,11 @@ public class VespaModelFactory implements ModelFactory { } } - private List<ConfigChangeAction> validateModel(VespaModel model, DeployState deployState, boolean ignoreValidationErrors) { + private List<ConfigChangeAction> validateModel(VespaModel model, DeployState deployState, ValidationParameters validationParameters) { try { - return Validation.validate(model, ignoreValidationErrors, deployState); + return Validation.validate(model, validationParameters, deployState); } catch (IllegalArgumentException e) { - rethrowUnlessIgnoreErrors(e, ignoreValidationErrors); + rethrowUnlessIgnoreErrors(e, validationParameters.ignoreValidationErrors()); } catch (Exception e) { throw new RuntimeException(e); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index 30dd56c14c1..b7c9116dc60 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -322,10 +322,10 @@ public class VespaMetricSet { metrics.add(new Metric("vds.datastored.alldisks.bytes.average","bytes")); metrics.add(new Metric("vds.visitor.allthreads.averagevisitorlifetime.sum.average","visitorlifetime")); metrics.add(new Metric("vds.visitor.allthreads.averagequeuewait.sum.average","visitorqueuewait")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.put.sum.count.rate","put")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.remove.sum.count.rate","remove")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.get.sum.count.rate","get")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.update.sum.count.rate","update")); + metrics.add(new Metric("vds.filestor.alldisks.allthreads.put.sum.count.rate")); + metrics.add(new Metric("vds.filestor.alldisks.allthreads.remove.sum.count.rate")); + metrics.add(new Metric("vds.filestor.alldisks.allthreads.get.sum.count.rate")); + metrics.add(new Metric("vds.filestor.alldisks.allthreads.update.sum.count.rate")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.visit.sum.count.rate")); metrics.add(new Metric("vds.filestor.alldisks.queuesize.average","diskqueuesize")); metrics.add(new Metric("vds.filestor.alldisks.averagequeuewait.sum.average","diskqueuewait")); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java index c08e81b250f..e44acf61466 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java @@ -5,6 +5,7 @@ import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.model.api.ConfigChangeAction; import com.yahoo.config.model.api.Model; +import com.yahoo.config.model.api.ValidationParameters; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.application.validation.change.ChangeValidator; @@ -21,8 +22,8 @@ import com.yahoo.vespa.model.application.validation.change.StreamingSearchCluste import com.yahoo.vespa.model.application.validation.first.AccessControlValidator; import java.time.Instant; -import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Optional; @@ -35,18 +36,14 @@ import static java.util.stream.Collectors.toList; */ public class Validation { - /** Validate everything */ - public static List<ConfigChangeAction> validate(VespaModel model, boolean force, DeployState deployState) { - return validate(model, true, force, deployState); - } - /** - * Validate with optional checking of routing, which cannot always be valid in unit tests + * Validates the model supplied, and if there already exists a model for the application validates changes + * between the previous and current model * * @return a list of required changes needed to make this configuration live */ - public static List<ConfigChangeAction> validate(VespaModel model, boolean checkRouting, boolean force, DeployState deployState) { - if (checkRouting) { + public static List<ConfigChangeAction> validate(VespaModel model, ValidationParameters validationParameters, DeployState deployState) { + if (validationParameters.checkRouting()) { new RoutingValidator().validate(model, deployState); new RoutingSelectorValidator().validate(model, deployState); } @@ -54,19 +51,22 @@ public class Validation { new SearchDataTypeValidator().validate(model, deployState); new ComplexAttributeFieldsValidator().validate(model, deployState); new StreamingValidator().validate(model, deployState); - new RankSetupValidator(force).validate(model, deployState); + new RankSetupValidator(validationParameters.ignoreValidationErrors()).validate(model, deployState); new NoPrefixForIndexes().validate(model, deployState); new DeploymentFileValidator().validate(model, deployState); new RankingConstantsValidator().validate(model, deployState); new SecretStoreValidator().validate(model, deployState); - Optional<Model> currentActiveModel = deployState.getPreviousModel(); - if (currentActiveModel.isPresent() && (currentActiveModel.get() instanceof VespaModel)) - return validateChanges((VespaModel)currentActiveModel.get(), model, - deployState.validationOverrides(), deployState.getDeployLogger(), deployState.now()); - else + List<ConfigChangeAction> result = Collections.emptyList(); + if (deployState.getProperties().isFirstTimeDeployment()) { validateFirstTimeDeployment(model, deployState); - return new ArrayList<>(); + } else { + Optional<Model> currentActiveModel = deployState.getPreviousModel(); + if (currentActiveModel.isPresent() && (currentActiveModel.get() instanceof VespaModel)) + result = validateChanges((VespaModel) currentActiveModel.get(), model, + deployState.validationOverrides(), deployState.getDeployLogger(), deployState.now()); + } + return result; } private static List<ConfigChangeAction> validateChanges(VespaModel currentModel, VespaModel nextModel, diff --git a/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java b/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java index d3e8136cdec..643a3bd0b91 100644 --- a/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java @@ -238,7 +238,7 @@ public class ApplicationDeployTest { } private List<SearchDefinition> getSearchDefinitions(FilesApplicationPackage app) { - return new DeployState.Builder().applicationPackage(app).build(true).getSearchDefinitions(); + return new DeployState.Builder().applicationPackage(app).build().getSearchDefinitions(); } public FilesApplicationPackage createAppPkg(String appPkg) throws IOException { @@ -374,7 +374,7 @@ public class ApplicationDeployTest { final File appDir = new File("src/test/cfg/application/configdeftest"); FilesApplicationPackage app = FilesApplicationPackage.fromFile(appDir); - DeployState deployState = new DeployState.Builder().applicationPackage(app).build(true); + DeployState deployState = new DeployState.Builder().applicationPackage(app).build(); ConfigDefinition def = deployState.getConfigDefinition(new ConfigDefinitionKey("foo", CNode.DEFAULT_NAMESPACE)).get(); assertThat(def.getNamespace(), is(CNode.DEFAULT_NAMESPACE)); @@ -394,7 +394,7 @@ public class ApplicationDeployTest { @Test(expected=IllegalArgumentException.class) public void testDifferentNameOfSdFileAndSearchName() throws IOException { FilesApplicationPackage app = createAppPkg(TESTDIR + "sdfilenametest"); - new DeployState.Builder().applicationPackage(app).build(true); + new DeployState.Builder().applicationPackage(app).build(); } } diff --git a/config-model/src/test/java/com/yahoo/config/model/MockModelContext.java b/config-model/src/test/java/com/yahoo/config/model/MockModelContext.java index 6821b7e3b4b..5bd95334396 100644 --- a/config-model/src/test/java/com/yahoo/config/model/MockModelContext.java +++ b/config-model/src/test/java/com/yahoo/config/model/MockModelContext.java @@ -130,6 +130,9 @@ public class MockModelContext implements ModelContext { @Override public boolean isBootstrap() { return false; } + + @Override + public boolean isFirstTimeDeployment() { return false; } }; } } diff --git a/config-model/src/test/java/com/yahoo/config/model/deploy/DeployStateTest.java b/config-model/src/test/java/com/yahoo/config/model/deploy/DeployStateTest.java index 2a9547df839..aa29ebd9825 100644 --- a/config-model/src/test/java/com/yahoo/config/model/deploy/DeployStateTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/deploy/DeployStateTest.java @@ -41,7 +41,7 @@ public class DeployStateTest { DeployState.Builder builder = new DeployState.Builder(); HostProvisioner provisioner = new InMemoryProvisioner(true, "foo.yahoo.com"); builder.modelHostProvisioner(provisioner); - DeployState state = builder.build(true); + DeployState state = builder.build(); assertThat(state.getProvisioner(), is(provisioner)); } @@ -50,7 +50,7 @@ public class DeployStateTest { DeployState.Builder builder = new DeployState.Builder(); ApplicationPackage app = MockApplicationPackage.createEmpty(); builder.permanentApplicationPackage(Optional.of(app)); - DeployState state = builder.build(true); + DeployState state = builder.build(); assertThat(state.getPermanentApplicationPackage().get(), is(app)); } @@ -58,20 +58,20 @@ public class DeployStateTest { public void testPreviousModelIsProvided() throws IOException, SAXException { VespaModel prevModel = new VespaModel(MockApplicationPackage.createEmpty()); DeployState.Builder builder = new DeployState.Builder(); - assertThat(builder.previousModel(prevModel).build(true).getPreviousModel().get(), is(prevModel)); + assertThat(builder.previousModel(prevModel).build().getPreviousModel().get(), is(prevModel)); } @Test public void testProperties() { DeployState.Builder builder = new DeployState.Builder(); - DeployState state = builder.build(true); + DeployState state = builder.build(); assertThat(state.getProperties().applicationId(), is(ApplicationId.defaultId())); ApplicationId customId = new ApplicationId.Builder() .tenant("bar") .applicationName("foo").instanceName("quux").build(); DeployProperties properties = new DeployProperties.Builder().applicationId(customId).build(); builder.properties(properties); - state = builder.build(true); + state = builder.build(); assertThat(state.getProperties().applicationId(), is(customId)); } @@ -112,11 +112,11 @@ public class DeployStateTest { @Test public void testRotations() { Set<Rotation> rotations = new HashSet<>(); - assertThat(new DeployState.Builder().rotations(rotations).build(true).getRotations().size(), is(0)); + assertThat(new DeployState.Builder().rotations(rotations).build().getRotations().size(), is(0)); for (String name : new String[]{"rotation-001.vespa.a02.yahoodns.net", "rotation-002.vespa.a02.yahoodns.net"}) { rotations.add(new Rotation(name)); } - assertThat(new DeployState.Builder().rotations(rotations).build(true).getRotations(), equalTo(rotations)); + assertThat(new DeployState.Builder().rotations(rotations).build().getRotations(), equalTo(rotations)); } private DeployState createDeployState(ApplicationPackage app, Map<ConfigDefinitionKey, com.yahoo.vespa.config.buildergen.ConfigDefinition> defs) { @@ -131,7 +131,7 @@ public class DeployStateTest { return null; } }); - return builder.build(true); + return builder.build(); } } diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java index 56b9ad04f78..85c75309d23 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java @@ -1523,7 +1523,7 @@ public class ModelProvisioningTest { ApplicationPackage appPkg = modelCreatorWithMockPkg.appPkg; DeployState deployState = new DeployState.Builder().applicationPackage(appPkg). properties((new DeployProperties.Builder()).multitenant(multitenant).build()). - build(true); + build(); return modelCreatorWithMockPkg.create(false, deployState); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/VespaModelFactoryTest.java b/config-model/src/test/java/com/yahoo/vespa/model/VespaModelFactoryTest.java index 2261affb65b..094494073df 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/VespaModelFactoryTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/VespaModelFactoryTest.java @@ -12,6 +12,7 @@ import com.yahoo.config.model.api.Model; import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.api.ModelCreateResult; import com.yahoo.config.model.api.ServiceInfo; +import com.yahoo.config.model.api.ValidationParameters; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; @@ -65,14 +66,14 @@ public class VespaModelFactoryTest { @Test(expected = IllegalArgumentException.class) public void testThatFactoryModelValidationFailsWithIllegalArgumentException() { VespaModelFactory modelFactory = new VespaModelFactory(new NullConfigModelRegistry()); - modelFactory.createAndValidateModel(new MockModelContext(createApplicationPackageThatFailsWhenValidating()), false); + modelFactory.createAndValidateModel(new MockModelContext(createApplicationPackageThatFailsWhenValidating()), new ValidationParameters()); } // Uses a MockApplicationPackage that throws throws UnsupportedOperationException (rethrown as RuntimeException) when validating @Test(expected = RuntimeException.class) public void testThatFactoryModelValidationFails() { VespaModelFactory modelFactory = new VespaModelFactory(new NullConfigModelRegistry()); - modelFactory.createAndValidateModel(testModelContext, false); + modelFactory.createAndValidateModel(testModelContext, new ValidationParameters()); } @Test @@ -80,7 +81,7 @@ public class VespaModelFactoryTest { VespaModelFactory modelFactory = new VespaModelFactory(new NullConfigModelRegistry()); ModelCreateResult createResult = modelFactory.createAndValidateModel( new MockModelContext(createApplicationPackageThatFailsWhenValidating()), - true); + new ValidationParameters(ValidationParameters.IgnoreValidationErrors.TRUE)); assertNotNull(createResult.getModel()); assertNotNull(createResult.getConfigChangeActions()); assertTrue(createResult.getConfigChangeActions().isEmpty()); @@ -212,6 +213,9 @@ public class VespaModelFactoryTest { @Override public boolean isBootstrap() { return false; } + + @Override + public boolean isFirstTimeDeployment() { return false; } }; } }; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java index 5d8a49b86eb..9cafe6541e4 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java @@ -161,7 +161,7 @@ public class AdminTestCase { .instanceName("bim") .build()) .build()) - .build(true); + .build(); TestRoot root = new TestDriver().buildModel(state); String localhost = HostName.getLocalhost(); SentinelConfig config = root.getConfig(SentinelConfig.class, "hosts/" + localhost); @@ -293,7 +293,7 @@ public class AdminTestCase { applicationName("foo").instanceName("bim") .build()) .build()) - .build(true); + .build(); TestRoot root = new TestDriver().buildModel(state); String localhost = HostName.getLocalhost(); SentinelConfig sentinelConfig = root.getConfig(SentinelConfig.class, "hosts/" + localhost); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/DedicatedAdminV4Test.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/DedicatedAdminV4Test.java index dbff72b4125..7b586354394 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/DedicatedAdminV4Test.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/DedicatedAdminV4Test.java @@ -204,7 +204,7 @@ public class DedicatedAdminV4Test { return new VespaModel(new NullConfigModelRegistry(), new DeployState.Builder().applicationPackage(app).modelHostProvisioner( new InMemoryProvisioner(Hosts.readFrom(app.getHosts()), true)) - .build(true)); + .build()); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidatorTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidatorTestCase.java index 146369d1620..6483933385d 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidatorTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidatorTestCase.java @@ -3,6 +3,8 @@ package com.yahoo.vespa.model.application.validation; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.NullConfigModelRegistry; +import com.yahoo.config.model.api.ValidationParameters; +import com.yahoo.config.model.api.ValidationParameters.CheckRouting; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.vespa.model.VespaModel; @@ -49,7 +51,8 @@ public class ComplexAttributeFieldsValidatorTestCase { private static void createModelAndValidate(String searchDefinition) throws IOException, SAXException { DeployState deployState = createDeployState(servicesXml(), searchDefinition); VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); - Validation.validate(model, false, false, deployState); + ValidationParameters validationParameters = new ValidationParameters(CheckRouting.FALSE); + Validation.validate(model, validationParameters, deployState); } private static DeployState createDeployState(String servicesXml, String searchDefinition) { @@ -57,7 +60,7 @@ public class ComplexAttributeFieldsValidatorTestCase { .withServices(servicesXml) .withSearchDefinition(searchDefinition) .build(); - return new DeployState.Builder().applicationPackage(app).build(true); + return new DeployState.Builder().applicationPackage(app).build(); } private static String servicesXml() { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/DeploymentFileValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/DeploymentFileValidatorTest.java index 9f8ff485d72..86e9ccb8a29 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/DeploymentFileValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/DeploymentFileValidatorTest.java @@ -56,7 +56,7 @@ public class DeploymentFileValidatorTest { .build(); DeployState.Builder builder = new DeployState.Builder().applicationPackage(app); try { - final DeployState deployState = builder.build(true); + final DeployState deployState = builder.build(); VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); new DeploymentFileValidator().validate(model, deployState); fail("Did not get expected exception"); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/SecretStoreValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/SecretStoreValidatorTest.java index cac3e65de89..03d004e93a4 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/SecretStoreValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/SecretStoreValidatorTest.java @@ -84,7 +84,7 @@ public class SecretStoreValidatorTest { .properties(new DeployProperties.Builder() .hostedVespa(true) .build()); - final DeployState deployState = builder.build(true); + final DeployState deployState = builder.build(); assertTrue("Test must emulate a hosted deployment.", deployState.isHosted()); return deployState; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/first/AccessControlValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/first/AccessControlValidatorTest.java index 3f109b53bd9..d3549eb6513 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/first/AccessControlValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/first/AccessControlValidatorTest.java @@ -144,7 +144,7 @@ public class AccessControlValidatorTest { .properties(new DeployProperties.Builder() .hostedVespa(true) .build()); - final DeployState deployState = builder.build(true); + final DeployState deployState = builder.build(); assertTrue("Test must emulate a hosted deployment.", deployState.isHosted()); assertEquals("Test must emulate a prod environment.", prod, deployState.zone().environment()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java index 2dbd15b600b..d9c151480fe 100755 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java @@ -78,7 +78,7 @@ public class ContainerClusterTest { public void requreThatWeCanGetTheZoneConfig() { DeployState state = new DeployState.Builder().properties(new DeployProperties.Builder().hostedVespa(true).build()) .zone(new Zone(SystemName.cd, Environment.test, RegionName.from("some-region"))) - .build(true); + .build(); MockRoot root = new MockRoot("foo", state); ContainerCluster cluster = new ContainerCluster(root, "container0", "container1"); ConfigserverConfig.Builder builder = new ConfigserverConfig.Builder(); @@ -107,7 +107,7 @@ public class ContainerClusterTest { } private ContainerCluster createContainerCluster(boolean isHosted, boolean isCombinedCluster, Optional<Integer> memoryPercentage, Optional<ContainerClusterVerifier> extraComponents) { - DeployState state = new DeployState.Builder().properties(new DeployProperties.Builder().hostedVespa(isHosted).build()).build(true); + DeployState state = new DeployState.Builder().properties(new DeployProperties.Builder().hostedVespa(isHosted).build()).build(); MockRoot root = new MockRoot("foo", state); ContainerCluster cluster = extraComponents.isPresent() @@ -255,7 +255,7 @@ public class ContainerClusterTest { @Test public void requireThatRoutingProviderIsDisabledForNonHosted() { - DeployState state = new DeployState.Builder().properties(new DeployProperties.Builder().hostedVespa(false).build()).build(true); + DeployState state = new DeployState.Builder().properties(new DeployProperties.Builder().hostedVespa(false).build()).build(); MockRoot root = new MockRoot("foo", state); ContainerCluster cluster = new ContainerCluster(root, "container0", "container1"); RoutingProviderConfig.Builder builder = new RoutingProviderConfig.Builder(); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/ConfigserverClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/ConfigserverClusterTest.java index 0c91b8d4a16..d4209c9c788 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/ConfigserverClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/ConfigserverClusterTest.java @@ -38,7 +38,7 @@ public class ConfigserverClusterTest { new ConfigServerContainerModelBuilder(new TestOptions().rpcPort(12345).useVespaVersionInRequest(true) .hostedVespa(true).environment("test").region("bar") .numParallelTenantLoaders(99)) - .build(new DeployState.Builder().build(true), null, root, XML.getDocument(services).getDocumentElement()); + .build(new DeployState.Builder().build(), null, root, XML.getDocument(services).getDocumentElement()); root.freezeModelTopology(); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java index 0fbe44742de..e3dfa093735 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java @@ -116,7 +116,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { .properties(new DeployProperties.Builder() .hostedVespa(true) .build()) - .build(true)); + .build()); assertFalse(logger.msgs.isEmpty()); assertThat(logger.msgs.get(0).getSecond(), containsString(String.format("You cannot set port to anything else than %d", Container.BASEPORT))); } @@ -512,7 +512,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { " </nodes>", "</jdisc>"); - DeployState deployState = new DeployState.Builder().zone(new Zone(Environment.dev, RegionName.from("us-east-1"))).build(true); + DeployState deployState = new DeployState.Builder().zone(new Zone(Environment.dev, RegionName.from("us-east-1"))).build(); createModel(root, deployState, clusterElem); assertEquals(0, getContainerCluster("default").serviceAliases().size()); assertEquals(0, getContainerCluster("default").endpointAliases().size()); @@ -532,7 +532,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { .multitenant(true) .hostedVespa(true) .build()) - .build(true)); + .build()); assertEquals(1, model.getHostSystem().getHosts().size()); } @@ -565,7 +565,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { .properties(new DeployProperties.Builder() .hostedVespa(true) .build()) - .build(true)); + .build()); AbstractConfigProducerRoot modelRoot = model.getRoot(); VipStatusConfig vipStatusConfig = modelRoot.getConfig(VipStatusConfig.class, "jdisc/component/status.html-status-handler"); @@ -593,7 +593,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { VespaModel model = new VespaModel(new NullConfigModelRegistry(), new DeployState.Builder() .applicationPackage(applicationPackage) .properties(new DeployProperties.Builder().build()) - .build(true)); + .build()); String hostname = HostName.getLocalhost(); // Using the same way of getting hostname as filedistribution model diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/RoutingBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/RoutingBuilderTest.java index 8ba6aacdc41..a2f32694340 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/RoutingBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/RoutingBuilderTest.java @@ -67,7 +67,7 @@ public class RoutingBuilderTest extends ContainerModelBuilderTestBase { DeployState deployState = new DeployState.Builder() .applicationPackage(applicationPackage) .zone(new Zone(Environment.prod, RegionName.from(region))) - .build(true); + .build(); root = new MockRoot("root", deployState); createModel(root, deployState, clusterElem); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterUtils.java b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterUtils.java index 8e0c0d0b253..b0d6c94947a 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterUtils.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterUtils.java @@ -39,10 +39,10 @@ public class ContentClusterUtils { private static MockRoot createMockRoot(HostProvisioner provisioner, List<String> searchDefinitions, DeployState.Builder deployStateBuilder) { ApplicationPackage applicationPackage = new MockApplicationPackage.Builder().withSearchDefinitions(searchDefinitions).build(); - deployStateBuilder.applicationPackage(applicationPackage) + DeployState deployState = deployStateBuilder.applicationPackage(applicationPackage) .modelHostProvisioner(provisioner) - .build(true); - return new MockRoot("", deployStateBuilder.build(true)); + .build(); + return new MockRoot("", deployState); } public static MockRoot createMockRoot(String[] hosts, List<String> searchDefinitions) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java index ce72e784125..1a5ce0a10f4 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java @@ -10,6 +10,7 @@ import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.NullConfigModelRegistry; import com.yahoo.config.model.api.HostInfo; +import com.yahoo.config.model.api.ValidationParameters; import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.config.model.deploy.DeployProperties; import com.yahoo.config.model.deploy.DeployState; @@ -24,7 +25,6 @@ import com.yahoo.messagebus.MessagebusConfig; import com.yahoo.net.HostName; import com.yahoo.vespa.config.UnknownConfigIdException; import com.yahoo.vespa.model.ConfigProducer; -import com.yahoo.vespa.model.HostSystem; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.admin.Admin; import com.yahoo.vespa.model.admin.Configserver; @@ -246,9 +246,9 @@ public class VespaModelTestCase { .withHosts(simpleHosts) .withServices(services) .build(); - DeployState deployState = builder.deployLogger(logger).applicationPackage(app).build(true); + DeployState deployState = builder.deployLogger(logger).applicationPackage(app).build(); VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); - Validation.validate(model, true, deployState); + Validation.validate(model, new ValidationParameters(ValidationParameters.IgnoreValidationErrors.TRUE), deployState); assertFalse(logger.msgs.isEmpty()); } @@ -283,7 +283,7 @@ public class VespaModelTestCase { .configServerSpecs(Arrays.asList(new Configserver.Spec("cfghost", 1234, 1235, 1236))) .multitenant(true) .build()) - .build(true); + .build(); VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); AllocatedHosts info = model.allocatedHosts(); assertEquals("Admin version 3 is ignored, and there are no other hosts to borrow for admin services", 0, info.getHosts().size()); @@ -302,9 +302,9 @@ public class VespaModelTestCase { public void testPermanentServices() throws IOException, SAXException { ApplicationPackage app = MockApplicationPackage.createEmpty(); DeployState.Builder builder = new DeployState.Builder().applicationPackage(app); - VespaModel model = new VespaModel(new NullConfigModelRegistry(), builder.build(true)); + VespaModel model = new VespaModel(new NullConfigModelRegistry(), builder.build()); assertThat(model.getContainerClusters().size(), is(0)); - model = new VespaModel(new NullConfigModelRegistry(), builder.permanentApplicationPackage(Optional.of(FilesApplicationPackage.fromFile(new File(TESTDIR, "app_permanent")))).build(true)); + model = new VespaModel(new NullConfigModelRegistry(), builder.permanentApplicationPackage(Optional.of(FilesApplicationPackage.fromFile(new File(TESTDIR, "app_permanent")))).build()); assertThat(model.getContainerClusters().size(), is(1)); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java index 715b84c7093..8cc5144c2a3 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java @@ -124,7 +124,7 @@ public class VespaModelTester { .applicationPackage(appPkg) .modelHostProvisioner(provisioner) .properties(properties) - .build(true); + .build(); return modelCreatorWithMockPkg.create(false, deployState, configModelRegistry); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithFilePkg.java b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithFilePkg.java index 18fbf68497f..59c35913e62 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithFilePkg.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithFilePkg.java @@ -4,6 +4,10 @@ package com.yahoo.vespa.model.test.utils; import com.yahoo.component.Version; import com.yahoo.config.model.ConfigModelRegistry; import com.yahoo.config.model.NullConfigModelRegistry; +import com.yahoo.config.model.api.ValidationParameters; +import com.yahoo.config.model.api.ValidationParameters.CheckRouting; +import com.yahoo.config.model.api.ValidationParameters.FailOnIncompatibleChange; +import com.yahoo.config.model.api.ValidationParameters.IgnoreValidationErrors; import com.yahoo.config.model.application.provider.*; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.vespa.model.VespaModel; @@ -56,12 +60,13 @@ public class VespaModelCreatorWithFilePkg { if (validateApplicationWithSchema) { validate(); } - DeployState deployState = new DeployState.Builder().applicationPackage(applicationPkg).build(true); + DeployState deployState = new DeployState.Builder().applicationPackage(applicationPkg).build(); VespaModel model = new VespaModel(configModelRegistry, deployState); // Validate, but without checking configSources or routing (routing // is constructed in a special way and cannot always be validated in // this step for unit tests) - Validation.validate(model, false, false, deployState); + ValidationParameters validationParameters = new ValidationParameters(IgnoreValidationErrors.TRUE, FailOnIncompatibleChange.TRUE, CheckRouting.FALSE); + Validation.validate(model, validationParameters, deployState); return model; } catch (Exception e) { throw e instanceof RuntimeException ? (RuntimeException) e : new RuntimeException(e); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithMockPkg.java b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithMockPkg.java index 211589c100d..0a445ffc8b7 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithMockPkg.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithMockPkg.java @@ -6,6 +6,8 @@ import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.ConfigModelRegistry; import com.yahoo.config.model.NullConfigModelRegistry; import com.yahoo.config.model.api.ConfigChangeAction; +import com.yahoo.config.model.api.ValidationParameters; +import com.yahoo.config.model.api.ValidationParameters.CheckRouting; import com.yahoo.config.model.application.provider.SchemaValidators; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.test.MockApplicationPackage; @@ -38,12 +40,12 @@ public class VespaModelCreatorWithMockPkg { } public VespaModel create() { - DeployState deployState = new DeployState.Builder().applicationPackage(appPkg).build(true); + DeployState deployState = new DeployState.Builder().applicationPackage(appPkg).build(); return create(true, deployState); } public VespaModel create(DeployState.Builder deployStateBuilder) { - return create(true, deployStateBuilder.applicationPackage(appPkg).build(true)); + return create(true, deployStateBuilder.applicationPackage(appPkg).build()); } public VespaModel create(boolean validate, DeployState deployState) { @@ -72,7 +74,8 @@ public class VespaModelCreatorWithMockPkg { // Validate, but without checking configSources or routing (routing // is constructed in a special way and cannot always be validated in // this step for unit tests) - configChangeActions = Validation.validate(model, false, false, deployState); + ValidationParameters validationParameters = new ValidationParameters(CheckRouting.FALSE); + configChangeActions = Validation.validate(model, validationParameters, deployState); } return model; } catch (Exception e) { diff --git a/configdefinitions/src/vespa/configserver.def b/configdefinitions/src/vespa/configserver.def index 228a5c6fb4f..b92a865783a 100644 --- a/configdefinitions/src/vespa/configserver.def +++ b/configdefinitions/src/vespa/configserver.def @@ -61,3 +61,6 @@ tenantsMaintainerIntervalMinutes int default=525600 # How long bootstrapping can take before giving up (in seconds) maxDurationOfBootstrap long default=7200 + +# Feature Flags (poor man's feature flags, to be overridden in configserver-config.xml if needed) +deleteApplicationLegacy bool default=false diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 1bfe14566f7..94d758237de 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -270,10 +270,12 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye */ public boolean delete(ApplicationId applicationId) { // TODO: Use deleteApplication() in all zones - if ( ! configserverConfig.hostedVespa() || SystemName.from(configserverConfig.system()) == SystemName.cd) { - return deleteApplication(applicationId); - } else { + if (configserverConfig.deleteApplicationLegacy() || + (configserverConfig.hostedVespa() && SystemName.from(configserverConfig.system()) == SystemName.main + && !Arrays.asList("corp-us-east-1", "aws-us-east-1a").contains(configserverConfig.region()))) { return deleteApplicationLegacy(applicationId); + } else { + return deleteApplication(applicationId); } } @@ -648,6 +650,9 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye Set<ApplicationId> applicationsNotRedeployed = listApplications(); do { applicationsNotRedeployed = redeployApplications(applicationsNotRedeployed); + if ( ! applicationsNotRedeployed.isEmpty()) { + Thread.sleep(Duration.ofSeconds(30).toMillis()); + } } while ( ! applicationsNotRedeployed.isEmpty() && Instant.now().isBefore(end)); if ( ! applicationsNotRedeployed.isEmpty()) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 28dc0cc8414..914d6963ff0 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -124,6 +124,7 @@ public class ModelContextImpl implements ModelContext { private final Zone zone; private final Set<Rotation> rotations; private final boolean isBootstrap; + private final boolean isFirstTimeDeployment; public Properties(ApplicationId applicationId, boolean multitenant, @@ -134,7 +135,8 @@ public class ModelContextImpl implements ModelContext { boolean hostedVespa, Zone zone, Set<Rotation> rotations, - boolean isBootstrap) { + boolean isBootstrap, + boolean isFirstTimeDeployment) { this.applicationId = applicationId; this.multitenant = multitenant; this.configServerSpecs = configServerSpecs; @@ -145,6 +147,7 @@ public class ModelContextImpl implements ModelContext { this.zone = zone; this.rotations = rotations; this.isBootstrap = isBootstrap; + this.isFirstTimeDeployment = isFirstTimeDeployment; } @Override @@ -181,6 +184,8 @@ public class ModelContextImpl implements ModelContext { @Override public boolean isBootstrap() { return isBootstrap; } + @Override + public boolean isFirstTimeDeployment() { return isFirstTimeDeployment; } } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java index 314b6d78ae3..2955d948d69 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java @@ -116,7 +116,8 @@ public class ActivatedModelsBuilder extends ModelsBuilder<Application> { configserverConfig.hostedVespa(), zone(), new Rotations(curator, TenantRepository.getTenantPath(tenant)).readRotationsFromZooKeeper(applicationId), - false); // We may be bootstrapping, but we only know and care during prepare + false, // We may be bootstrapping, but we only know and care during prepare + false); // Always false, assume no one uses it when activating } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java index 969a2d49586..090e7aac494 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java @@ -8,6 +8,7 @@ import com.yahoo.config.model.api.ModelFactory; import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationLockException; +import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.OutOfCapacityException; import com.yahoo.config.provision.Version; import com.yahoo.config.provision.Zone; @@ -19,6 +20,7 @@ import com.yahoo.vespa.config.server.provision.StaticProvisioner; import java.time.Instant; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.Comparator; import java.util.List; @@ -140,7 +142,10 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> { List<MODELRESULT> allApplicationVersions = new ArrayList<>(); allApplicationVersions.add(latestModelVersion); - versions = keepThoseUsedOn(allocatedHosts.get(), versions); + // TODO: Enable for all zones + // Note: Cannot be enabled for prod zones yet, due to an issue with how AccessControlValidator works + if (Arrays.asList(Environment.dev, Environment.test, Environment.staging).contains(zone().environment())) + versions = keepThoseUsedOn(allocatedHosts.get(), versions); // Make sure we build wanted version if we are building models for this major version Version wantedVersion = Version.fromIntValues(wantedNodeVespaVersion.getMajor(), wantedNodeVespaVersion.getMinor(), wantedNodeVespaVersion.getMicro()); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java index 0d9346101e9..ba093ca9db1 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java @@ -10,6 +10,8 @@ import com.yahoo.config.model.api.Model; import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.api.ModelCreateResult; import com.yahoo.config.model.api.ModelFactory; +import com.yahoo.config.model.api.ValidationParameters; +import com.yahoo.config.model.api.ValidationParameters.IgnoreValidationErrors; import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.AllocatedHosts; @@ -104,7 +106,9 @@ public class PreparedModelsBuilder extends ModelsBuilder<PreparedModelsBuilder.P wantedNodeVespaVersion); log.log(LogLevel.DEBUG, "Create and validate model " + modelVersion + " for " + applicationId); - ModelCreateResult result = modelFactory.createAndValidateModel(modelContext, params.ignoreValidationErrors()); + ValidationParameters validationParameters = + new ValidationParameters(params.ignoreValidationErrors() ? IgnoreValidationErrors.TRUE : IgnoreValidationErrors.FALSE); + ModelCreateResult result = modelFactory.createAndValidateModel(modelContext, validationParameters); validateModelHosts(context.getHostValidator(), applicationId, result.getModel()); log.log(LogLevel.DEBUG, "Done building model " + modelVersion + " for " + applicationId); return new PreparedModelsBuilder.PreparedModelResult(modelVersion, result.getModel(), fileDistributionProvider, result.getConfigChangeActions()); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java index 0d9f8ce64b1..6a240806004 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java @@ -158,7 +158,8 @@ public class SessionPreparer { configserverConfig.hostedVespa(), zone, rotationsSet, - params.isBootstrap()); + params.isBootstrap(), + ! currentActiveApplicationSet.isPresent()); this.preparedModelsBuilder = new PreparedModelsBuilder(modelFactoryRegistry, permanentApplicationPackage, configDefinitionRepo, diff --git a/configserver/src/test/apps/hosted-no-write-access-control/searchdefinitions/music.sd b/configserver/src/test/apps/hosted-no-write-access-control/searchdefinitions/music.sd new file mode 100644 index 00000000000..78d58b27d4a --- /dev/null +++ b/configserver/src/test/apps/hosted-no-write-access-control/searchdefinitions/music.sd @@ -0,0 +1,10 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +search music { + document music { + field title type string { + indexing: index | summary + # index-to: default + } + } +} + diff --git a/configserver/src/test/apps/hosted-no-write-access-control/services.xml b/configserver/src/test/apps/hosted-no-write-access-control/services.xml new file mode 100644 index 00000000000..c2257ab34f7 --- /dev/null +++ b/configserver/src/test/apps/hosted-no-write-access-control/services.xml @@ -0,0 +1,25 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<services version="1.0"> + + <admin version="3.0"> + <nodes count='1'/> + </admin> + + <jdisc version="1.0"> + <http> + <server id="foo" port="4080" /> + </http> + <search/> + <nodes count='1'/> + </jdisc> + + <content id="music" version="1.0"> + <redundancy>1</redundancy> + <documents> + <document type="music" mode="index" /> + </documents> + <nodes count="2" groups="2"/> + </content> + +</services> diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java index f9c99b323af..dbb02e367a7 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java @@ -2,10 +2,8 @@ package com.yahoo.vespa.config.server; import com.yahoo.cloud.config.ConfigserverConfig; -import com.yahoo.config.model.api.HostProvisioner; import com.yahoo.config.model.provision.InMemoryProvisioner; import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.Version; import com.yahoo.container.handler.VipStatus; import com.yahoo.container.jdisc.config.HealthMonitorConfig; import com.yahoo.container.jdisc.state.StateMonitor; @@ -41,8 +39,8 @@ public class ConfigServerBootstrapTest { public void testBootstrap() throws Exception { ConfigserverConfig configserverConfig = createConfigserverConfig(temporaryFolder); InMemoryProvisioner provisioner = new InMemoryProvisioner(true, "host0", "host1", "host3"); - DeployTester tester = new DeployTester("src/test/apps/hosted/", configserverConfig, provisioner); - tester.deployApp("myApp", "4.5.6", Instant.now()); + DeployTester tester = new DeployTester(configserverConfig, provisioner); + tester.deployApp("src/test/apps/hosted/", "myApp", "4.5.6", Instant.now()); File versionFile = temporaryFolder.newFile(); VersionState versionState = new VersionState(versionFile); @@ -72,8 +70,8 @@ public class ConfigServerBootstrapTest { @Test public void testBootstrapWhenRedeploymentFails() throws Exception { ConfigserverConfig configserverConfig = createConfigserverConfig(temporaryFolder); - DeployTester tester = new DeployTester("src/test/apps/hosted/", configserverConfig); - tester.deployApp("myApp", "4.5.6", Instant.now()); + DeployTester tester = new DeployTester(configserverConfig); + tester.deployApp("src/test/apps/hosted/", "myApp", "4.5.6", Instant.now()); File versionFile = temporaryFolder.newFile(); VersionState versionState = new VersionState(versionFile); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java index ce53dc3f2fb..28fc179770a 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java @@ -51,6 +51,7 @@ public class ModelContextImplTest { false, Zone.defaultZone(), rotations, + false, false), Optional.empty(), new Version(6), @@ -67,5 +68,6 @@ public class ModelContextImplTest { assertTrue(context.properties().zone() instanceof Zone); assertFalse(context.properties().hostedVespa()); assertThat(context.properties().rotations(), equalTo(rotations)); + assertThat(context.properties().isFirstTimeDeployment(), equalTo(false)); } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ModelFactoryRegistryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ModelFactoryRegistryTest.java index 761d78ee7e0..65cbe65f0d9 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ModelFactoryRegistryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ModelFactoryRegistryTest.java @@ -6,6 +6,7 @@ import com.yahoo.config.model.api.Model; import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.api.ModelCreateResult; import com.yahoo.config.model.api.ModelFactory; +import com.yahoo.config.model.api.ValidationParameters; import com.yahoo.config.provision.Version; import com.yahoo.vespa.config.server.http.UnknownVespaVersionException; import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry; @@ -88,7 +89,7 @@ public class ModelFactoryRegistryTest { } @Override - public ModelCreateResult createAndValidateModel(ModelContext modelContext, boolean ignoreValidationErrors) { + public ModelCreateResult createAndValidateModel(ModelContext modelContext, ValidationParameters validationParameters) { return null; } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java index b15356a172e..8685065d001 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java @@ -10,6 +10,7 @@ import com.yahoo.config.model.api.Model; import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.api.ModelCreateResult; import com.yahoo.config.model.api.ModelFactory; +import com.yahoo.config.model.api.ValidationParameters; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.provision.InMemoryProvisioner; import com.yahoo.config.model.test.MockApplicationPackage; @@ -62,58 +63,56 @@ public class DeployTester { private final Clock clock; private final TenantRepository tenantRepository; - private final File testApp; private final ApplicationRepository applicationRepository; private ApplicationId id; - public DeployTester(String appPath) { - this(appPath, Collections.singletonList(createModelFactory(Clock.systemUTC()))); + public DeployTester() { + this(Collections.singletonList(createModelFactory(Clock.systemUTC()))); } - public DeployTester(String appPath, List<ModelFactory> modelFactories) { - this(appPath, modelFactories, + public DeployTester(List<ModelFactory> modelFactories) { + this(modelFactories, new ConfigserverConfig(new ConfigserverConfig.Builder() .configServerDBDir(Files.createTempDir().getAbsolutePath()) .configDefinitionsDir(Files.createTempDir().getAbsolutePath())), Clock.systemUTC()); } - public DeployTester(String appPath, ConfigserverConfig configserverConfig) { - this(appPath, Collections.singletonList(createModelFactory(Clock.systemUTC())), configserverConfig, Clock.systemUTC()); + public DeployTester(ConfigserverConfig configserverConfig) { + this(Collections.singletonList(createModelFactory(Clock.systemUTC())), configserverConfig, Clock.systemUTC()); } - public DeployTester(String appPath, ConfigserverConfig configserverConfig, HostProvisioner provisioner) { - this(appPath, Collections.singletonList(createModelFactory(Clock.systemUTC())), configserverConfig, Clock.systemUTC(), provisioner); + public DeployTester(ConfigserverConfig configserverConfig, HostProvisioner provisioner) { + this(Collections.singletonList(createModelFactory(Clock.systemUTC())), configserverConfig, Clock.systemUTC(), provisioner); } - public DeployTester(String appPath, ConfigserverConfig configserverConfig, Clock clock) { - this(appPath, Collections.singletonList(createModelFactory(clock)), configserverConfig, clock); + public DeployTester(ConfigserverConfig configserverConfig, Clock clock) { + this(Collections.singletonList(createModelFactory(clock)), configserverConfig, clock); } - public DeployTester(String appPath, List<ModelFactory> modelFactories, ConfigserverConfig configserverConfig) { - this(appPath, modelFactories, configserverConfig, Clock.systemUTC()); + public DeployTester(List<ModelFactory> modelFactories, ConfigserverConfig configserverConfig) { + this(modelFactories, configserverConfig, Clock.systemUTC()); } - public DeployTester(String appPath, List<ModelFactory> modelFactories, ConfigserverConfig configserverConfig, Clock clock) { - this(appPath, modelFactories, configserverConfig, clock, Zone.defaultZone()); + public DeployTester(List<ModelFactory> modelFactories, ConfigserverConfig configserverConfig, Clock clock) { + this(modelFactories, configserverConfig, clock, Zone.defaultZone()); } - public DeployTester(String appPath, List<ModelFactory> modelFactories, ConfigserverConfig configserverConfig, Clock clock, HostProvisioner provisioner) { - this(appPath, modelFactories, configserverConfig, clock, Zone.defaultZone(), provisioner); + public DeployTester(List<ModelFactory> modelFactories, ConfigserverConfig configserverConfig, Clock clock, HostProvisioner provisioner) { + this(modelFactories, configserverConfig, clock, Zone.defaultZone(), provisioner); } - public DeployTester(String appPath, List<ModelFactory> modelFactories, ConfigserverConfig configserverConfig, Clock clock, Zone zone) { - this(appPath, modelFactories, configserverConfig, clock, zone, createProvisioner()); + public DeployTester(List<ModelFactory> modelFactories, ConfigserverConfig configserverConfig, Clock clock, Zone zone) { + this(modelFactories, configserverConfig, clock, zone, createProvisioner()); } - public DeployTester(String appPath, List<ModelFactory> modelFactories, ConfigserverConfig configserverConfig, Clock clock, Zone zone, HostProvisioner provisioner) { + public DeployTester(List<ModelFactory> modelFactories, ConfigserverConfig configserverConfig, Clock clock, Zone zone, HostProvisioner provisioner) { this.clock = clock; TestComponentRegistry componentRegistry = createComponentRegistry(new MockCurator(), Metrics.createTestMetrics(), modelFactories, configserverConfig, clock, zone, provisioner); try { - this.testApp = new File(appPath); this.tenantRepository = new TenantRepository(componentRegistry); tenantRepository.addTenant(tenantName); } @@ -133,6 +132,11 @@ public class DeployTester { } /** Create a model factory for a particular version */ + public static CountingModelFactory createModelFactory(Version version) { + return new CountingModelFactory(version, Clock.systemUTC()); + } + + /** Create a model factory for a particular version */ public static CountingModelFactory createModelFactory(Version version, Clock clock) { return new CountingModelFactory(version, clock); } @@ -143,14 +147,14 @@ public class DeployTester { /** * Do the initial "deploy" with the existing API-less code as the deploy API doesn't support first deploys yet. */ - public ApplicationId deployApp(String appName, Instant now) { - return deployApp(appName, null, now); + public ApplicationId deployApp(String applicationPath, String appName, Instant now) { + return deployApp(applicationPath, appName, null, now); } /** * Do the initial "deploy" with the existing API-less code as the deploy API doesn't support first deploys yet. */ - public ApplicationId deployApp(String appName, String vespaVersion, Instant now) { + public ApplicationId deployApp(String applicationPath, String appName, String vespaVersion, Instant now) { Tenant tenant = tenant(); TimeoutBudget timeoutBudget = new TimeoutBudget(clock, Duration.ofSeconds(60)); ApplicationId id = ApplicationId.from(tenant.getName(), ApplicationName.from(appName), InstanceName.defaultName()); @@ -158,7 +162,7 @@ public class DeployTester { if (vespaVersion != null) paramsBuilder.vespaVersion(vespaVersion); - long sessionId = applicationRepository.createSession(id, timeoutBudget, testApp); + long sessionId = applicationRepository.createSession(id, timeoutBudget, new File(applicationPath)); applicationRepository.prepare(tenant, sessionId, paramsBuilder.build(), now); applicationRepository.activate(tenant, sessionId, timeoutBudget, false, false); this.id = id; @@ -256,7 +260,7 @@ public class DeployTester { try { Instant now = LocalDate.parse("2000-01-01", DateTimeFormatter.ISO_DATE).atStartOfDay().atZone(ZoneOffset.UTC).toInstant(); ApplicationPackage application = new MockApplicationPackage.Builder().withEmptyHosts().withEmptyServices().build(); - DeployState deployState = new DeployState.Builder().applicationPackage(application).now(now).build(true); + DeployState deployState = new DeployState.Builder().applicationPackage(application).now(now).build(); return new VespaModel(deployState); } catch (Exception e) { throw new RuntimeException(e); @@ -264,8 +268,8 @@ public class DeployTester { } @Override - public ModelCreateResult createAndValidateModel(ModelContext modelContext, boolean ignoreValidationErrors) { - if ( ! ignoreValidationErrors) + public ModelCreateResult createAndValidateModel(ModelContext modelContext, ValidationParameters validationParameters) { + if ( ! validationParameters.ignoreValidationErrors()) throw new IllegalArgumentException("Validation fails"); return new ModelCreateResult(createModel(modelContext), Collections.emptyList()); } @@ -300,8 +304,8 @@ public class DeployTester { } @Override - public ModelCreateResult createAndValidateModel(ModelContext modelContext, boolean ignoreValidationErrors) { - ModelCreateResult result = wrapped.createAndValidateModel(modelContext, ignoreValidationErrors); + public ModelCreateResult createAndValidateModel(ModelContext modelContext, ValidationParameters validationParameters) { + ModelCreateResult result = wrapped.createAndValidateModel(modelContext, validationParameters); creationCount++; return result; } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java index 6ee4720b6b0..a184a461ce1 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java @@ -22,9 +22,12 @@ import java.time.Clock; import java.time.Duration; import java.time.Instant; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.stream.Collectors; +import java.util.stream.IntStream; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -39,8 +42,8 @@ public class HostedDeployTest { @Test public void testRedeployWithVersion() { CountingModelFactory modelFactory = DeployTester.createModelFactory(Version.fromString("4.5.6"), Clock.systemUTC()); - DeployTester tester = new DeployTester("src/test/apps/hosted/", Collections.singletonList(modelFactory), createConfigserverConfig()); - tester.deployApp("myApp", "4.5.6", Instant.now()); + DeployTester tester = new DeployTester(Collections.singletonList(modelFactory), createConfigserverConfig()); + tester.deployApp("src/test/apps/hosted/", "myApp", "4.5.6", Instant.now()); Optional<com.yahoo.config.provision.Deployment> deployment = tester.redeployFromLocalActive(); assertTrue(deployment.isPresent()); @@ -51,8 +54,8 @@ public class HostedDeployTest { @Test public void testRedeploy() { - DeployTester tester = new DeployTester("src/test/apps/hosted/", createConfigserverConfig()); - ApplicationId appId = tester.deployApp("myApp", Instant.now()); + DeployTester tester = new DeployTester(createConfigserverConfig()); + ApplicationId appId = tester.deployApp("src/test/apps/hosted/", "myApp", Instant.now()); LocalSession s1 = tester.applicationRepository().getActiveSession(appId); System.out.println("First session: " + s1.getSessionId()); assertFalse(tester.applicationRepository().getActiveSession(appId).getMetaData().isInternalRedeploy()); @@ -73,10 +76,9 @@ public class HostedDeployTest { modelFactories.add(DeployTester.createModelFactory(Version.fromString("6.1.0"), clock)); modelFactories.add(DeployTester.createModelFactory(Version.fromString("6.2.0"), clock)); modelFactories.add(DeployTester.createModelFactory(Version.fromString("7.0.0"), clock)); - DeployTester tester = new DeployTester("src/test/apps/hosted/", modelFactories, createConfigserverConfig(), clock, Zone.defaultZone()); - ApplicationId app = tester.deployApp("myApp", "6.2.0", Instant.now()); + DeployTester tester = new DeployTester(modelFactories, createConfigserverConfig(), clock, Zone.defaultZone()); + ApplicationId app = tester.deployApp("src/test/apps/hosted/", "myApp", "6.2.0", Instant.now()); assertEquals(3, tester.getAllocatedHostsOf(app).getHosts().size()); - } /** Test that only the minimal set of models are created (model versions used on hosts, the wanted version and the latest version) */ @@ -103,10 +105,10 @@ public class HostedDeployTest { modelFactories.add(factory710); modelFactories.add(factory720); - DeployTester tester = new DeployTester("src/test/apps/hosted/", modelFactories, createConfigserverConfig(), + DeployTester tester = new DeployTester(modelFactories, createConfigserverConfig(), clock, new Zone(Environment.dev, RegionName.defaultName()), provisioner); // Deploy with version that does not exist on hosts, the model for this version should also be created - ApplicationId app = tester.deployApp("myApp", "7.0.0", Instant.now()); + ApplicationId app = tester.deployApp("src/test/apps/hosted/", "myApp", "7.0.0", Instant.now()); assertEquals(3, tester.getAllocatedHostsOf(app).getHosts().size()); // Check >0 not ==0 as the session watcher thread is running and will redeploy models in the background @@ -119,14 +121,37 @@ public class HostedDeployTest { } @Test + public void testAccessControlIsOnlyCheckedWhenNoProdDeploymentExists() { + // Provisioner does not reuse hosts, so need twice as many hosts as app requires + List<Host> hosts = IntStream.rangeClosed(1,6).mapToObj(i -> createHost("host" + i, "6.0.0")).collect(Collectors.toList()); + InMemoryProvisioner provisioner = new InMemoryProvisioner(new Hosts(hosts), true); + + CountingModelFactory factory600 = DeployTester.createModelFactory(Version.fromString("6.0.0")); + CountingModelFactory factory610 = DeployTester.createModelFactory(Version.fromString("6.1.0")); + CountingModelFactory factory620 = DeployTester.createModelFactory(Version.fromString("6.2.0")); + List<ModelFactory> modelFactories = Arrays.asList(factory600, factory610, factory620); + + DeployTester tester = new DeployTester(modelFactories, createConfigserverConfig(), + Clock.systemUTC(), new Zone(Environment.prod, RegionName.defaultName()), provisioner); + // Deploy with oldest version + ApplicationId app = tester.deployApp("src/test/apps/hosted/", "myApp", "6.0.0", Instant.now()); + assertEquals(3, tester.getAllocatedHostsOf(app).getHosts().size()); + + // Deploy with version that does not exist on hosts and with app package that has no write access control, + // validation of access control should not be done, since the app is already deployed in prod + app = tester.deployApp("src/test/apps/hosted-no-write-access-control", "myApp", "6.1.0", Instant.now()); + assertEquals(3, tester.getAllocatedHostsOf(app).getHosts().size()); + } + + @Test public void testRedeployAfterExpiredValidationOverride() { // Old version of model fails, but application disables loading old models until 2016-10-10, so deployment works ManualClock clock = new ManualClock("2016-10-09T00:00:00"); List<ModelFactory> modelFactories = new ArrayList<>(); modelFactories.add(DeployTester.createModelFactory(clock)); modelFactories.add(DeployTester.createFailingModelFactory(Version.fromIntValues(1, 0, 0))); // older than default - DeployTester tester = new DeployTester("src/test/apps/validationOverride/", modelFactories, createConfigserverConfig()); - tester.deployApp("myApp", clock.instant()); + DeployTester tester = new DeployTester(modelFactories, createConfigserverConfig()); + tester.deployApp("src/test/apps/validationOverride/", "myApp", clock.instant()); // Redeployment from local active works { @@ -149,7 +174,7 @@ public class HostedDeployTest { // However, redeployment from the outside fails after this date { try { - tester.deployApp("myApp", Instant.now()); + tester.deployApp("src/test/apps/validationOverride/", "myApp", Instant.now()); fail("Expected redeployment to fail"); } catch (Exception expected) { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/RedeployTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/RedeployTest.java index fd023a19617..9bc9a93e9fa 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/RedeployTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/RedeployTest.java @@ -34,8 +34,8 @@ public class RedeployTest { @Test public void testRedeploy() { - DeployTester tester = new DeployTester("src/test/apps/app"); - tester.deployApp("myapp", Instant.now()); + DeployTester tester = new DeployTester(); + tester.deployApp("src/test/apps/app", "myapp", Instant.now()); Optional<com.yahoo.config.provision.Deployment> deployment = tester.redeployFromLocalActive(); assertTrue(deployment.isPresent()); @@ -54,7 +54,7 @@ public class RedeployTest { List<ModelFactory> modelFactories = new ArrayList<>(); modelFactories.add(DeployTester.createModelFactory(Clock.systemUTC())); modelFactories.add(DeployTester.createFailingModelFactory(Version.fromIntValues(1, 0, 0))); - DeployTester tester = new DeployTester("ignored/app/path", modelFactories); + DeployTester tester = new DeployTester(modelFactories); ApplicationId id = ApplicationId.from(tester.tenant().getName(), ApplicationName.from("default"), InstanceName.from("default")); @@ -70,8 +70,8 @@ public class RedeployTest { .configDefinitionsDir(Files.createTempDir() .getAbsolutePath()) .sessionLifetime(60)); - DeployTester tester = new DeployTester("src/test/apps/app", configserverConfig, clock); - tester.deployApp("myapp", Instant.now()); // session 2 (numbering starts at 2) + DeployTester tester = new DeployTester(configserverConfig, clock); + tester.deployApp("src/test/apps/app", "myapp", Instant.now()); // session 2 (numbering starts at 2) clock.advance(Duration.ofSeconds(10)); Optional<com.yahoo.config.provision.Deployment> deployment2 = tester.redeployFromLocalActive(); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java index 8b89027e4a1..6fa36ba27bb 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java @@ -197,7 +197,7 @@ public class LbServicesProducerTest { } private Model createVespaModel(ApplicationPackage applicationPackage, DeployState.Builder deployStateBuilder) throws IOException, SAXException { - return new VespaModel(new NullConfigModelRegistry(), deployStateBuilder.applicationPackage(applicationPackage).build(true)); + return new VespaModel(new NullConfigModelRegistry(), deployStateBuilder.applicationPackage(applicationPackage).build()); } private void assertConfig(LbServicesConfig expected, LbServicesConfig actual) { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/model/RoutingProducerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/model/RoutingProducerTest.java index 8c48cfe4b99..8a6d772ff14 100755 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/model/RoutingProducerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/model/RoutingProducerTest.java @@ -102,6 +102,6 @@ public class RoutingProducerTest { } private Model createVespaModel(ApplicationPackage applicationPackage, DeployState.Builder deployStateBuilder) throws IOException, SAXException { - return new VespaModel(new NullConfigModelRegistry(), deployStateBuilder.applicationPackage(applicationPackage).build(true)); + return new VespaModel(new NullConfigModelRegistry(), deployStateBuilder.applicationPackage(applicationPackage).build()); } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/model/TestModelFactory.java b/configserver/src/test/java/com/yahoo/vespa/config/server/model/TestModelFactory.java index 48caba2baef..d736611cdcd 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/model/TestModelFactory.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/model/TestModelFactory.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.config.server.model; import com.yahoo.config.model.NullConfigModelRegistry; import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.api.ModelCreateResult; +import com.yahoo.config.model.api.ValidationParameters; import com.yahoo.config.provision.Version; import com.yahoo.vespa.model.VespaModelFactory; @@ -21,9 +22,9 @@ public class TestModelFactory extends VespaModelFactory { // Needed for testing (to get hold of ModelContext) @Override - public ModelCreateResult createAndValidateModel(ModelContext modelContext, boolean ignoreValidationErrors) { + public ModelCreateResult createAndValidateModel(ModelContext modelContext, ValidationParameters validationParameters) { this.modelContext = modelContext; - return super.createAndValidateModel(modelContext, ignoreValidationErrors); + return super.createAndValidateModel(modelContext, validationParameters); } @Override diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/provision/StaticProvisionerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/provision/StaticProvisionerTest.java index 07ba1925bf6..badcdf53b77 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/provision/StaticProvisionerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/provision/StaticProvisionerTest.java @@ -54,7 +54,7 @@ public class StaticProvisionerTest { .multitenant(true) .hostedVespa(true) .build()) - .build(true); + .build(); return new VespaModel(new NullConfigModelRegistry(), deployState); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/RemoteSessionTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/RemoteSessionTest.java index b57d2d1a1a1..459604fa333 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/RemoteSessionTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/RemoteSessionTest.java @@ -277,7 +277,7 @@ public class RemoteSessionTest { public Model loadModel() { try { ApplicationPackage application = new MockApplicationPackage.Builder().withEmptyHosts().withEmptyServices().withValidationOverrides(validationOverrides).build(); - DeployState deployState = new DeployState.Builder().applicationPackage(application).now(clock.instant()).build(true); + DeployState deployState = new DeployState.Builder().applicationPackage(application).now(clock.instant()).build(); return new VespaModel(deployState); } catch (Exception e) { throw new RuntimeException(e); @@ -285,7 +285,7 @@ public class RemoteSessionTest { } @Override - public ModelCreateResult createAndValidateModel(ModelContext modelContext, boolean ignoreValidationErrors) { + public ModelCreateResult createAndValidateModel(ModelContext modelContext, ValidationParameters validationParameters) { if (throwOnLoad) { throw new IllegalArgumentException("Foo"); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java index 6f26323b558..a221a496e0c 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java @@ -7,6 +7,7 @@ import com.yahoo.config.model.api.ConfigChangeAction; import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.api.ModelCreateResult; import com.yahoo.config.model.api.ServiceInfo; +import com.yahoo.config.model.api.ValidationParameters; import com.yahoo.config.model.application.provider.*; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.InstanceName; @@ -267,7 +268,7 @@ public class SessionPreparerTest { } @Override - public ModelCreateResult createAndValidateModel(ModelContext modelContext, boolean ignoreValidationErrors) { + public ModelCreateResult createAndValidateModel(ModelContext modelContext, ValidationParameters validationParameters) { throw exception; } } @@ -285,8 +286,8 @@ public class SessionPreparerTest { } @Override - public ModelCreateResult createAndValidateModel(ModelContext modelContext, boolean ignoreValidationErrors) { - ModelCreateResult result = super.createAndValidateModel(modelContext, ignoreValidationErrors); + public ModelCreateResult createAndValidateModel(ModelContext modelContext, ValidationParameters validationParameters) { + ModelCreateResult result = super.createAndValidateModel(modelContext, validationParameters); return new ModelCreateResult(result.getModel(), Arrays.asList(action)); } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java index 06908dbab51..22d5901a29a 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java @@ -55,7 +55,7 @@ public class ZKApplicationPackageTest { assertTrue(Pattern.compile(".*<slobroks>.*",Pattern.MULTILINE+Pattern.DOTALL).matcher(IOUtils.readAll(zkApp.getServices())).matches()); assertTrue(Pattern.compile(".*<alias>.*",Pattern.MULTILINE+Pattern.DOTALL).matcher(IOUtils.readAll(zkApp.getHosts())).matches()); assertTrue(Pattern.compile(".*<slobroks>.*",Pattern.MULTILINE+Pattern.DOTALL).matcher(IOUtils.readAll(zkApp.getFile(Path.fromString("services.xml")).createReader())).matches()); - DeployState deployState = new DeployState.Builder().applicationPackage(zkApp).build(true); + DeployState deployState = new DeployState.Builder().applicationPackage(zkApp).build(); assertEquals(deployState.getSearchDefinitions().size(), 5); assertEquals(zkApp.searchDefinitionContents().size(), 5); assertEquals(IOUtils.readAll(zkApp.getRankingExpression("foo.expression")), "foo()+1\n"); diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/RequestHandlerTestDriver.java b/container-core/src/main/java/com/yahoo/container/jdisc/RequestHandlerTestDriver.java index 132b1153fc5..22933556d9f 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/RequestHandlerTestDriver.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/RequestHandlerTestDriver.java @@ -63,10 +63,16 @@ public class RequestHandlerTestDriver implements AutoCloseable { return sendRequest(uri, method, ""); } + /** Send a POST request */ public MockResponseHandler sendRequest(String uri, HttpRequest.Method method, String body) { return sendRequest(uri, method, ByteBuffer.wrap(body.getBytes(StandardCharsets.UTF_8))); } + /** Send a POST request with defined content type */ + public MockResponseHandler sendRequest(String uri, HttpRequest.Method method, String body, String contentType) { + return sendRequest(uri, method, ByteBuffer.wrap(body.getBytes(StandardCharsets.UTF_8)), contentType); + } + public MockResponseHandler sendRequest(String uri, HttpRequest.Method method, ByteBuffer body) { responseHandler = new MockResponseHandler(); Request request = HttpRequest.newServerRequest(driver, URI.create(uri), method); @@ -78,6 +84,18 @@ public class RequestHandlerTestDriver implements AutoCloseable { return responseHandler; } + public MockResponseHandler sendRequest(String uri, HttpRequest.Method method, ByteBuffer body, String contentType) { + responseHandler = new MockResponseHandler(); + Request request = HttpRequest.newServerRequest(driver, URI.create(uri), method); + request.context().put("contextVariable", 37); // TODO: Add a method for accepting a Request instead + request.headers().put(com.yahoo.jdisc.http.HttpHeaders.Names.CONTENT_TYPE, contentType); + ContentChannel requestContent = request.connect(responseHandler); + requestContent.write(body, null); + requestContent.close(null); + request.release(); + return responseHandler; + } + /** Replaces all occurrences of 0-9 digits by d's */ public String censorDigits(String s) { return s.replaceAll("[0-9]","d"); diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java index 932d31c0036..3aed7108f2b 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java @@ -284,13 +284,13 @@ public final class ConfiguredApplication implements Application { startShutdownDeadlineExecutor(); shutdownReconfigurerThread(); - configurer.shutdown(new Deconstructor(false)); - for (ServerProvider server : Container.get().getServerProviderRegistry().allComponents()) { if (startedServers.contains(server)) { closeServer(server); } } + + configurer.shutdown(new Deconstructor(false)); Container.get().shutdown(); } diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/component/Deconstructor.java b/container-disc/src/main/java/com/yahoo/container/jdisc/component/Deconstructor.java index d39357fab46..b885a627d38 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/component/Deconstructor.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/component/Deconstructor.java @@ -45,7 +45,7 @@ public class Deconstructor implements ComponentDeconstructor { } else if (component instanceof Provider) { // TODO Providers should most likely be deconstructed similarily to AbstractComponent log.info("Starting deconstruction of provider " + component); - ((Provider)component).deconstruct(); + ((Provider<?>)component).deconstruct(); log.info("Finished deconstruction of provider " + component); } else if (component instanceof SharedResource) { log.info("Releasing container reference to resource " + component); diff --git a/container-disc/src/test/java/com/yahoo/container/jdisc/metric/MetricConsumerProviderTest.java b/container-disc/src/test/java/com/yahoo/container/jdisc/metric/MetricConsumerProviderTest.java index 8ee51d403ae..f5f89eafbf1 100644 --- a/container-disc/src/test/java/com/yahoo/container/jdisc/metric/MetricConsumerProviderTest.java +++ b/container-disc/src/test/java/com/yahoo/container/jdisc/metric/MetricConsumerProviderTest.java @@ -2,14 +2,12 @@ package com.yahoo.container.jdisc.metric; import com.yahoo.jdisc.application.MetricConsumer; -import org.junit.Ignore; import org.junit.Test; import org.mockito.Mockito; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotSame; import static org.junit.Assert.assertSame; -import static org.junit.Assume.assumeTrue; /** * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> diff --git a/container-search/src/main/java/com/yahoo/search/Query.java b/container-search/src/main/java/com/yahoo/search/Query.java index ab6976e29d9..dd3cc853742 100644 --- a/container-search/src/main/java/com/yahoo/search/Query.java +++ b/container-search/src/main/java/com/yahoo/search/Query.java @@ -46,7 +46,6 @@ import com.yahoo.search.yql.VespaSerializer; import com.yahoo.search.yql.YqlParser; import com.yahoo.yolean.Exceptions; import edu.umd.cs.findbugs.annotations.Nullable; - import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collections; @@ -270,6 +269,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { this(""); } + /** * Construct a query from a string formatted in the http style, e.g <code>?query=test&offset=10&hits=13</code> * The query must be uri encoded. @@ -278,6 +278,16 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { this(query, null); } + + /** + * Creates a query from a request + * + * @param request the HTTP request from which this is created + */ + public Query(HttpRequest request) { + this(request, null); + } + /** * Construct a query from a string formatted in the http style, e.g <code>?query=test&offset=10&hits=13</code> * The query must be uri encoded. @@ -293,20 +303,24 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { * @param queryProfile the query profile to use for this query, or null if none. */ public Query(HttpRequest request, CompiledQueryProfile queryProfile) { - super(new QueryPropertyAliases(propertyAliases)); - this.httpRequest = request; - init(request.propertyMap(), queryProfile); + this(request, request.propertyMap(), queryProfile); } /** * Creates a query from a request * - * @param request the HTTP request from which this is created + * @param request the HTTP request from which this is created. + * @param requestMap the property map of the query. + * @param queryProfile the query profile to use for this query, or null if none. */ - public Query(HttpRequest request) { - this(request, null); + public Query(HttpRequest request, Map<String, String> requestMap, CompiledQueryProfile queryProfile) { + super(new QueryPropertyAliases(propertyAliases)); + this.httpRequest = request; + init(requestMap, queryProfile); } + + private void init(Map<String, String> requestMap, CompiledQueryProfile queryProfile) { startTime = System.currentTimeMillis(); if (queryProfile != null) { diff --git a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java index e9e4e34727c..7406d492bc8 100644 --- a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java +++ b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java @@ -20,8 +20,8 @@ import com.yahoo.container.jdisc.LoggingRequestHandler; import com.yahoo.container.jdisc.VespaHeaders; import com.yahoo.container.logging.AccessLog; import com.yahoo.container.protect.FreezeDetector; +import com.yahoo.io.IOUtils; import com.yahoo.jdisc.Metric; -import com.yahoo.jdisc.Response; import com.yahoo.language.Linguistics; import com.yahoo.log.LogLevel; import com.yahoo.net.HostName; @@ -33,6 +33,9 @@ import com.yahoo.prelude.query.parser.ParseException; import com.yahoo.prelude.query.parser.SpecialTokenRegistry; import com.yahoo.processing.rendering.Renderer; import com.yahoo.processing.request.CompoundName; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.ObjectTraverser; +import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.yolean.Exceptions; import com.yahoo.search.Query; import com.yahoo.search.Result; @@ -55,9 +58,10 @@ import com.yahoo.statistics.Statistics; import com.yahoo.statistics.Value; import com.yahoo.vespa.configdefinition.SpecialtokensConfig; import edu.umd.cs.findbugs.annotations.NonNull; - -import java.util.ArrayList; -import java.util.Iterator; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Map; import java.util.Optional; import java.util.concurrent.Executor; import java.util.concurrent.ThreadPoolExecutor; @@ -82,6 +86,8 @@ public class SearchHandler extends LoggingRequestHandler { /** Event name for number of connections to the search subsystem */ private static final String SEARCH_CONNECTIONS = "search_connections"; + private static final String JSON_CONTENT_TYPE = "application/json"; + private static Logger log = Logger.getLogger(SearchHandler.class.getName()); private Value searchConnections; @@ -275,14 +281,14 @@ public class SearchHandler extends LoggingRequestHandler { return errorResponse(request, ErrorMessage.createInternalServerError(Exceptions.toMessageString(e))); } - private HttpSearchResponse handleBody(HttpRequest request) { + + private HttpSearchResponse handleBody(HttpRequest request){ // Find query profile String queryProfileName = request.getProperty("queryProfile"); CompiledQueryProfile queryProfile = queryProfileRegistry.findQueryProfile(queryProfileName); boolean benchmarkOutput = VespaHeaders.benchmarkOutput(request); - // Create query - Query query = new Query(request, queryProfile); + Query query = queryFromRequest(request, queryProfile); boolean benchmarkCoverage = VespaHeaders.benchmarkCoverage(benchmarkOutput, request.getJDiscRequest().headers()); @@ -552,4 +558,62 @@ public class SearchHandler extends LoggingRequestHandler { return searchChainRegistry; } + + private Query queryFromRequest(HttpRequest request, CompiledQueryProfile queryProfile){ + if (request.getMethod() == com.yahoo.jdisc.http.HttpRequest.Method.POST && request.getHeader(com.yahoo.jdisc.http.HttpHeaders.Names.CONTENT_TYPE).equals(JSON_CONTENT_TYPE)) { + Inspector inspector; + try { + byte[] byteArray = IOUtils.readBytes(request.getData(), 1 << 20); + inspector = SlimeUtils.jsonToSlime(byteArray).get(); + if (inspector.field("error_message").valid()){ + throw new QueryException("Illegal query: "+inspector.field("error_message").asString() + ", at: "+ new String(inspector.field("offending_input").asData(), StandardCharsets.UTF_8)); + } + + } catch (IOException e) { + throw new RuntimeException("Problem with reading from input-stream", e); + } + + // Create request-mapping + Map<String, String> requestMap = new HashMap<>(); + createRequestMapping(inspector, requestMap, ""); + return new Query(request, requestMap, queryProfile); + + + } else { + return new Query(request, queryProfile); + + } + } + + public void createRequestMapping(Inspector inspector, Map<String, String> map, String parent){ + inspector.traverse((ObjectTraverser) (key, value) -> { + String qualifiedKey = parent + key; + switch (value.type()) { + case BOOL: + map.put(qualifiedKey, Boolean.toString(value.asBool())); + break; + case DOUBLE: + map.put(qualifiedKey, Double.toString(value.asDouble())); + break; + case LONG: + map.put(qualifiedKey, Long.toString(value.asLong())); + break; + case STRING: + map.put(qualifiedKey , value.asString()); + break; + case ARRAY: + map.put(qualifiedKey, value.asString()); + break; + case OBJECT: + createRequestMapping(value, map, qualifiedKey+"."); + break; + } + + }); + } + + + } + + diff --git a/container-search/src/test/java/com/yahoo/search/handler/test/JSONSearchHandlerTestCase.java b/container-search/src/test/java/com/yahoo/search/handler/test/JSONSearchHandlerTestCase.java new file mode 100644 index 00000000000..eea58d5444e --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/handler/test/JSONSearchHandlerTestCase.java @@ -0,0 +1,466 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.handler.test; + +import com.yahoo.container.Container; +import com.yahoo.container.core.config.testutil.HandlersConfigurerTestWrapper; +import com.yahoo.container.jdisc.HttpRequest; + +import com.yahoo.container.jdisc.RequestHandlerTestDriver; +import com.yahoo.container.protect.Error; +import com.yahoo.io.IOUtils; +import com.yahoo.net.HostName; +import com.yahoo.search.handler.SearchHandler; +import com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase; +import com.yahoo.slime.Inspector; +import com.yahoo.vespa.config.SlimeUtils; +import org.json.JSONObject; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import java.io.File; +import java.io.IOException; +import java.util.Map; +import java.util.HashMap; + +import static com.yahoo.jdisc.http.HttpRequest.Method.GET; +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.*; + + +public class JSONSearchHandlerTestCase { + + private static final String testDir = "src/test/java/com/yahoo/search/handler/test/config"; + private static final String myHostnameHeader = "my-hostname-header"; + private static final String selfHostname = HostName.getLocalhost(); + + private static String tempDir = ""; + private static String configId = null; + private static final String uri = "http://localhost?"; + private static final String JSON_CONTENT_TYPE = "application/json"; + + @Rule + public TemporaryFolder tempfolder = new TemporaryFolder(); + + private RequestHandlerTestDriver driver = null; + private HandlersConfigurerTestWrapper configurer = null; + private SearchHandler searchHandler; + + @Before + public void startUp() throws IOException { + File cfgDir = tempfolder.newFolder("SearchHandlerTestCase"); + tempDir = cfgDir.getAbsolutePath(); + configId = "dir:" + tempDir; + + IOUtils.copyDirectory(new File(testDir), cfgDir, 1); // make configs active + generateComponentsConfigForActive(); + + configurer = new HandlersConfigurerTestWrapper(new Container(), configId); + searchHandler = (SearchHandler)configurer.getRequestHandlerRegistry().getComponent(SearchHandler.class.getName()); + driver = new RequestHandlerTestDriver(searchHandler); + } + + @After + public void shutDown() { + if (configurer != null) configurer.shutdown(); + if (driver != null) driver.close(); + } + + private void generateComponentsConfigForActive() throws IOException { + File activeConfig = new File(tempDir); + SearchChainConfigurerTestCase. + createComponentsConfig(new File(activeConfig, "chains.cfg").getPath(), + new File(activeConfig, "handlers.cfg").getPath(), + new File(activeConfig, "components.cfg").getPath()); + } + + private SearchHandler fetchSearchHandler(HandlersConfigurerTestWrapper configurer) { + return (SearchHandler) configurer.getRequestHandlerRegistry().getComponent(SearchHandler.class.getName()); + } + + @Test + public void testBadJSON() throws Exception{ + String json = "Not a valid JSON-string"; + RequestHandlerTestDriver.MockResponseHandler responseHandler = driver.sendRequest(uri, com.yahoo.jdisc.http.HttpRequest.Method.POST, json, JSON_CONTENT_TYPE); + String response = responseHandler.readAll(); + assertThat(responseHandler.getStatus(), is(400)); + assertThat(response, containsString("errors")); + assertThat(response, containsString("\"code\":" + Error.ILLEGAL_QUERY.code)); + } + + @Test + public void testFailing() throws Exception { + JSONObject json = new JSONObject(); + json.put("query", "test"); + json.put("searchChain", "classLoadingError"); + assertTrue(driver.sendRequest(uri, com.yahoo.jdisc.http.HttpRequest.Method.POST, json.toString(), JSON_CONTENT_TYPE).readAll().contains("NoClassDefFoundError")); + } + + + @Test + public synchronized void testPluginError() throws Exception { + JSONObject json = new JSONObject(); + json.put("query", "test"); + json.put("searchChain", "exceptionInPlugin"); + assertTrue(driver.sendRequest(uri, com.yahoo.jdisc.http.HttpRequest.Method.POST, json.toString(), JSON_CONTENT_TYPE).readAll().contains("NullPointerException")); + } + + @Test + public synchronized void testWorkingReconfiguration() throws Exception { + JSONObject json = new JSONObject(); + json.put("query", "abc"); + assertJsonResult(json, driver); + + // reconfiguration + IOUtils.copyDirectory(new File(testDir, "handlers2"), new File(tempDir), 1); + generateComponentsConfigForActive(); + configurer.reloadConfig(); + + // ...and check the resulting config + SearchHandler newSearchHandler = fetchSearchHandler(configurer); + assertNotSame("Have a new instance of the search handler", searchHandler, newSearchHandler); + assertNotNull("Have the new search chain", fetchSearchHandler(configurer).getSearchChainRegistry().getChain("hello")); + assertNull("Don't have the new search chain", fetchSearchHandler(configurer).getSearchChainRegistry().getChain("classLoadingError")); + try (RequestHandlerTestDriver newDriver = new RequestHandlerTestDriver(searchHandler)) { + assertJsonResult(json, newDriver); + } + } + + @Test + public void testInvalidYqlQuery() throws Exception { + IOUtils.copyDirectory(new File(testDir, "config_yql"), new File(tempDir), 1); + generateComponentsConfigForActive(); + configurer.reloadConfig(); + + SearchHandler newSearchHandler = fetchSearchHandler(configurer); + assertTrue("Do I have a new instance of the search handler?", searchHandler != newSearchHandler); + try (RequestHandlerTestDriver newDriver = new RequestHandlerTestDriver(newSearchHandler)) { + JSONObject json = new JSONObject(); + json.put("yql", "select * from foo where bar > 1453501295"); + RequestHandlerTestDriver.MockResponseHandler responseHandler = newDriver.sendRequest(uri, com.yahoo.jdisc.http.HttpRequest.Method.POST, json.toString(), JSON_CONTENT_TYPE); + responseHandler.readAll(); + assertThat(responseHandler.getStatus(), is(400)); + } + } + + // Query handling takes a different code path when a query profile is active, so we test both paths. + @Test + public void testInvalidQueryParamWithQueryProfile() throws Exception { + try (RequestHandlerTestDriver newDriver = driverWithConfig("config_invalid_param")) { + testInvalidQueryParam(newDriver); + } + } + + private void testInvalidQueryParam(final RequestHandlerTestDriver testDriver) throws Exception{ + JSONObject json = new JSONObject(); + json.put("query", "status_code:0"); + json.put("hits", 20); + json.put("offset", -20); + RequestHandlerTestDriver.MockResponseHandler responseHandler = + testDriver.sendRequest(uri, com.yahoo.jdisc.http.HttpRequest.Method.POST, json.toString(), JSON_CONTENT_TYPE); + String response = responseHandler.readAll(); + assertThat(responseHandler.getStatus(), is(400)); + assertThat(response, containsString("offset")); + assertThat(response, containsString("\"code\":" + com.yahoo.container.protect.Error.INVALID_QUERY_PARAMETER.code)); + } + + + + + @Test + public void testNormalResultJsonAliasRendering() throws Exception { + JSONObject json = new JSONObject(); + json.put("format", "json"); + json.put("query", "abc"); + assertJsonResult(json, driver); + } + + + + @Test + public void testNullQuery() throws Exception { + JSONObject json = new JSONObject(); + json.put("format", "xml"); + + assertEquals("<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n" + + "<result total-hit-count=\"0\">\n" + + " <hit relevancy=\"1.0\">\n" + + " <field name=\"relevancy\">1.0</field>\n" + + " <field name=\"uri\">testHit</field>\n" + + " </hit>\n" + + "</result>\n", driver.sendRequest(uri, com.yahoo.jdisc.http.HttpRequest.Method.POST, json.toString(), JSON_CONTENT_TYPE).readAll()); + } + + + + @Test + public void testWebServiceStatus() throws Exception { + JSONObject json = new JSONObject(); + json.put("query", "web_service_status_code"); + RequestHandlerTestDriver.MockResponseHandler responseHandler = + driver.sendRequest(uri, com.yahoo.jdisc.http.HttpRequest.Method.POST, json.toString(), JSON_CONTENT_TYPE); + String response = responseHandler.readAll(); + assertThat(responseHandler.getStatus(), is(406)); + assertThat(response, containsString("\"code\":" + 406)); + } + + @Test + public void testNormalResultImplicitDefaultRendering() throws Exception { + JSONObject json = new JSONObject(); + json.put("query", "abc"); + assertJsonResult(json, driver); + } + + @Test + public void testNormalResultExplicitDefaultRendering() throws Exception { + JSONObject json = new JSONObject(); + json.put("query", "abc"); + json.put("format", "default"); + assertJsonResult(json, driver); + } + + @Test + public void testNormalResultXmlAliasRendering() throws Exception { + JSONObject json = new JSONObject(); + json.put("query", "abc"); + json.put("format", "xml"); + assertXmlResult(json, driver); + } + + + @Test + public void testNormalResultExplicitDefaultRenderingFullRendererName1() throws Exception { + JSONObject json = new JSONObject(); + json.put("query", "abc"); + json.put("format", "DefaultRenderer"); + assertXmlResult(json, driver); + } + + @Test + public void testNormalResultExplicitDefaultRenderingFullRendererName2() throws Exception { + JSONObject json = new JSONObject(); + json.put("query", "abc"); + json.put("format", "JsonRenderer"); + assertJsonResult(json, driver); + } + + @Test + public void testResultLegacyTiledFormat() throws Exception { + JSONObject json = new JSONObject(); + json.put("query", "abc"); + json.put("format", "tiled"); + assertTiledResult(json, driver); + } + + @Test + public void testResultLegacyPageFormat() throws Exception { + JSONObject json = new JSONObject(); + json.put("query", "abc"); + json.put("format", "page"); + assertPageResult(json, driver); + } + + + private static final String xmlResult = + "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n" + + "<result total-hit-count=\"0\">\n" + + " <hit relevancy=\"1.0\">\n" + + " <field name=\"relevancy\">1.0</field>\n" + + " <field name=\"uri\">testHit</field>\n" + + " </hit>\n" + + "</result>\n"; + + private void assertXmlResult(JSONObject json, RequestHandlerTestDriver driver) throws Exception { + assertOkResult(driver.sendRequest(uri, com.yahoo.jdisc.http.HttpRequest.Method.POST, json.toString(), JSON_CONTENT_TYPE), xmlResult); + } + + + private static final String jsonResult = "{\"root\":{" + + "\"id\":\"toplevel\",\"relevance\":1.0,\"fields\":{\"totalCount\":0}," + + "\"children\":[" + + "{\"id\":\"testHit\",\"relevance\":1.0,\"fields\":{\"uri\":\"testHit\"}}" + + "]}}"; + + private void assertJsonResult(JSONObject json, RequestHandlerTestDriver driver) throws Exception { + assertOkResult(driver.sendRequest(uri, com.yahoo.jdisc.http.HttpRequest.Method.POST, json.toString(), JSON_CONTENT_TYPE), jsonResult); + + } + + private static final String tiledResult = + "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n" + + "<result version=\"1.0\">\n" + + "\n" + + " <hit relevance=\"1.0\">\n" + + " <id>testHit</id>\n" + + " <uri>testHit</uri>\n" + + " </hit>\n" + + "\n" + + "</result>\n"; + + private void assertTiledResult(JSONObject json, RequestHandlerTestDriver driver) throws Exception { + assertOkResult(driver.sendRequest(uri, com.yahoo.jdisc.http.HttpRequest.Method.POST, json.toString(), JSON_CONTENT_TYPE), tiledResult); + } + + private static final String pageResult = + "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n" + + "<page version=\"1.0\">\n" + + "\n" + + " <content>\n" + + " <hit relevance=\"1.0\">\n" + + " <id>testHit</id>\n" + + " <uri>testHit</uri>\n" + + " </hit>\n" + + " </content>\n" + + "\n" + + "</page>\n"; + + private void assertPageResult(JSONObject json, RequestHandlerTestDriver driver) throws Exception { + assertOkResult(driver.sendRequest(uri, com.yahoo.jdisc.http.HttpRequest.Method.POST, json.toString(), JSON_CONTENT_TYPE), pageResult); + } + + private void assertOkResult(RequestHandlerTestDriver.MockResponseHandler response, String expected) { + assertEquals(expected, response.readAll()); + assertEquals(200, response.getStatus()); + assertEquals(selfHostname, response.getResponse().headers().get(myHostnameHeader).get(0)); + } + + + private RequestHandlerTestDriver driverWithConfig(String configDirectory) throws Exception { + IOUtils.copyDirectory(new File(testDir, configDirectory), new File(tempDir), 1); + generateComponentsConfigForActive(); + configurer.reloadConfig(); + + SearchHandler newSearchHandler = fetchSearchHandler(configurer); + assertTrue("Do I have a new instance of the search handler?", searchHandler != newSearchHandler); + return new RequestHandlerTestDriver(newSearchHandler); + } + + + + @Test + public void testRequestMapping() throws Exception { + JSONObject json = new JSONObject(); + json.put("yql", "select * from sources * where sddocname contains \"blog_post\" limit 0 | all(group(date) max(3) order(-count())each(output(count())));"); + json.put("hits", 10.0); + json.put("offset", 5); + json.put("queryProfile", "foo"); + json.put("nocache", false); + json.put("groupingSessionCache", false); + json.put("searchChain", "exceptionInPlugin"); + json.put("timeout", 0); + json.put("select", "_all"); + + + JSONObject model = new JSONObject(); + model.put("defaultIndex", 1); + model.put("encoding", "json"); + model.put("filter", "default"); + model.put("language", "en"); + model.put("queryString", "abc"); + model.put("restrict", "_doc,json,xml"); + model.put("searchPath", "node1"); + model.put("sources", "source1,source2"); + model.put("type", "yql"); + json.put("model", model); + + JSONObject ranking = new JSONObject(); + ranking.put("location", "123789.89123N;128123W"); + ranking.put("features", "none"); + ranking.put("listFeatures", false); + ranking.put("profile", "1"); + ranking.put("properties", "default"); + ranking.put("sorting", "desc"); + ranking.put("freshness", "0.05"); + ranking.put("queryCache", false); + + JSONObject matchPhase = new JSONObject(); + matchPhase.put("maxHits", "100"); + matchPhase.put("attribute", "title"); + matchPhase.put("ascending", true); + + JSONObject diversity = new JSONObject(); + diversity.put("attribute", "title"); + diversity.put("minGroups", 1); + matchPhase.put("diversity", diversity); + ranking.put("matchPhase", matchPhase); + json.put("ranking", ranking); + + JSONObject presentation = new JSONObject(); + presentation.put("bolding", true); + presentation.put("format", "json"); + presentation.put("summary", "none"); + presentation.put("template", "json"); + presentation.put("timing", false); + json.put("presentation", presentation); + + JSONObject collapse = new JSONObject(); + collapse.put("field", "none"); + collapse.put("size", 2); + collapse.put("summary", "default"); + json.put("collapse", collapse); + + JSONObject trace = new JSONObject(); + trace.put("level", 1); + trace.put("timestamps", false); + trace.put("rules", "none"); + json.put("trace", trace); + + JSONObject pos = new JSONObject(); + pos.put("ll", "1263123N;1231.9W"); + pos.put("radius", "71234m"); + pos.put("bb", "1237123W;123218N"); + pos.put("attribute", "default"); + json.put("pos", pos); + + JSONObject streaming = new JSONObject(); + streaming.put("userid", 123); + streaming.put("groupname", "abc"); + streaming.put("selection", "none"); + streaming.put("priority", 10); + streaming.put("maxbucketspervisitor", 5); + json.put("streaming", streaming); + + JSONObject rules = new JSONObject(); + rules.put("off", false); + rules.put("rulebase", "default"); + json.put("rules", rules); + + JSONObject metrics = new JSONObject(); + metrics.put("ignore", "_all"); + json.put("metrics", metrics); + + json.put("recall", "none"); + json.put("user", 123); + json.put("nocachewrite", false); + json.put("hitcountestimate", true); + + + + // Create mapping + Inspector inspector = SlimeUtils.jsonToSlime(json.toString().getBytes("utf-8")).get(); + Map<String, String> map = new HashMap<>(); + searchHandler.createRequestMapping(inspector, map, ""); + + // Create GET-request with same query + String url = uri + "&model.sources=source1%2Csource2&select=_all&model.language=en&presentation.timing=false&pos.attribute=default&pos.radius=71234m&model.searchPath=node1&nocachewrite=false&ranking.matchPhase.maxHits=100&presentation.summary=none" + + "&nocache=false&model.type=yql&collapse.summary=default&ranking.matchPhase.diversity.minGroups=1&ranking.location=123789.89123N%3B128123W&ranking.queryCache=false&offset=5&streaming.groupname=abc&groupingSessionCache=false" + + "&presentation.template=json&trace.rules=none&rules.off=false&ranking.properties=default&searchChain=exceptionInPlugin&pos.ll=1263123N%3B1231.9W&ranking.sorting=desc&ranking.matchPhase.ascending=true&ranking.features=none&hitcountestimate=true" + + "&model.filter=default&metrics.ignore=_all&collapse.field=none&ranking.profile=1&rules.rulebase=default&model.defaultIndex=1&trace.level=1&ranking.listFeatures=false&timeout=0&presentation.format=json" + + "&yql=select+%2A+from+sources+%2A+where+sddocname+contains+%22blog_post%22+limit+0+%7C+all%28group%28date%29+max%283%29+order%28-count%28%29%29each%28output%28count%28%29%29%29%29%3B&recall=none&streaming.maxbucketspervisitor=5" + + "&queryProfile=foo&presentation.bolding=true&model.encoding=json&model.queryString=abc&streaming.selection=none&trace.timestamps=false&collapse.size=2&streaming.priority=10&ranking.matchPhase.diversity.attribute=title" + + "&ranking.matchPhase.attribute=title&hits=10&streaming.userid=123&pos.bb=1237123W%3B123218N&model.restrict=_doc%2Cjson%2Cxml&ranking.freshness=0.05&user=123"; + + + + final HttpRequest request = HttpRequest.createTestRequest(url, GET); + + // Get mapping + Map<String, String> propertyMap = request.propertyMap(); + assertEquals("Should have same mapping for properties", map, propertyMap); + } + + + +} diff --git a/container-search/src/test/java/com/yahoo/search/handler/test/SearchHandlerTestCase.java b/container-search/src/test/java/com/yahoo/search/handler/test/SearchHandlerTestCase.java index ce40bd1f06b..6dcb34ec3e9 100644 --- a/container-search/src/test/java/com/yahoo/search/handler/test/SearchHandlerTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/handler/test/SearchHandlerTestCase.java @@ -3,16 +3,13 @@ package com.yahoo.search.handler.test; import com.yahoo.container.Container; import com.yahoo.container.core.config.testutil.HandlersConfigurerTestWrapper; -import com.yahoo.container.jdisc.AsyncHttpResponse; import com.yahoo.container.jdisc.HttpRequest; - import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.RequestHandlerTestDriver; import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.io.IOUtils; import com.yahoo.jdisc.handler.RequestHandler; import com.yahoo.net.HostName; -import com.yahoo.processing.handler.ResponseStatus; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; @@ -30,12 +27,9 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; -import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; import java.net.URI; -import java.util.List; -import java.util.Map; import java.util.concurrent.Executors; import static org.hamcrest.CoreMatchers.containsString; @@ -187,6 +181,8 @@ public class SearchHandlerTestCase { } } + + // Query handling takes a different code path when a query profile is active, so we test both paths. @Test public void testInvalidQueryParamWithQueryProfile() throws Exception { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/concurrent/TimeoutException.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/concurrent/TimeoutException.java deleted file mode 100644 index 260761fa6ac..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/concurrent/TimeoutException.java +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.concurrent; - -/** - * Throws on timeout - * - * @author bratseth - */ -public class TimeoutException extends RuntimeException { - - public TimeoutException(String message) { - super(message); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainer.java index b889179750e..a046ed87a05 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainer.java @@ -22,7 +22,7 @@ import java.util.Map; */ public class ClusterUtilizationMaintainer extends Maintainer { - Controller controller; + private final Controller controller; public ClusterUtilizationMaintainer(Controller controller, Duration duration, JobControl jobControl) { super(controller, duration, jobControl); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java index 2f1345eb49e..feec83d226e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java @@ -106,9 +106,7 @@ public class MetricsReporter extends Maintainer { node.getValue("tenant").ifPresent(tenant -> dimensions.put("tenantName", tenant)); Optional<String> application = node.getValue("application"); - if (application.isPresent()) { - dimensions.put("app",String.format("%s.%s", application.get(), node.getValue("instance").orElse("default"))); - } + application.ifPresent(app -> dimensions.put("app", String.format("%s.%s", app, node.getValue("instance").orElse("default")))); Metric.Context context = metric.createContext(dimensions); metric.set(convergeMetric, secondsSinceConverge, context); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java index 953e95e25c8..a71ce4299b1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java @@ -32,12 +32,7 @@ public class SystemUpgrader extends Maintainer { @Override protected void maintain() { - Optional<Version> target = targetVersion(); - if (!target.isPresent()) { - return; - } - - deployAll(target.get(), SystemApplication.all()); + targetVersion().ifPresent(target -> deployAll(target, SystemApplication.all())); } /** Deploy a list of system applications until they converge on the given version */ @@ -69,17 +64,18 @@ public class SystemUpgrader extends Maintainer { if (convergedOn(target, application.dependencies(), zone)) { deploy(target, application, zone); } - converged &= convergedOn(target, application, zone) & application.configConvergedIn(zone, controller()); + converged &= convergedOn(target, application, zone); } return converged; } /** Deploy application on given version idempotently */ private void deploy(Version target, SystemApplication application, ZoneId zone) { - if (!wantedVersion(zone, application, target).equals(target)) { - log.info(String.format("Deploying %s version %s in %s", application.id(), target, zone)); - controller().applications().deploy(application, zone, target); + if (wantedVersion(zone, application, target).equals(target)) { + return; } + log.info(String.format("Deploying %s version %s in %s", application.id(), target, zone)); + controller().applications().deploy(application, zone, target); } private boolean convergedOn(Version target, List<SystemApplication> applications, ZoneId zone) { @@ -87,7 +83,7 @@ public class SystemUpgrader extends Maintainer { } private boolean convergedOn(Version target, SystemApplication application, ZoneId zone) { - return currentVersion(zone, application, target).equals(target); + return currentVersion(zone, application, target).equals(target) && application.configConvergedIn(zone, controller()); } private Version wantedVersion(ZoneId zone, SystemApplication application, Version defaultVersion) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java index 22cbe942932..6286f89cc9d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java @@ -117,15 +117,15 @@ public class Upgrader extends Maintainer { /** Returns the number of applications to upgrade in this run */ private int numberOfApplicationsToUpgrade() { - return Math.max(1, (int)(maintenanceInterval().getSeconds() * (upgradesPerMinute() / 60))); + return Math.max(1, (int) (maintenanceInterval().getSeconds() * (upgradesPerMinute() / 60))); } - /** Returns number upgrades per minute */ + /** Returns number of upgrades per minute */ public double upgradesPerMinute() { return curator.readUpgradesPerMinute(); } - /** Sets the number upgrades per minute */ + /** Sets the number of upgrades per minute */ public void setUpgradesPerMinute(double n) { curator.writeUpgradesPerMinute(n); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java index e5e03cf5dfc..ae1102e2cef 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java @@ -145,7 +145,7 @@ public class ConfigServerRestExecutorImpl implements ConfigServerRestExecutor { .setSocketTimeout((int) PROXY_REQUEST_TIMEOUT.toMillis()).build(); try ( CloseableHttpClient client = createHttpClient(config, sslContextProvider, zoneRegistry, proxyRequest); - CloseableHttpResponse response = client.execute(requestBase); + CloseableHttpResponse response = client.execute(requestBase) ) { String content = getContent(response); int status = response.getStatusLine().getStatusCode(); @@ -246,7 +246,7 @@ public class ConfigServerRestExecutorImpl implements ConfigServerRestExecutor { .setSocketTimeout(timeout).build(); try ( CloseableHttpClient client = createHttpClient(config, sslContextProvider, zoneRegistry, proxyRequest); - CloseableHttpResponse response = client.execute(httpget); + CloseableHttpResponse response = client.execute(httpget) ) { if (response.getStatusLine().getStatusCode() == 200) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/package-info.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/package-info.java index f6c300268a2..c4472fb79e4 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/package-info.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/package-info.java @@ -1,8 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage -package com.yahoo.vespa.hosted.controller.proxy; - /** * @author Haakon Dybdahl */ +@ExportPackage +package com.yahoo.vespa.hosted.controller.proxy; + import com.yahoo.osgi.annotation.ExportPackage; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 4b09e78ede2..2652b49f86f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -345,13 +345,15 @@ public class ApplicationApiHandler extends LoggingRequestHandler { private void toSlime(Cursor object, Application application, HttpRequest request) { object.setString("application", application.id().application().value()); object.setString("instance", application.id().instance().value()); + // Currently deploying change if (application.change().isPresent()) { - Cursor deployingObject = object.setObject("deploying"); - application.change().platform().ifPresent(v -> deployingObject.setString("version", v.toString())); - application.change().application() - .filter(v -> v != ApplicationVersion.unknown) - .ifPresent(v -> toSlime(v, deployingObject.setObject("revision"))); + toSlime(object.setObject("deploying"), application.change()); + } + + // Outstanding change + if (application.outstandingChange().isPresent()) { + toSlime(object.setObject("outstandingChange"), application.outstandingChange()); } // Jobs sorted according to deployment spec @@ -454,6 +456,13 @@ public class ApplicationApiHandler extends LoggingRequestHandler { return new SlimeJsonResponse(slime); } + private void toSlime(Cursor object, Change change) { + change.platform().ifPresent(version -> object.setString("version", version.toString())); + change.application() + .filter(version -> !version.isUnknown()) + .ifPresent(version -> toSlime(version, object.setObject("revision"))); + } + private void toSlime(Cursor response, DeploymentId deploymentId, Deployment deployment, HttpRequest request) { Cursor serviceUrlArray = response.setArray("serviceUrls"); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/EmptyJsonResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/EmptyJsonResponse.java index 3e8d4182c42..be3222cc1a8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/EmptyJsonResponse.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/EmptyJsonResponse.java @@ -2,9 +2,7 @@ package com.yahoo.vespa.hosted.controller.restapi.application; import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.slime.Slime; -import java.io.IOException; import java.io.OutputStream; /** @@ -17,7 +15,7 @@ public class EmptyJsonResponse extends HttpResponse { } @Override - public void render(OutputStream stream) throws IOException { } + public void render(OutputStream stream) {} @Override public String getContentType() { return "application/json"; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/BlockingRequestFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/BlockingRequestFilter.java index 5d0a66a040e..3bd504b37d8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/BlockingRequestFilter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/BlockingRequestFilter.java @@ -11,6 +11,7 @@ import com.yahoo.jdisc.http.filter.SecurityRequestFilter; /** * @author bjorncs */ +@SuppressWarnings("unused") // Injected public class BlockingRequestFilter implements SecurityRequestFilter { @Override diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/statuspage/StatusPageClient.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/statuspage/StatusPageClient.java index ecb2f611171..93213172048 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/statuspage/StatusPageClient.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/statuspage/StatusPageClient.java @@ -1,6 +1,8 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.statuspage; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.config.SlimeUtils; import org.apache.http.HttpStatus; import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.CloseableHttpResponse; @@ -36,8 +38,8 @@ public class StatusPageClient { this.key = Objects.requireNonNull(key, "key cannot be null"); } - /** GET given page and return response body */ - public byte[] get(String page, Optional<String> since) { + /** GET given page and return response body as slime */ + public Slime get(String page, Optional<String> since) { HttpGet get = new HttpGet(pageUrl(page, since)); try (CloseableHttpClient client = client()) { try (CloseableHttpResponse response = client.execute(get)) { @@ -45,7 +47,8 @@ public class StatusPageClient { throw new IllegalArgumentException("Received status " + response.getStatusLine().getStatusCode() + " from StatusPage"); } - return EntityUtils.toByteArray(response.getEntity()); + byte[] body = EntityUtils.toByteArray(response.getEntity()); + return SlimeUtils.jsonToSlime(body); } } catch (IOException e) { throw new UncheckedIOException(e); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/statuspage/StatusPageProxyHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/statuspage/StatusPageProxyHandler.java index e652e293c26..ad69681fe7e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/statuspage/StatusPageProxyHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/statuspage/StatusPageProxyHandler.java @@ -6,13 +6,13 @@ import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.LoggingRequestHandler; import com.yahoo.container.jdisc.secretstore.SecretStore; +import com.yahoo.slime.Slime; import com.yahoo.vespa.hosted.controller.restapi.ErrorResponse; import com.yahoo.vespa.hosted.controller.restapi.Path; +import com.yahoo.vespa.hosted.controller.restapi.SlimeJsonResponse; import com.yahoo.vespa.hosted.controller.statuspage.config.StatuspageConfig; import com.yahoo.yolean.Exceptions; -import java.io.IOException; -import java.io.OutputStream; import java.net.URI; import java.util.Optional; import java.util.logging.Level; @@ -60,28 +60,8 @@ public class StatusPageProxyHandler extends LoggingRequestHandler { } StatusPageClient client = StatusPageClient.create(apiUrl, secretStore.getSecret(secretKey)); Optional<String> since = Optional.ofNullable(request.getProperty("since")); - byte[] response = client.get(path.get("page"), since); - return new ByteArrayHttpResponse(response); - } - - private static class ByteArrayHttpResponse extends HttpResponse { - - private final byte[] data; - - public ByteArrayHttpResponse(byte[] data) { - super(200); - this.data = data; - } - - @Override - public void render(OutputStream out) throws IOException { - out.write(data); - } - - @Override - public String getContentType() { - return "application/json"; - } + Slime statusPageResponse = client.get(path.get("page"), since); + return new SlimeJsonResponse(statusPageResponse); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/TestIdentities.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/TestIdentities.java index a2fec7cfdbf..18d3e92620d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/TestIdentities.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/TestIdentities.java @@ -12,14 +12,14 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.RegionId; */ public class TestIdentities { - public static EnvironmentId environment = new EnvironmentId("dev"); + public static final EnvironmentId environment = new EnvironmentId("dev"); - public static RegionId region = new RegionId("us-east-1"); + public static final RegionId region = new RegionId("us-east-1"); - public static InstanceId instance = new InstanceId("default"); + public static final InstanceId instance = new InstanceId("default"); - public static Property property = new Property("property"); + public static final Property property = new Property("property"); - public static NToken userNToken = new NToken("dummy"); + public static final NToken userNToken = new NToken("dummy"); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/athenz/filter/AthenzPrincipalFilterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/athenz/filter/AthenzPrincipalFilterTest.java index ee9c65013a8..301fc461b6f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/athenz/filter/AthenzPrincipalFilterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/athenz/filter/AthenzPrincipalFilterTest.java @@ -144,7 +144,7 @@ public class AthenzPrincipalFilterTest { } private static void assertUnauthorized(ResponseHandlerMock responseHandler, String expectedMessageSubstring) { - assertNotNull(responseHandler.response);; + assertNotNull(responseHandler.response); assertEquals(UNAUTHORIZED, responseHandler.response.getStatus()); assertTrue(responseHandler.getResponseContent().contains(expectedMessageSubstring)); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/athenz/filter/NTokenValidatorTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/athenz/filter/NTokenValidatorTest.java index a70c1572c21..510c806383c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/athenz/filter/NTokenValidatorTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/athenz/filter/NTokenValidatorTest.java @@ -13,7 +13,6 @@ import org.junit.Test; import org.junit.rules.ExpectedException; import java.security.KeyPair; -import java.security.NoSuchAlgorithmException; import java.security.PrivateKey; import java.time.Instant; import java.util.Optional; @@ -34,7 +33,7 @@ public class NTokenValidatorTest { public ExpectedException exceptionRule = ExpectedException.none(); @Test - public void valid_token_is_accepted() throws NoSuchAlgorithmException, InvalidTokenException { + public void valid_token_is_accepted() throws InvalidTokenException { NTokenValidator validator = new NTokenValidator(createKeystore()); NToken token = createNToken(IDENTITY, Instant.now(), TRUSTED_KEY.getPrivate(), "0"); AthenzPrincipal principal = validator.validate(token); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java index a81c4adcb2e..2f9703b91e1 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java @@ -26,12 +26,13 @@ import java.util.zip.ZipOutputStream; */ public class ApplicationPackageBuilder { - private String upgradePolicy = null; - private Environment environment = Environment.prod; - private String globalServiceId = null; private final StringBuilder environmentBody = new StringBuilder(); private final StringBuilder validationOverridesBody = new StringBuilder(); private final StringBuilder blockChange = new StringBuilder(); + + private String upgradePolicy = null; + private Environment environment = Environment.prod; + private String globalServiceId = null; private String athenzIdentityAttributes = null; private String searchDefinition = "search test { }"; @@ -145,8 +146,7 @@ public class ApplicationPackageBuilder { public ApplicationPackage build() { ByteArrayOutputStream zip = new ByteArrayOutputStream(); - ZipOutputStream out = new ZipOutputStream(zip); - try { + try (ZipOutputStream out = new ZipOutputStream(zip)) { out.putNextEntry(new ZipEntry("deployment.xml")); out.write(deploymentSpec()); out.closeEntry(); @@ -158,10 +158,6 @@ public class ApplicationPackageBuilder { out.closeEntry(); } catch (IOException e) { throw new UncheckedIOException(e); - } finally { - try { - out.close(); - } catch (IOException ignored) {} } return new ApplicationPackage(zip.toByteArray()); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java index 8039f1d5cce..354f736202a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java @@ -46,8 +46,8 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer private final Map<URI, Version> versions = new HashMap<>(); private final NodeRepositoryMock nodeRepository = new NodeRepositoryMock(); private final Map<DeploymentId, ServiceConvergence> serviceStatus = new HashMap<>(); + private final Version initialVersion = new Version(6, 1, 0); - private Version initialVersion = new Version(6, 1, 0); private Version lastPrepareVersion = null; private RuntimeException prepareException = null; @@ -95,14 +95,6 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer this.prepareException = prepareException; } - /** - * Returns the (initially empty) mutable map of config server urls to versions. - * This API will return defaultVersion as response to any version(url) call for versions not added to the map. - */ - public Map<URI, Version> versions() { - return versions; - } - /** Set version for system applications in given zone */ public void setVersion(ApplicationId application, ZoneId zone, Version version) { for (Node node : nodeRepository().list(zone, application)) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryClientMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryClientMock.java index 330127b7670..1b12b441272 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryClientMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryClientMock.java @@ -9,7 +9,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepo import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeState; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; -import java.io.IOException; import java.util.Arrays; import java.util.Collection; @@ -17,28 +16,29 @@ import java.util.Collection; * @author bjorncs */ public class NodeRepositoryClientMock implements NodeRepositoryClientInterface { + @Override - public void addNodes(ZoneId zone, Collection<NodeRepositoryNode> nodes) throws IOException { + public void addNodes(ZoneId zone, Collection<NodeRepositoryNode> nodes) { throw new UnsupportedOperationException(); } @Override - public NodeRepositoryNode getNode(ZoneId zone, String hostname) throws IOException { + public NodeRepositoryNode getNode(ZoneId zone, String hostname) { throw new UnsupportedOperationException(); } @Override - public void deleteNode(ZoneId zone, String hostname) throws IOException { + public void deleteNode(ZoneId zone, String hostname) { throw new UnsupportedOperationException(); } @Override - public NodeList listNodes(ZoneId zone, boolean recursive) throws IOException { + public NodeList listNodes(ZoneId zone, boolean recursive) { throw new UnsupportedOperationException(); } @Override - public NodeList listNodes(ZoneId zone, String tenant, String applicationId, String instance) throws IOException { + public NodeList listNodes(ZoneId zone, String tenant, String applicationId, String instance) { NodeRepositoryNode nodeA = createNodeA(); NodeRepositoryNode nodeB = createNodeB(); return new NodeList(Arrays.asList(nodeA, nodeB)); @@ -69,57 +69,58 @@ public class NodeRepositoryClientMock implements NodeRepositoryClientInterface { } @Override - public String resetFailureInformation(ZoneId zone, String nodename) throws IOException { + public String resetFailureInformation(ZoneId zone, String nodename) { throw new UnsupportedOperationException(); } @Override - public String restart(ZoneId zone, String nodename) throws IOException { + public String restart(ZoneId zone, String nodename) { throw new UnsupportedOperationException(); } @Override - public String reboot(ZoneId zone, String nodename) throws IOException { + public String reboot(ZoneId zone, String nodename) { throw new UnsupportedOperationException(); } @Override - public String cancelReboot(ZoneId zone, String nodename) throws IOException { + public String cancelReboot(ZoneId zone, String nodename) { throw new UnsupportedOperationException(); } @Override - public String wantTo(ZoneId zone, String nodename, WantTo... actions) throws IOException { + public String wantTo(ZoneId zone, String nodename, WantTo... actions) { throw new UnsupportedOperationException(); } @Override - public String cancelRestart(ZoneId zone, String nodename) throws IOException { + public String cancelRestart(ZoneId zone, String nodename) { throw new UnsupportedOperationException(); } @Override - public String setHardwareFailureDescription(ZoneId zone, String nodename, String hardwareFailureDescription) throws IOException { + public String setHardwareFailureDescription(ZoneId zone, String nodename, String hardwareFailureDescription) { throw new UnsupportedOperationException(); } @Override - public void setState(ZoneId zone, NodeState nodeState, String nodename) throws IOException { + public void setState(ZoneId zone, NodeState nodeState, String nodename) { throw new UnsupportedOperationException(); } @Override - public String enableMaintenanceJob(ZoneId zone, String jobName) throws IOException { + public String enableMaintenanceJob(ZoneId zone, String jobName) { throw new UnsupportedOperationException(); } @Override - public String disableMaintenanceJob(ZoneId zone, String jobName) throws IOException { + public String disableMaintenanceJob(ZoneId zone, String jobName) { throw new UnsupportedOperationException(); } @Override - public MaintenanceJobList listMaintenanceJobs(ZoneId zone) throws IOException { + public MaintenanceJobList listMaintenanceJobs(ZoneId zone) { throw new UnsupportedOperationException(); } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporterTest.java index a1aef28bd63..2b76d386fdd 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporterTest.java @@ -168,8 +168,8 @@ public class DeploymentIssueReporterTest { class MockDeploymentIssues extends LoggingDeploymentIssues { - Map<ApplicationId, IssueId> applicationIssues = new HashMap<>(); - Map<IssueId, Integer> issueLevels = new HashMap<>(); + private final Map<ApplicationId, IssueId> applicationIssues = new HashMap<>(); + private final Map<IssueId, Integer> issueLevels = new HashMap<>(); MockDeploymentIssues() { super(tester.clock()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java index 03e324cd406..fa6edd939c4 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java @@ -52,7 +52,7 @@ public class MetricsReporterTest { @Test public void test_chef_metrics() { - Clock clock = Clock.fixed(Instant.ofEpochSecond(1475497913), ZoneId.systemDefault());; + Clock clock = Clock.fixed(Instant.ofEpochSecond(1475497913), ZoneId.systemDefault()); ControllerTester tester = new ControllerTester(); MetricsReporter metricsReporter = createReporter(clock, tester.controller(), metrics, SystemName.cd); metricsReporter.maintain(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java index a0cca52c192..87e145f9f93 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java @@ -12,6 +12,7 @@ import org.junit.Test; import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.stream.Collectors; @@ -28,7 +29,7 @@ public class VersionStatusSerializerTest { List<VespaVersion> vespaVersions = new ArrayList<>(); DeploymentStatistics statistics = new DeploymentStatistics( Version.fromString("5.0"), - Arrays.asList(ApplicationId.from("tenant1", "failing1", "default")), + Collections.singletonList(ApplicationId.from("tenant1", "failing1", "default")), Arrays.asList(ApplicationId.from("tenant2", "success1", "default"), ApplicationId.from("tenant2", "success2", "default")), Arrays.asList(ApplicationId.from("tenant1", "failing1", "default"), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java index 18d13e0be44..60a05e4f938 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java @@ -83,7 +83,7 @@ public class ContainerControllerTester { return controller().applications().createApplication(app, Optional.of(TestIdentities.userNToken)); } - public Application deploy(Application application, ApplicationPackage applicationPackage, ZoneId zone, long projectId) { + public Application deploy(Application application, ApplicationPackage applicationPackage, ZoneId zone) { controller().applications().deploy(application.id(), zone, Optional.of(applicationPackage), new DeployOptions(false, Optional.empty(), false, false)); return application; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java index 53f32dbcacd..cc275b0636f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java @@ -7,18 +7,21 @@ import com.yahoo.application.container.handler.Response; import com.yahoo.component.ComponentSpecification; import com.yahoo.component.Version; import com.yahoo.container.http.filter.FilterChainRepository; -import com.yahoo.io.IOUtils; import com.yahoo.jdisc.http.filter.SecurityRequestFilter; import com.yahoo.jdisc.http.filter.SecurityRequestFilterChain; -import com.yahoo.vespa.hosted.controller.integration.ConfigServerMock; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.SystemApplication; +import com.yahoo.vespa.hosted.controller.integration.ConfigServerMock; import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import org.junit.ComparisonFailure; import java.io.File; import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.charset.CharacterCodingException; +import java.nio.file.Files; +import java.nio.file.Paths; import java.util.Optional; import java.util.function.Supplier; import java.util.regex.Pattern; @@ -65,26 +68,31 @@ public class ContainerTester { computeVersionStatus(); } - public void assertResponse(Supplier<Request> request, File responseFile) throws IOException { + public void assertResponse(Supplier<Request> request, File responseFile) { assertResponse(request.get(), responseFile); } - public void assertResponse(Request request, File responseFile) throws IOException { + public void assertResponse(Request request, File responseFile) { assertResponse(request, responseFile, 200); } - public void assertResponse(Supplier<Request> request, File responseFile, int expectedStatusCode) throws IOException { + public void assertResponse(Supplier<Request> request, File responseFile, int expectedStatusCode) { assertResponse(request.get(), responseFile, expectedStatusCode); } - public void assertResponse(Request request, File responseFile, int expectedStatusCode) throws IOException { - String expectedResponse = IOUtils.readFile(new File(responseFilePath + responseFile.toString())); + public void assertResponse(Request request, File responseFile, int expectedStatusCode) { + String expectedResponse = readTestFile(responseFile.toString()); expectedResponse = include(expectedResponse); expectedResponse = expectedResponse.replaceAll("(\"[^\"]*\")|\\s*", "$1"); // Remove whitespace FilterResult filterResult = invokeSecurityFilters(request); request = filterResult.request; Response response = filterResult.response != null ? filterResult.response : container.handleRequest(request); - String responseString = response.getBodyAsString(); + String responseString; + try { + responseString = response.getBodyAsString(); + } catch (CharacterCodingException e) { + throw new UncheckedIOException(e); + } if (expectedResponse.contains("(ignore)")) { // Convert expected response to a literal pattern and replace any ignored field with a pattern that matches // until the first stop character @@ -141,7 +149,7 @@ public class ContainerTester { } /** Replaces @include(localFile) with the content of the file */ - private String include(String response) throws IOException { + private String include(String response) { // Please don't look at this code int includeIndex = response.indexOf("@include("); if (includeIndex < 0) return response; @@ -149,14 +157,22 @@ public class ContainerTester { String rest = response.substring(includeIndex + "@include(".length()); int filenameEnd = rest.indexOf(")"); String includeFileName = rest.substring(0, filenameEnd); - String includedContent = IOUtils.readFile(new File(responseFilePath + includeFileName)); + String includedContent = readTestFile(includeFileName); includedContent = include(includedContent); String postFix = rest.substring(filenameEnd + 1); postFix = include(postFix); return prefix + includedContent + postFix; } - static class FilterResult { + private String readTestFile(String name) { + try { + return new String(Files.readAllBytes(Paths.get(responseFilePath, name))); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private static class FilterResult { final Request request; final Response response; @@ -165,5 +181,6 @@ public class ContainerTester { this.response = response; } } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 4c37a9c3128..45513c2294f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -18,7 +18,6 @@ import com.yahoo.vespa.athenz.api.AthenzUser; import com.yahoo.vespa.athenz.api.NToken; import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.hosted.controller.Application; -import com.yahoo.vespa.hosted.controller.integration.ConfigServerMock; import com.yahoo.vespa.hosted.controller.api.identifiers.Property; import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; import com.yahoo.vespa.hosted.controller.api.identifiers.ScrewdriverId; @@ -33,6 +32,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.organization.MockOrgani import com.yahoo.vespa.hosted.controller.api.integration.organization.User; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.ClusterInfo; import com.yahoo.vespa.hosted.controller.application.ClusterUtilization; import com.yahoo.vespa.hosted.controller.application.Deployment; @@ -43,6 +43,7 @@ import com.yahoo.vespa.hosted.controller.athenz.mock.AthenzClientFactoryMock; import com.yahoo.vespa.hosted.controller.athenz.mock.AthenzDbMock; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.BuildJob; +import com.yahoo.vespa.hosted.controller.integration.ConfigServerMock; import com.yahoo.vespa.hosted.controller.restapi.ContainerControllerTester; import com.yahoo.vespa.hosted.controller.restapi.ContainerTester; import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest; @@ -97,6 +98,7 @@ public class ApplicationApiTest extends ControllerContainerTest { .build(); private static final AthenzDomain ATHENZ_TENANT_DOMAIN = new AthenzDomain("domain1"); + private static final AthenzDomain ATHENZ_TENANT_DOMAIN_2 = new AthenzDomain("domain2"); private static final ScrewdriverId SCREWDRIVER_ID = new ScrewdriverId("12345"); private static final UserId USER_ID = new UserId("myuser"); private static final UserId HOSTED_VESPA_OPERATOR = new UserId("johnoperator"); @@ -145,7 +147,7 @@ public class ApplicationApiTest extends ControllerContainerTest { // Add another Athens domain, so we can try to create more tenants - createAthenzDomainWithAdmin(new AthenzDomain("domain2"), USER_ID); // New domain to test tenant w/property ID + createAthenzDomainWithAdmin(ATHENZ_TENANT_DOMAIN_2, USER_ID); // New domain to test tenant w/property ID // Add property info for that property id, as well, in the mock organization. addPropertyData((MockOrganization) controllerTester.controller().organization(), "1234"); // POST (add) a tenant with property ID @@ -194,7 +196,7 @@ public class ApplicationApiTest extends ControllerContainerTest { ATHENZ_TENANT_DOMAIN, new com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId(id.application().value())); // (Necessary but not provided in this API) - // Trigger deployment from completion of component job + // Pipeline notifies about completed component job controllerTester.jobCompletion(JobType.component) .application(id) .projectId(screwdriverProjectId) @@ -203,13 +205,13 @@ public class ApplicationApiTest extends ControllerContainerTest { // ... systemtest tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/test/region/us-east-1/instance/default/", POST) - .data(createApplicationDeployData(applicationPackage, false)) + .data(createApplicationDeployData(Optional.empty(), false)) .screwdriverIdentity(SCREWDRIVER_ID), new File("deploy-result.json")); tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/test/region/us-east-1/instance/default", DELETE) .screwdriverIdentity(SCREWDRIVER_ID), "Deactivated tenant/tenant1/application/application1/environment/test/region/us-east-1/instance/default"); - // Called through the separate screwdriver/v1 API + controllerTester.jobCompletion(JobType.systemTest) .application(id) .projectId(screwdriverProjectId) @@ -217,7 +219,7 @@ public class ApplicationApiTest extends ControllerContainerTest { // ... staging tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/staging/region/us-east-3/instance/default/", POST) - .data(createApplicationDeployData(applicationPackage, false)) + .data(createApplicationDeployData(Optional.empty(), false)) .screwdriverIdentity(SCREWDRIVER_ID), new File("deploy-result.json")); tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/staging/region/us-east-3/instance/default", DELETE) @@ -230,7 +232,7 @@ public class ApplicationApiTest extends ControllerContainerTest { // ... prod zone tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default/", POST) - .data(createApplicationDeployData(applicationPackage, false)) + .data(createApplicationDeployData(Optional.empty(), false)) .screwdriverIdentity(SCREWDRIVER_ID), new File("deploy-result.json")); controllerTester.jobCompletion(JobType.productionCorpUsEast1) @@ -239,6 +241,43 @@ public class ApplicationApiTest extends ControllerContainerTest { .unsuccessful() .submit(); + // POST (create) another application + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .environment(Environment.prod) + .region("us-west-1") + .build(); + + tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2", POST) + .userIdentity(USER_ID) + .nToken(N_TOKEN), + new File("application-reference-2.json")); + + ApplicationId app2 = ApplicationId.from("tenant2", "application2", "default"); + long screwdriverProjectId2 = 456; + addScrewdriverUserToDeployRole(SCREWDRIVER_ID, + ATHENZ_TENANT_DOMAIN_2, + new com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId(app2.application().value())); + + // Trigger upgrade and then application change + controllerTester.controller().applications().deploymentTrigger().triggerChange(app2, Change.of(Version.fromString("7.0"))); + + controllerTester.jobCompletion(JobType.component) + .application(app2) + .projectId(screwdriverProjectId2) + .uploadArtifact(applicationPackage) + .submit(); + + // GET application having both change and outstanding change + tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2", GET) + .userIdentity(USER_ID), + new File("application2.json")); + + // DELETE application + tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2", DELETE) + .userIdentity(USER_ID) + .nToken(N_TOKEN), + ""); + // GET tenant screwdriver projects tester.assertResponse(request("/application/v4/tenant-pipeline/", GET) .userIdentity(USER_ID), @@ -383,8 +422,6 @@ public class ApplicationApiTest extends ControllerContainerTest { tester.assertResponse(request("/application/v4/tenant/tenant1", DELETE).userIdentity(USER_ID) .nToken(N_TOKEN), new File("tenant-without-applications.json")); - - controllerTester.controller().deconstruct(); } private void addIssues(ContainerControllerTester tester, ApplicationId id) { @@ -395,7 +432,7 @@ public class ApplicationApiTest extends ControllerContainerTest { } @Test - public void testDeployDirectly() throws Exception { + public void testDeployDirectly() { // Setup ContainerControllerTester controllerTester = new ContainerControllerTester(container, responseFiles); ContainerTester tester = controllerTester.containerTester(); @@ -427,11 +464,10 @@ public class ApplicationApiTest extends ControllerContainerTest { new File("deploy-result.json")); } - // Tests deployment to config server when using just on API call // For now this depends on a switch in ApplicationController that does this for by- tenants in CD only @Test - public void testDeployDirectlyUsingOneCallForDeploy() throws Exception { + public void testDeployDirectlyUsingOneCallForDeploy() { // Setup ContainerControllerTester controllerTester = new ContainerControllerTester(container, responseFiles); ContainerTester tester = controllerTester.containerTester(); @@ -813,7 +849,7 @@ public class ApplicationApiTest extends ControllerContainerTest { } @Test - public void deployment_succeeds_when_correct_domain_is_used() throws IOException { + public void deployment_succeeds_when_correct_domain_is_used() { ContainerControllerTester controllerTester = new ContainerControllerTester(container, responseFiles); ContainerTester tester = controllerTester.containerTester(); ApplicationPackage applicationPackage = new ApplicationPackageBuilder() @@ -844,7 +880,7 @@ public class ApplicationApiTest extends ControllerContainerTest { } @Test - public void testJobStatusReporting() throws Exception { + public void testJobStatusReporting() { ContainerControllerTester tester = new ContainerControllerTester(container, responseFiles); addUserToHostedOperatorRole(HostedAthenzIdentities.from(HOSTED_VESPA_OPERATOR)); tester.containerTester().computeVersionStatus(); @@ -861,7 +897,7 @@ public class ApplicationApiTest extends ControllerContainerTest { .application(app) .projectId(projectId); job.type(JobType.component).uploadArtifact(applicationPackage).submit(); - tester.deploy(app, applicationPackage, TEST_ZONE, projectId); + tester.deploy(app, applicationPackage, TEST_ZONE); job.type(JobType.systemTest).submit(); // Notifying about unknown job fails @@ -909,9 +945,9 @@ public class ApplicationApiTest extends ControllerContainerTest { .projectId(projectId); job.type(JobType.component).uploadArtifact(applicationPackage).submit(); - tester.deploy(app, applicationPackage, TEST_ZONE, projectId); + tester.deploy(app, applicationPackage, TEST_ZONE); job.type(JobType.systemTest).submit(); - tester.deploy(app, applicationPackage, STAGING_ZONE, projectId); + tester.deploy(app, applicationPackage, STAGING_ZONE); job.type(JobType.stagingTest).error(DeploymentJobs.JobError.outOfCapacity).submit(); // Appropriate error is recorded @@ -1104,8 +1140,6 @@ public class ApplicationApiTest extends ControllerContainerTest { * Cluster info, utilization and application and deployment metrics are maintained async by maintainers. * * This sets these values as if the maintainers has been ran. - * - * @param controllerTester */ private void setDeploymentMaintainedInfo(ContainerControllerTester controllerTester) { for (Application application : controllerTester.controller().applications().asList()) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/PathTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/PathTest.java index 7b1d8d17a5c..0bce930175d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/PathTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/PathTest.java @@ -15,12 +15,12 @@ public class PathTest { @Test public void testPath() { - assertFalse(new Path("").matches("/a/{foo}/bar/{b}"));; - assertFalse(new Path("///").matches("/a/{foo}/bar/{b}"));; - assertFalse(new Path("///foo").matches("/a/{foo}/bar/{b}"));; - assertFalse(new Path("///bar/").matches("/a/{foo}/bar/{b}"));; + assertFalse(new Path("").matches("/a/{foo}/bar/{b}")); + assertFalse(new Path("///").matches("/a/{foo}/bar/{b}")); + assertFalse(new Path("///foo").matches("/a/{foo}/bar/{b}")); + assertFalse(new Path("///bar/").matches("/a/{foo}/bar/{b}")); Path path = new Path("/a/1/bar/fuz"); - assertTrue(path.matches("/a/{foo}/bar/{b}"));; + assertTrue(path.matches("/a/{foo}/bar/{b}")); assertEquals("1", path.get("foo")); assertEquals("fuz", path.get("b")); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-reference-2.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-reference-2.json new file mode 100644 index 00000000000..a4026d6a812 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-reference-2.json @@ -0,0 +1,5 @@ +{ + "application": "application2", + "instance": "default", + "url": "http://localhost:8080/application/v4/tenant/tenant2/application/application2" +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application2.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application2.json new file mode 100644 index 00000000000..fa51d645cfc --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application2.json @@ -0,0 +1,78 @@ +{ + "application": "application2", + "instance": "default", + "deploying": { + "version": "(ignore)" + }, + "outstandingChange": { + "revision": { + "hash": "(ignore)", + "source": { + "gitRepository": "repository1", + "gitBranch": "master", + "gitCommit": "commit1" + } + } + }, + "deploymentJobs": [ + { + "type": "component", + "success": true, + "lastCompleted": { + "id": 42, + "version": "(ignore)", + "revision": { + "hash": "(ignore)", + "source": { + "gitRepository": "repository1", + "gitBranch": "master", + "gitCommit": "commit1" + } + }, + "reason": "Application commit", + "at": "(ignore)" + }, + "lastSuccess": { + "id": 42, + "version": "(ignore)", + "revision": { + "hash": "(ignore)", + "source": { + "gitRepository": "repository1", + "gitBranch": "master", + "gitCommit": "commit1" + } + }, + "reason": "Application commit", + "at": "(ignore)" + } + }, + { + "type": "system-test", + "success": false, + "lastTriggered": { + "id": -1, + "version": "7.0.0", + "revision": { + "hash": "1.0.42-commit1", + "source": { + "gitRepository": "repository1", + "gitBranch": "master", + "gitCommit": "commit1" + } + }, + "reason": "Testing last changes outside prod", + "at": "(ignore)" + } + } + ], + "changeBlockers": [], + "compileVersion": "6.1.0", + "globalRotations": [], + "instances": [], + "metrics": { + "queryServiceQuality": 0.0, + "writeServiceQuality": 0.0 + }, + "activity": {} +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java index 61f915f3c1e..1ed2af1f7b9 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java @@ -107,14 +107,14 @@ public class DeploymentApiTest extends ControllerContainerTest { .projectId(projectId) .uploadArtifact(applicationPackage) .submit(); - tester.deploy(application, applicationPackage, ZoneId.from(Environment.test, RegionName.from("us-east-1")), - projectId); + tester.deploy(application, applicationPackage, ZoneId.from(Environment.test, RegionName.from("us-east-1")) + ); tester.jobCompletion(JobType.systemTest) .application(application) .projectId(projectId) .submit(); - tester.deploy(application, applicationPackage, ZoneId.from(Environment.staging, RegionName.from("us-east-3")), - projectId); + tester.deploy(application, applicationPackage, ZoneId.from(Environment.staging, RegionName.from("us-east-3")) + ); tester.jobCompletion(JobType.stagingTest) .application(application) .projectId(projectId) @@ -122,7 +122,7 @@ public class DeploymentApiTest extends ControllerContainerTest { .submit(); if (success) { tester.deploy(application, applicationPackage, ZoneId.from(Environment.prod, - RegionName.from("corp-us-east-1")), projectId); + RegionName.from("corp-us-east-1"))); tester.jobCompletion(JobType.productionCorpUsEast1) .application(application) .projectId(projectId) diff --git a/jdisc_core_test/test_bundles/cert-b/pom.xml b/jdisc_core_test/test_bundles/cert-b/pom.xml index fb2a18ebc97..c634dda581a 100644 --- a/jdisc_core_test/test_bundles/cert-b/pom.xml +++ b/jdisc_core_test/test_bundles/cert-b/pom.xml @@ -12,20 +12,17 @@ </parent> <artifactId>cert-b</artifactId> <version>6-SNAPSHOT</version> - <packaging>bundle</packaging> + <packaging>container-plugin</packaging> <name>${project.artifactId}</name> <build> <plugins> <plugin> - <groupId>org.apache.felix</groupId> - <artifactId>maven-bundle-plugin</artifactId> + <groupId>com.yahoo.vespa</groupId> + <artifactId>bundle-plugin</artifactId> <extensions>true</extensions> <configuration> - <instructions> - <Export-Package> - com.yahoo.jdisc.bundle.b - </Export-Package> - </instructions> + <!-- The Vespa bundle-plugin doesn't include groupId. TODO Vespa 7: remove if that is fixed. --> + <Bundle-SymbolicName>${project.groupId}.${project.artifactId}</Bundle-SymbolicName> </configuration> </plugin> </plugins> diff --git a/jdisc_core_test/test_bundles/cert-b/src/main/java/com/yahoo/jdisc/bundle/b/package-info.java b/jdisc_core_test/test_bundles/cert-b/src/main/java/com/yahoo/jdisc/bundle/b/package-info.java new file mode 100644 index 00000000000..2773287114b --- /dev/null +++ b/jdisc_core_test/test_bundles/cert-b/src/main/java/com/yahoo/jdisc/bundle/b/package-info.java @@ -0,0 +1,4 @@ +@ExportPackage +package com.yahoo.jdisc.bundle.b; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/standalone-container/src/main/java/com/yahoo/container/standalone/StandaloneContainerApplication.java b/standalone-container/src/main/java/com/yahoo/container/standalone/StandaloneContainerApplication.java index 72937301954..f5c18f2b255 100644 --- a/standalone-container/src/main/java/com/yahoo/container/standalone/StandaloneContainerApplication.java +++ b/standalone-container/src/main/java/com/yahoo/container/standalone/StandaloneContainerApplication.java @@ -230,7 +230,7 @@ public class StandaloneContainerApplication implements Application { ApplicationPackage applicationPackage = rawApplicationPackage.preprocess(Zone.defaultZone(), logger); validateApplication(applicationPackage); DeployState deployState = new DeployState.Builder().applicationPackage(applicationPackage).fileRegistry(fileRegistry) - .deployLogger(logger).configDefinitionRepo(configDefinitionRepo).build(true); + .deployLogger(logger).configDefinitionRepo(configDefinitionRepo).build(); VespaModel root = VespaModel.createIncomplete(deployState); ApplicationConfigProducerRoot vespaRoot = new ApplicationConfigProducerRoot(root, "vespa", deployState.getDocumentModel(), diff --git a/vespaclient/src/perl/lib/Yahoo/Vespa/VespaModel.pm b/vespaclient/src/perl/lib/Yahoo/Vespa/VespaModel.pm index b1675130ec4..fd324540bba 100644 --- a/vespaclient/src/perl/lib/Yahoo/Vespa/VespaModel.pm +++ b/vespaclient/src/perl/lib/Yahoo/Vespa/VespaModel.pm @@ -170,10 +170,9 @@ sub retrieveModelConfigDefault { # () if (!defined $CONFIG_SERVER_HOST) { my $temp = `${VESPA_HOME}/bin/vespa-print-default configservers`; - my @configServerHosts = split(' ', $temp); - $CONFIG_SERVER_HOST = $configServerHosts[0]; + chomp($temp); + $CONFIG_SERVER_HOST = $temp; } - $cmd .= " -s $CONFIG_SERVER_HOST"; if (!defined $CONFIG_SERVER_PORT) { my $temp = `${VESPA_HOME}/bin/vespa-print-default configserver_rpc_port`; @@ -182,16 +181,23 @@ sub retrieveModelConfigDefault { # () } $cmd .= " -p $CONFIG_SERVER_PORT"; - printDebug "Fetching model config '$cmd'.\n"; - my @data = `$cmd 2>&1`; - if ($? != 0 || join(' ', @data) =~ /^error/) { - printError "Failed to get model config from config command line tool:\n" - . "Command: $cmd\n" + my $errors = ""; + foreach my $cfshost (split(' ', $CONFIG_SERVER_HOST)) { + my $hostcmd = $cmd . " -s $cfshost"; + + printDebug "Fetching model config '$hostcmd'.\n"; + my @data = `$hostcmd 2>&1`; + if ($? != 0 || join(' ', @data) =~ /^error/) { + $errors .= "Failed to get model config from config command line tool:\n" + . "Command: $hostcmd\n" . "Exit code: $?\n" . "Output: " . join("\n", @data) . "\n"; - exitApplication(1); + } else { + return @data; + } } - return @data; + printError $errors; + exitApplication(1); } sub fetch { # () my @data = &$RETRIEVE_MODEL_CONFIG(); diff --git a/vespajlib/src/main/java/com/yahoo/slime/JsonDecoder.java b/vespajlib/src/main/java/com/yahoo/slime/JsonDecoder.java index 3f73faf289d..71eaa7f8b25 100644 --- a/vespajlib/src/main/java/com/yahoo/slime/JsonDecoder.java +++ b/vespajlib/src/main/java/com/yahoo/slime/JsonDecoder.java @@ -210,6 +210,7 @@ public class JsonDecoder { private static byte[] unicodeStart = {'\\', 'u'}; private long dequoteUtf16() { + next(); long codepoint = readHexValue(4); if (codepoint >= 0xd800) { if (codepoint < 0xdc00) { // high diff --git a/vespajlib/src/test/java/com/yahoo/slime/JsonFormatTestCase.java b/vespajlib/src/test/java/com/yahoo/slime/JsonFormatTestCase.java index d82cb61a08c..44aa4ab2ff7 100644 --- a/vespajlib/src/test/java/com/yahoo/slime/JsonFormatTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/slime/JsonFormatTestCase.java @@ -198,6 +198,15 @@ public class JsonFormatTestCase { } @Test + public void testDecodeUnicodeAmp() { + final String json = "{\"body\":\"some text\\u0026more text\"}"; + Slime slime = new Slime(); + new JsonDecoder().decode(slime, Utf8.toBytesStd(json)); + Cursor a = slime.get().field("body"); + assertThat(a.asString(), is("some text&more text")); + } + + @Test public void testDecodeEncodeUtf8() { final String json = "{\n" + " \"rules\": \"# Use unicode equivalents in java source:\\n" + |