diff options
35 files changed, 231 insertions, 164 deletions
diff --git a/config-application-package/pom.xml b/config-application-package/pom.xml index 7f49e9fb8b6..8236458c532 100644 --- a/config-application-package/pom.xml +++ b/config-application-package/pom.xml @@ -27,6 +27,12 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> + <artifactId>component</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> <artifactId>vespajlib</artifactId> <version>${project.version}</version> <scope>provided</scope> diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/ApplicationPackageXmlFilesValidator.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/ApplicationPackageXmlFilesValidator.java index 201c18527bb..3bc956ae377 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/ApplicationPackageXmlFilesValidator.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/ApplicationPackageXmlFilesValidator.java @@ -2,8 +2,8 @@ package com.yahoo.config.model.application.provider; import com.yahoo.collections.Tuple2; +import com.yahoo.component.Version; import com.yahoo.config.application.api.ApplicationPackage; -import com.yahoo.config.provision.Version; import com.yahoo.path.Path; import com.yahoo.io.reader.NamedReader; @@ -21,22 +21,24 @@ import java.util.Optional; public class ApplicationPackageXmlFilesValidator { private final AppSubDirs appDirs; - private final Optional<Version> vespaVersion; + + /** The Vespa version this package tshould be validated against */ + private final Version vespaVersion; private static final FilenameFilter xmlFilter = (dir, name) -> name.endsWith(".xml"); - public ApplicationPackageXmlFilesValidator(AppSubDirs appDirs, Optional<Version> vespaVersion) { + public ApplicationPackageXmlFilesValidator(AppSubDirs appDirs, Version vespaVersion) { this.appDirs = appDirs; this.vespaVersion = vespaVersion; } - public static ApplicationPackageXmlFilesValidator createDefaultXMLValidator(File appDir, Optional<Version> vespaVersion) { + public static ApplicationPackageXmlFilesValidator createDefaultXMLValidator(File appDir, Version vespaVersion) { return new ApplicationPackageXmlFilesValidator(new AppSubDirs(appDir), vespaVersion); } - public static ApplicationPackageXmlFilesValidator createTestXmlValidator(File appDir) { - return new ApplicationPackageXmlFilesValidator(new AppSubDirs(appDir), Optional.empty()); + public static ApplicationPackageXmlFilesValidator createTestXmlValidator(File appDir, Version vespaVersion) { + return new ApplicationPackageXmlFilesValidator(new AppSubDirs(appDir), vespaVersion); } @SuppressWarnings("deprecation") @@ -56,11 +58,11 @@ public class ApplicationPackageXmlFilesValidator { } // For testing - public static void checkIncludedDirs(ApplicationPackage app) throws IOException { + public static void checkIncludedDirs(ApplicationPackage app, Version vespaVersion) throws IOException { for (String includedDir : app.getUserIncludeDirs()) { List<NamedReader> includedFiles = app.getFiles(Path.fromString(includedDir), ".xml", true); for (NamedReader file : includedFiles) { - createSchemaValidator("container-include.rnc", Optional.empty()).validate(file); + createSchemaValidator("container-include.rnc", vespaVersion).validate(file); } } } @@ -115,7 +117,7 @@ public class ApplicationPackageXmlFilesValidator { } } - private static SchemaValidator createSchemaValidator(String schemaFile, Optional<Version> vespaVersion) { + private static SchemaValidator createSchemaValidator(String schemaFile, Version vespaVersion) { return new SchemaValidator(schemaFile, new BaseDeployLogger(), vespaVersion); } diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java index 06e74e08307..63f19e9abf3 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java @@ -1,6 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.model.application.provider; +import com.yahoo.component.Vtag; import com.yahoo.config.application.ConfigDefinitionDir; import com.yahoo.config.application.Xml; import com.yahoo.config.application.XmlPreProcessor; @@ -630,9 +631,10 @@ public class FilesApplicationPackage implements ApplicationPackage { @Override public void validateXML(Optional<Version> vespaVersion) throws IOException { - ApplicationPackageXmlFilesValidator xmlFilesValidator = ApplicationPackageXmlFilesValidator.createDefaultXMLValidator(appDir, vespaVersion); + com.yahoo.component.Version modelVersion = vespaVersion.map(v -> new com.yahoo.component.Version(vespaVersion.toString())).orElse(Vtag.currentVersion); + ApplicationPackageXmlFilesValidator xmlFilesValidator = ApplicationPackageXmlFilesValidator.createDefaultXMLValidator(appDir, modelVersion); xmlFilesValidator.checkApplication(); - ApplicationPackageXmlFilesValidator.checkIncludedDirs(this); + ApplicationPackageXmlFilesValidator.checkIncludedDirs(this, modelVersion); } @Override diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/SchemaValidator.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/SchemaValidator.java index 698fa8fdce7..fd1156d435a 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/SchemaValidator.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/SchemaValidator.java @@ -6,8 +6,8 @@ import com.thaiopensource.util.PropertyMapBuilder; import com.thaiopensource.validate.ValidateProperty; import com.thaiopensource.validate.ValidationDriver; import com.thaiopensource.validate.rng.CompactSchemaReader; +import com.yahoo.component.Version; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.config.provision.Version; import com.yahoo.io.IOUtils; import com.yahoo.io.reader.NamedReader; import com.yahoo.log.LogLevel; @@ -27,7 +27,6 @@ import java.io.Reader; import java.net.JarURLConnection; import java.net.URL; import java.util.Enumeration; -import java.util.Optional; import java.util.jar.JarEntry; import java.util.jar.JarFile; import java.util.logging.Level; @@ -53,8 +52,9 @@ public class SchemaValidator { * Initializes the validator by using the given file as schema file * @param schema a schema file in RNC format * @param logger a logger + * @param vespaVersion the version of Vespa we should validate against */ - public SchemaValidator(String schema, DeployLogger logger, Optional<Version> vespaVersion) { + public SchemaValidator(String schema, DeployLogger logger, Version vespaVersion) { this.deployLogger = logger; driver = new ValidationDriver(PropertyMap.EMPTY, instanceProperties(), CompactSchemaReader.getInstance()); File schemaDir = new File(schemaDirBase); @@ -70,26 +70,27 @@ public class SchemaValidator { /** * Initializes the validator by using the given file as schema file * @param schema a schema file in RNC format + * @param vespaVersion the version we should validate against * @throws IOException if it is not possible to read schema files */ - public SchemaValidator(String schema) throws IOException { - this(schema, new BaseDeployLogger(), Optional.empty()); + public SchemaValidator(String schema, Version vespaVersion) throws IOException { + this(schema, new BaseDeployLogger(), vespaVersion); } /** * Create a validator for services.xml for tests * @throws IOException if it is not possible to read schema files */ - public static SchemaValidator createTestValidatorServices() throws IOException { - return new SchemaValidator(servicesXmlSchemaName); + public static SchemaValidator createTestValidatorServices(Version vespaVersion) throws IOException { + return new SchemaValidator(servicesXmlSchemaName, vespaVersion); } /** * Create a validator for hosts.xml for tests * @throws IOException if it is not possible to read schema files */ - public static SchemaValidator createTestValidatorHosts() throws IOException { - return new SchemaValidator(hostsXmlSchemaName); + public static SchemaValidator createTestValidatorHosts(Version vespaVersion) throws IOException { + return new SchemaValidator(hostsXmlSchemaName, vespaVersion); } /** @@ -97,8 +98,8 @@ public class SchemaValidator { * * @throws IOException if it is not possible to read schema files */ - public static SchemaValidator createTestValidatorDeployment() throws IOException { - return new SchemaValidator(deploymentXmlSchemaName); + public static SchemaValidator createTestValidatorDeployment(Version vespaVersion) throws IOException { + return new SchemaValidator(deploymentXmlSchemaName, vespaVersion); } private class CustomErrorHandler implements ErrorHandler { @@ -129,7 +130,7 @@ public class SchemaValidator { * @return the directory the schema files are stored in * @throws IOException if it is not possible to read schema files */ - public File saveSchemasFromJar(File tmpBase, Optional<Version> vespaVersion) throws IOException { + private File saveSchemasFromJar(File tmpBase, Version vespaVersion) throws IOException { final Class<? extends SchemaValidator> schemaValidatorClass = this.getClass(); final ClassLoader classLoader = schemaValidatorClass.getClassLoader(); Enumeration<URL> uris = classLoader.getResources("schema"); @@ -158,7 +159,7 @@ public class SchemaValidator { // TODO: Hack to handle cases where bundle=null if (bundle == null) { File schemaPath; - if (vespaVersion.isPresent() && vespaVersion.get().getMajor() == 5) { + if (vespaVersion.getMajor() == 5) { schemaPath = new File(Defaults.getDefaults().vespaHome() + "share/vespa/schema/version/5.x/schema/"); } else { schemaPath = new File(Defaults.getDefaults().vespaHome() + "share/vespa/schema/"); @@ -234,12 +235,12 @@ public class SchemaValidator { errorHandler.fileName = (fileName == null ? " input" : fileName); try { if ( ! driver.validate(inputSource)) { - //Shouldn't happen, error handler should have thrown + // Shouldn't happen, error handler should have thrown throw new RuntimeException("Aborting due to earlier XML errors."); } } catch (SAXException e) { - //This should never happen, as it is handled by the ErrorHandler - //installed for the driver. + // This should never happen, as it is handled by the ErrorHandler + // installed for the driver. throw new IllegalArgumentException( "XML error in " + (fileName == null ? " input" : fileName) + ": " + Exceptions.toMessageString(e)); } diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/SimpleApplicationValidator.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/SimpleApplicationValidator.java index 2d960f00d66..97b989617b3 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/SimpleApplicationValidator.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/SimpleApplicationValidator.java @@ -1,6 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.model.application.provider; +import com.yahoo.component.Version; + import java.io.IOException; import java.io.Reader; @@ -11,7 +13,7 @@ import java.io.Reader; */ public class SimpleApplicationValidator { - public static void checkServices(Reader reader) throws IOException { - SchemaValidator.createTestValidatorServices().validate(reader); + public static void checkServices(Reader reader, Version version) throws IOException { + SchemaValidator.createTestValidatorServices(version).validate(reader); } } diff --git a/config-model-api/pom.xml b/config-model-api/pom.xml index cf016ec4ee0..8fa2d482cab 100644 --- a/config-model-api/pom.xml +++ b/config-model-api/pom.xml @@ -29,6 +29,12 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> + <artifactId>component</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> <artifactId>defaults</artifactId> <version>${project.version}</version> <scope>provided</scope> diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 9226e3bbe11..9b9869a5a3c 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -1,12 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.model.api; +import com.yahoo.component.Version; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.application.api.FileRegistry; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Rotation; -import com.yahoo.config.provision.Version; import com.yahoo.config.provision.Zone; import java.io.File; @@ -30,7 +30,14 @@ public interface ModelContext { FileRegistry getFileRegistry(); Properties properties(); default Optional<File> appDir() { return Optional.empty();} - default Optional<Version> vespaVersion() { return Optional.empty(); } + + /** @deprecated TODO: Remove this when no config models older than 6.98 are used */ + @SuppressWarnings("unused") + @Deprecated + default Optional<com.yahoo.config.provision.Version> vespaVersion() { return Optional.empty(); } + + Version modelVespaVersion(); + Version wantedNodeVespaVersion(); interface Properties { boolean multitenant(); diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelFactory.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelFactory.java index a61001cc9d9..2f80e66af05 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelFactory.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelFactory.java @@ -9,10 +9,9 @@ import com.yahoo.config.provision.Version; public interface ModelFactory { /** - * Gets version of this {@link ModelFactory}. The version will be used to dispatch deployments to the correct - * {@link ModelFactory}. + * Returns the Vespa version of the models this builds. * - * @return The version of a {@link Model} instance that this factory can create. + * @return the version of a {@link Model} instance that this factory can create. */ Version getVersion(); @@ -21,7 +20,7 @@ public interface ModelFactory { * validation will be done, calling this method assumes that {@link #createAndValidateModel} has already * been called at some point for this model. * - * @param modelContext An instance of {@link ModelContext}, containing dependencies for creating a {@link Model}. + * @param modelContext an instance of {@link ModelContext}, containing dependencies for creating a {@link Model}. * @return a {@link Model} instance. */ Model createModel(ModelContext modelContext); @@ -30,8 +29,8 @@ public interface ModelFactory { * Creates an instance of a {@link Model}. The resulting instance will be used to serve config. Any validation * of a {@link Model} and the {@link ModelContext} can be done in this method. * - * @param modelContext An instance of {@link ModelContext}, containing dependencies for creating a {@link Model}. - * @param ignoreValidationErrors true if validation errors should not trigger exceptions. + * @param modelContext an instance of {@link ModelContext}, containing dependencies for creating a {@link Model} + * @param ignoreValidationErrors true if validation errors should not trigger exceptions * @return a {@link ModelCreateResult} instance. */ ModelCreateResult createAndValidateModel(ModelContext modelContext, boolean ignoreValidationErrors); diff --git a/config-model/src/main/java/com/yahoo/config/model/test/TestDriver.java b/config-model/src/main/java/com/yahoo/config/model/test/TestDriver.java index 4518e44d441..38523b33549 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/TestDriver.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/TestDriver.java @@ -2,6 +2,7 @@ package com.yahoo.config.model.test; import com.google.common.annotations.Beta; +import com.yahoo.component.Version; import com.yahoo.config.model.MapConfigModelRegistry; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.application.provider.SchemaValidator; @@ -101,11 +102,11 @@ public class TestDriver { if (!validate) { return; } - SchemaValidator validator = SchemaValidator.createTestValidatorHosts(); + SchemaValidator validator = SchemaValidator.createTestValidatorHosts(new Version(6)); if (appPkg.getHosts() != null) { validator.validate(appPkg.getHosts()); } - validator = SchemaValidator.createTestValidatorServices(); + validator = SchemaValidator.createTestValidatorServices(new Version(6)); validator.validate(appPkg.getServices()); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java index 35ea2134813..8e384b6e875 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java @@ -81,14 +81,24 @@ public class VespaModelFactory implements ModelFactory { } @Override - public ModelCreateResult createAndValidateModel(ModelContext modelContext, boolean ignoreValidationErrors) { + public ModelCreateResult createAndValidateModel(ModelContext modelContext, + boolean ignoreValidationErrors) { + validateXml(modelContext, ignoreValidationErrors); + DeployState deployState = createDeployState(modelContext); + VespaModel model = buildModel(deployState); + List<ConfigChangeAction> changeActions = validateModel(model, deployState, ignoreValidationErrors); + return new ModelCreateResult(model, changeActions); + } + + private void validateXml(ModelContext modelContext, boolean ignoreValidationErrors) { if (modelContext.appDir().isPresent()) { ApplicationPackageXmlFilesValidator validator = ApplicationPackageXmlFilesValidator.createDefaultXMLValidator(modelContext.appDir().get(), - modelContext.vespaVersion()); + modelContext.modelVespaVersion()); try { validator.checkApplication(); - ApplicationPackageXmlFilesValidator.checkIncludedDirs(modelContext.applicationPackage()); + ApplicationPackageXmlFilesValidator.checkIncludedDirs(modelContext.applicationPackage(), + modelContext.modelVespaVersion()); } catch (IllegalArgumentException e) { rethrowUnlessIgnoreErrors(e, ignoreValidationErrors); } catch (Exception e) { @@ -98,10 +108,6 @@ public class VespaModelFactory implements ModelFactory { } else { validateXML(modelContext.applicationPackage(), ignoreValidationErrors); } - DeployState deployState = createDeployState(modelContext); - VespaModel model = buildModel(deployState); - List<ConfigChangeAction> changeActions = validateModel(model, deployState, ignoreValidationErrors); - return new ModelCreateResult(model, changeActions); } private VespaModel buildModel(DeployState deployState) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java index c83f6098a0f..8eab3b0082d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java @@ -33,6 +33,7 @@ public class NodesSpecification { private final Optional<String> flavor; + /** The identifier of the custom docker image layer to use (not supported yet) */ private final Optional<String> dockerImage; private NodesSpecification(boolean dedicated, int count, int groups, boolean required, diff --git a/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java b/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java index b531fc515ed..9c07c7fc73e 100644 --- a/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java @@ -2,6 +2,7 @@ package com.yahoo.config.model; import com.google.common.io.Files; +import com.yahoo.component.Version; import com.yahoo.config.ConfigInstance; import com.yahoo.config.application.api.ApplicationMetaData; import com.yahoo.config.application.api.UnparsedConfigDefinition; @@ -239,9 +240,9 @@ public class ApplicationDeployTest { public FilesApplicationPackage createAppPkg(String appPkg, boolean validateXml) throws IOException { final FilesApplicationPackage filesApplicationPackage = FilesApplicationPackage.fromFile(new File(appPkg)); if (validateXml) { - ApplicationPackageXmlFilesValidator validator = ApplicationPackageXmlFilesValidator.createTestXmlValidator(new File(appPkg)); + ApplicationPackageXmlFilesValidator validator = ApplicationPackageXmlFilesValidator.createTestXmlValidator(new File(appPkg), new Version(6)); validator.checkApplication(); - ApplicationPackageXmlFilesValidator.checkIncludedDirs(filesApplicationPackage); + ApplicationPackageXmlFilesValidator.checkIncludedDirs(filesApplicationPackage, new Version(6)); } return filesApplicationPackage; } diff --git a/config-model/src/test/java/com/yahoo/config/model/MockModelContext.java b/config-model/src/test/java/com/yahoo/config/model/MockModelContext.java index 84e8977a391..20a941eb690 100644 --- a/config-model/src/test/java/com/yahoo/config/model/MockModelContext.java +++ b/config-model/src/test/java/com/yahoo/config/model/MockModelContext.java @@ -1,6 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.model; +import com.yahoo.component.Version; import com.yahoo.config.model.api.*; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.application.provider.BaseDeployLogger; @@ -23,6 +24,7 @@ import java.util.Set; * @author hmusum */ public class MockModelContext implements ModelContext { + private final ApplicationPackage applicationPackage; public MockModelContext() { @@ -67,6 +69,12 @@ public class MockModelContext implements ModelContext { public FileRegistry getFileRegistry() { return new MockFileRegistry(); } + + @Override + public Version modelVespaVersion() { return new Version(6); } + + @Override + public Version wantedNodeVespaVersion() { return new Version(6); } @Override public Properties properties() { diff --git a/config-model/src/test/java/com/yahoo/config/model/application/provider/SchemaValidatorTest.java b/config-model/src/test/java/com/yahoo/config/model/application/provider/SchemaValidatorTest.java index ca4a2e70ee6..cbf7f4e9cd5 100644 --- a/config-model/src/test/java/com/yahoo/config/model/application/provider/SchemaValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/application/provider/SchemaValidatorTest.java @@ -1,6 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.model.application.provider; +import com.yahoo.component.Version; import org.junit.Test; import org.xml.sax.InputSource; import org.xml.sax.SAXException; @@ -64,6 +65,6 @@ public class SchemaValidatorTest { } private SchemaValidator createValidator() throws IOException { - return SchemaValidator.createTestValidatorServices(); + return SchemaValidator.createTestValidatorServices(new Version(6)); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/ClusterControllerTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/ClusterControllerTestCase.java index 200e51ef924..d8116d01294 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/ClusterControllerTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/ClusterControllerTestCase.java @@ -5,6 +5,7 @@ import com.google.common.base.Function; import com.google.common.collect.Collections2; import com.yahoo.cloud.config.ZookeeperServerConfig; import com.yahoo.cloud.config.ZookeepersConfig; +import com.yahoo.component.Version; import com.yahoo.config.model.application.provider.SimpleApplicationValidator; import com.yahoo.config.model.test.TestDriver; import com.yahoo.config.model.test.TestRoot; @@ -434,7 +435,7 @@ public class ClusterControllerTestCase extends DomBuilderTest { .withServices(servicesXml) .withSearchDefinitions(sds) .build()); - SimpleApplicationValidator.checkServices(new StringReader(servicesXml)); + SimpleApplicationValidator.checkServices(new StringReader(servicesXml), new Version(6)); return model; } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithFilePkg.java b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithFilePkg.java index 14216733d2a..77825b3f370 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithFilePkg.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithFilePkg.java @@ -1,6 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.test.utils; +import com.yahoo.component.Version; import com.yahoo.config.model.ConfigModelRegistry; import com.yahoo.config.model.NullConfigModelRegistry; import com.yahoo.config.model.application.provider.*; @@ -44,8 +45,8 @@ public class VespaModelCreatorWithFilePkg { } public void validate() throws IOException { - ApplicationPackageXmlFilesValidator.createTestXmlValidator(applicationPkg.getAppDir()).checkApplication(); - ApplicationPackageXmlFilesValidator.checkIncludedDirs(applicationPkg); + ApplicationPackageXmlFilesValidator.createTestXmlValidator(applicationPkg.getAppDir(), new Version(6)).checkApplication(); + ApplicationPackageXmlFilesValidator.checkIncludedDirs(applicationPkg, new Version(6)); } public VespaModel create(boolean validateApplicationWithSchema) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithMockPkg.java b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithMockPkg.java index 2f83d3bc394..6d5471f649e 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithMockPkg.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithMockPkg.java @@ -1,6 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.test.utils; +import com.yahoo.component.Version; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.ConfigModelRegistry; import com.yahoo.config.model.NullConfigModelRegistry; @@ -53,15 +54,16 @@ public class VespaModelCreatorWithMockPkg { try { this.deployState = deployState; VespaModel model = new VespaModel(configModelRegistry, deployState); + Version vespaVersion = new Version(6); if (validate) { try { if (appPkg.getHosts() != null) { - SchemaValidator.createTestValidatorHosts().validate(appPkg.getHosts()); + SchemaValidator.createTestValidatorHosts(vespaVersion).validate(appPkg.getHosts()); } if (appPkg.getDeployment().isPresent()) { - SchemaValidator.createTestValidatorDeployment().validate(appPkg.getDeployment().get()); + SchemaValidator.createTestValidatorDeployment(vespaVersion).validate(appPkg.getDeployment().get()); } - SchemaValidator.createTestValidatorServices().validate(appPkg.getServices()); + SchemaValidator.createTestValidatorServices(vespaVersion).validate(appPkg.getServices()); } catch (Exception e) { System.err.println(e.getClass()); throw e instanceof RuntimeException ? (RuntimeException) e : new RuntimeException(e); diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java index 25c83d288eb..07b120bed73 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java @@ -47,7 +47,7 @@ public final class ClusterSpec { /** Create a specification <b>requesting</b> a cluster with these attributes */ @Deprecated - // TODO: April 2017 - Remove this we no longer have old config-models using it + // TODO: April 2017 - Remove this when no version older than 6.94 is used anywhere public static ClusterSpec request(Type type, Id id, Optional<String> dockerImage) { return requestVersion(type, id, dockerImage.map(DockerImage::new).map(DockerImage::tagAsVersion)); } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/Version.java b/config-provisioning/src/main/java/com/yahoo/config/provision/Version.java index 6dcb5fe207e..6792e31afd9 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/Version.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/Version.java @@ -15,6 +15,7 @@ package com.yahoo.config.provision; * Loosely based on component/Version.java * {@link Version} objects are immutable. */ +// TODO: Replace usage of this by com.yahoo.component.Version public final class Version implements Comparable<Version> { private final int major; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ServerCache.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ServerCache.java index 59cb7b63114..6d2567f711a 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ServerCache.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ServerCache.java @@ -68,4 +68,5 @@ public class ServerCache { public int checkSumElems() { return md5Sums.size(); } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 7f26795a003..5fac9f61b4b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -1,13 +1,13 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.deploy; +import com.yahoo.component.Version; import com.yahoo.config.model.api.*; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.application.api.FileRegistry; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Rotation; -import com.yahoo.config.provision.Version; import com.yahoo.config.provision.Zone; import java.io.File; @@ -31,7 +31,18 @@ public class ModelContextImpl implements ModelContext { private final Optional<HostProvisioner> hostProvisioner; private final ModelContext.Properties properties; private final Optional<File> appDir; - Optional<Version> vespaVersion; + + /** The version of Vespa we are building a model for */ + private final Version modelVespaVersion; + + /** + * The Version of Vespa this model should specify that nodes should use. Note that this + * is separate from the version of this model, as upgrades are not immediate. + * We may build a config model of Vespa version "a" which specifies that nodes should + * use Vespa version "b". The "a" model will then be used by nodes who have not yet + * upgraded to version "b". + */ + private final Version wantedNodeVespaVersion; public ModelContextImpl(ApplicationPackage applicationPackage, Optional<Model> previousModel, @@ -42,7 +53,8 @@ public class ModelContextImpl implements ModelContext { Optional<HostProvisioner> hostProvisioner, ModelContext.Properties properties, Optional<File> appDir, - Optional<Version> vespaVersion) { + Version modelVespaVersion, + Version wantedNodeVespaVersion) { this.applicationPackage = applicationPackage; this.previousModel = previousModel; this.permanentApplicationPackage = permanentApplicationPackage; @@ -52,7 +64,8 @@ public class ModelContextImpl implements ModelContext { this.hostProvisioner = hostProvisioner; this.properties = properties; this.appDir = appDir; - this.vespaVersion = vespaVersion; + this.modelVespaVersion = modelVespaVersion; + this.wantedNodeVespaVersion = wantedNodeVespaVersion; } @Override @@ -101,7 +114,10 @@ public class ModelContextImpl implements ModelContext { } @Override - public Optional<Version> vespaVersion() { return vespaVersion; } + public Version modelVespaVersion() { return modelVespaVersion; } + + @Override + public Version wantedNodeVespaVersion() { return wantedNodeVespaVersion; } /** * @author lulf diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java index 3b15da0fbea..70c855f064c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java @@ -72,10 +72,10 @@ public class ActivatedModelsBuilder extends ModelsBuilder<Application> { @Override protected Application buildModelVersion(ModelFactory modelFactory, ApplicationPackage applicationPackage, - ApplicationId applicationId) { - Version version = modelFactory.getVersion(); + ApplicationId applicationId, com.yahoo.component.Version wantedNodeVespaVersion) { log.log(LogLevel.DEBUG, String.format("Loading model version %s for session %s application %s", - version, appGeneration, applicationId)); + modelFactory.getVersion(), appGeneration, applicationId)); + ServerCache cache = zkClient.loadServerCache(); ModelContext modelContext = new ModelContextImpl( applicationPackage, Optional.<Model>empty(), @@ -86,10 +86,10 @@ public class ActivatedModelsBuilder extends ModelsBuilder<Application> { createHostProvisioner(getForVersionOrLatest(applicationPackage.getProvisionInfoMap(), modelFactory.getVersion())), createModelContextProperties(applicationId), Optional.empty(), - Optional.empty()); - ServerCache cache = zkClient.loadServerCache(); + new com.yahoo.component.Version(modelFactory.getVersion().toString()), + wantedNodeVespaVersion); MetricUpdater applicationMetricUpdater = metrics.getOrCreateMetricUpdater(Metrics.createDimensions(applicationId)); - return new Application(modelFactory.createModel(modelContext), cache, appGeneration, version, + return new Application(modelFactory.createModel(modelContext), cache, appGeneration, modelFactory.getVersion(), applicationMetricUpdater, applicationId); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java index 1b32d6bde22..f2b78e9e0dd 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java @@ -40,7 +40,8 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> { this.modelFactoryRegistry = modelFactoryRegistry; } - public List<MODELRESULT> buildModels(ApplicationId applicationId, ApplicationPackage applicationPackage) { + public List<MODELRESULT> buildModels(ApplicationId applicationId, + com.yahoo.component.Version wantedNodeVespaVersion, ApplicationPackage applicationPackage) { Set<Version> versions = modelFactoryRegistry.allVersions(); // If the application specifies a major, load models only for that @@ -60,7 +61,7 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> { for (int i = 0; i < majorVersions.size(); i++) { try { allApplicationModels.addAll(buildModelVersion(filterByMajorVersion(majorVersions.get(i), versions), - applicationId, applicationPackage)); + applicationId, wantedNodeVespaVersion, applicationPackage)); // skip old config models after we have found a major version which works if (allApplicationModels.size() > 0 && allApplicationModels.get(0).getModel().skipOldConfigModels()) @@ -82,10 +83,11 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> { } private List<MODELRESULT> buildModelVersion(Set<Version> versions, ApplicationId applicationId, + com.yahoo.component.Version wantedNodeVespaVersion, ApplicationPackage applicationPackage) { Version latest = findLatest(versions); // load latest application version - MODELRESULT latestApplicationVersion = buildModelVersion(modelFactoryRegistry.getFactory(latest), applicationPackage, applicationId); + MODELRESULT latestApplicationVersion = buildModelVersion(modelFactoryRegistry.getFactory(latest), applicationPackage, applicationId, wantedNodeVespaVersion); if (latestApplicationVersion.getModel().skipOldConfigModels()) { return Collections.singletonList(latestApplicationVersion); } @@ -94,7 +96,7 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> { allApplicationVersions.add(latestApplicationVersion); for (Version version : versions) { if (version.equals(latest)) continue; // already loaded - allApplicationVersions.add(buildModelVersion(modelFactoryRegistry.getFactory(version), applicationPackage, applicationId)); + allApplicationVersions.add(buildModelVersion(modelFactoryRegistry.getFactory(version), applicationPackage, applicationId, wantedNodeVespaVersion)); } return allApplicationVersions; } @@ -114,7 +116,8 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> { } protected abstract MODELRESULT buildModelVersion(ModelFactory modelFactory, ApplicationPackage applicationPackage, - ApplicationId applicationId); + ApplicationId applicationId, + com.yahoo.component.Version wantedNodeVespaVersion); protected ModelContext.Properties createModelContextProperties(ApplicationId applicationId, ConfigserverConfig configserverConfig, diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java index cb2ffe1c2f9..c158f5ab03d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java @@ -1,6 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.modelfactory; +import com.yahoo.component.Vtag; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.api.ConfigChangeAction; @@ -75,17 +76,19 @@ public class PreparedModelsBuilder extends ModelsBuilder<PreparedModelsBuilder.P } @Override - protected PreparedModelResult buildModelVersion(ModelFactory modelFactory, ApplicationPackage applicationPackage, - ApplicationId applicationId) { - Version version = modelFactory.getVersion(); - log.log(LogLevel.DEBUG, "Start building model for Vespa version " + version); + protected PreparedModelResult buildModelVersion(ModelFactory modelFactory, + ApplicationPackage applicationPackage, + ApplicationId applicationId, + com.yahoo.component.Version wantedNodeVespaVersion) { + Version modelVersion = modelFactory.getVersion(); + log.log(LogLevel.DEBUG, "Start building model for Vespa version " + modelVersion); FileDistributionProvider fileDistributionProvider = fileDistributionFactory.createProvider( context.getServerDBSessionDir(), applicationId); Optional<HostProvisioner> hostProvisioner = createHostProvisionerAdapter(properties); Optional<Model> previousModel = currentActiveApplicationSet - .map(set -> set.getForVersionOrLatest(Optional.of(version)).getModel()); + .map(set -> set.getForVersionOrLatest(Optional.of(modelVersion)).getModel()); ModelContext modelContext = new ModelContextImpl( applicationPackage, previousModel, @@ -96,13 +99,14 @@ public class PreparedModelsBuilder extends ModelsBuilder<PreparedModelsBuilder.P hostProvisioner, properties, getAppDir(applicationPackage), - Optional.of(version)); + new com.yahoo.component.Version(modelVersion.toString()), + wantedNodeVespaVersion); - log.log(LogLevel.DEBUG, "Running createAndValidateModel for Vespa version " + version); + log.log(LogLevel.DEBUG, "Running createAndValidateModel for Vespa version " + modelVersion); ModelCreateResult result = modelFactory.createAndValidateModel(modelContext, params.ignoreValidationErrors()); validateModelHosts(context.getHostValidator(), applicationId, result.getModel()); - log.log(LogLevel.DEBUG, "Done building model for Vespa version " + version); - return new PreparedModelsBuilder.PreparedModelResult(version, result.getModel(), fileDistributionProvider, result.getConfigChangeActions()); + log.log(LogLevel.DEBUG, "Done building model for Vespa version " + modelVersion); + return new PreparedModelsBuilder.PreparedModelResult(modelVersion, result.getModel(), fileDistributionProvider, result.getConfigChangeActions()); } private Optional<File> getAppDir(ApplicationPackage applicationPackage) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java index a68ce0441ab..fcd10a35f39 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java @@ -1,6 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.session; +import com.yahoo.component.Version; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.ApplicationMetaData; @@ -83,7 +84,7 @@ public class LocalSession extends Session implements Comparable<LocalSession> { private Transaction setActive() { Transaction transaction = createSetStatusTransaction(Status.ACTIVATE); - transaction.add(applicationRepo.createPutApplicationTransaction(zooKeeperClient.readApplicationId(getTenant()), getSessionId()).operations()); + transaction.add(applicationRepo.createPutApplicationTransaction(zooKeeperClient.readApplicationId(), getSessionId()).operations()); return transaction; } @@ -153,6 +154,10 @@ public class LocalSession extends Session implements Comparable<LocalSession> { zooKeeperClient.writeApplicationId(applicationId); } + public void setVespaVersion(Version version) { + zooKeeperClient.writeVespaVersion(version); + } + public enum Mode { READ, WRITE } @@ -161,9 +166,9 @@ public class LocalSession extends Session implements Comparable<LocalSession> { return applicationPackage.getMetaData(); } - public ApplicationId getApplicationId() { - return zooKeeperClient.readApplicationId(getTenant()); - } + public ApplicationId getApplicationId() { return zooKeeperClient.readApplicationId(); } + + public Version getVespaVersion() { return zooKeeperClient.readVespaVersion(); } public ProvisionInfo getProvisionInfo() { return zooKeeperClient.getProvisionInfo(); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java index 03a7a3915d3..3890001518a 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java @@ -1,10 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.session; +import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Rotation; import com.yahoo.config.provision.TenantName; -import com.yahoo.config.provision.Version; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.vespa.config.server.TimeoutBudget; import com.yahoo.vespa.config.server.http.SessionHandler; @@ -29,7 +29,6 @@ public final class PrepareParams { static final String DRY_RUN_PARAM_NAME = "dryRun"; static final String VESPA_VERSION_PARAM_NAME = "vespaVersion"; static final String ROTATIONS_PARAM_NAME = "rotations"; - static final String DOCKER_VESPA_IMAGE_VERSION_PARAM_NAME = "dockerVespaImageVersion"; private final ApplicationId applicationId; private final TimeoutBudget timeoutBudget; @@ -37,18 +36,15 @@ public final class PrepareParams { private final boolean dryRun; private final Optional<Version> vespaVersion; private final Set<Rotation> rotations; - private final Optional<Version> dockerVespaImageVersion; private PrepareParams(ApplicationId applicationId, TimeoutBudget timeoutBudget, boolean ignoreValidationErrors, - boolean dryRun, Optional<Version> vespaVersion, Set<Rotation> rotations, - Optional<Version> dockerVespaImageVersion) { + boolean dryRun, Optional<Version> vespaVersion, Set<Rotation> rotations) { this.timeoutBudget = timeoutBudget; this.applicationId = applicationId; this.ignoreValidationErrors = ignoreValidationErrors; this.dryRun = dryRun; this.vespaVersion = vespaVersion; this.rotations = rotations; - this.dockerVespaImageVersion = dockerVespaImageVersion; } public static class Builder { @@ -58,7 +54,6 @@ public final class PrepareParams { private TimeoutBudget timeoutBudget = new TimeoutBudget(Clock.systemUTC(), Duration.ofSeconds(30)); private Optional<Version> vespaVersion = Optional.empty(); private Set<Rotation> rotations; - private Optional<Version> dockerVespaImageVersion = Optional.empty(); public Builder() { } @@ -102,18 +97,9 @@ public final class PrepareParams { return this; } - public Builder dockerVespaImageVersion(String dockerVespaImageVersion) { - Optional<Version> version = Optional.empty(); - if (dockerVespaImageVersion != null && !dockerVespaImageVersion.isEmpty()) { - version = Optional.of(Version.fromString(dockerVespaImageVersion)); - } - this.dockerVespaImageVersion = version; - return this; - } - public PrepareParams build() { - return new PrepareParams(applicationId, timeoutBudget, ignoreValidationErrors, dryRun, - vespaVersion, rotations, dockerVespaImageVersion); + return new PrepareParams(applicationId, timeoutBudget, ignoreValidationErrors, dryRun, + vespaVersion, rotations); } } @@ -125,7 +111,6 @@ public final class PrepareParams { .applicationId(createApplicationId(request, tenant)) .vespaVersion(request.getProperty(VESPA_VERSION_PARAM_NAME)) .rotations(request.getProperty(ROTATIONS_PARAM_NAME)) - .dockerVespaImageVersion(request.getProperty(DOCKER_VESPA_IMAGE_VERSION_PARAM_NAME)) .build(); } @@ -153,6 +138,7 @@ public final class PrepareParams { return applicationId; } + /** Returns the Vespa version the nodes running the prepared system should have, or empty to use the system version */ public Optional<Version> vespaVersion() { return vespaVersion; } public Set<Rotation> rotations() { return rotations; } @@ -169,15 +155,4 @@ public final class PrepareParams { return timeoutBudget; } - public Optional<Version> getVespaVersion() { - return vespaVersion; - } - - public Set<Rotation> getRotations() { - return rotations; - } - - public Optional<Version> getDockerVespaImageVersion() { - return dockerVespaImageVersion; - } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java index ca4fa21f804..72dc4cbb963 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java @@ -49,7 +49,8 @@ public class RemoteSession extends Session { } private ApplicationSet loadApplication() { - return ApplicationSet.fromList(applicationLoader.buildModels(zooKeeperClient.readApplicationId(getTenant()), + return ApplicationSet.fromList(applicationLoader.buildModels(zooKeeperClient.readApplicationId(), + zooKeeperClient.readVespaVersion(), zooKeeperClient.loadApplicationPackage())); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java index 652f27ddaf4..461019439e0 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java @@ -115,13 +115,13 @@ public class SessionFactoryImpl implements SessionFactory, LocalSessionLoader { TimeoutBudget timeoutBudget) { File existingApp = getSessionAppDir(existingSession.getSessionId()); ApplicationMetaData metaData = FilesApplicationPackage.readMetaData(existingApp); - final ApplicationId existingSessionId = existingSession.getApplicationId(); + ApplicationId existingApplicationId = existingSession.getApplicationId(); - - final long liveApp = getLiveApp(existingSessionId); - logger.log(LogLevel.DEBUG, "Create from existing application id " + existingSessionId + ", live app for it is " + liveApp); + long liveApp = getLiveApp(existingApplicationId); + logger.log(LogLevel.DEBUG, "Create from existing application id " + existingApplicationId + ", live app for it is " + liveApp); LocalSession session = create(existingApp, metaData.getApplicationName(), liveApp, timeoutBudget); - session.setApplicationId(existingSessionId); + session.setApplicationId(existingApplicationId); + session.setVespaVersion(existingSession.getVespaVersion()); return session; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java index d9f486a5604..cd10b031a93 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.config.server.session; import com.google.common.collect.ImmutableList; import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.component.Vtag; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.application.api.FileRegistry; @@ -91,7 +92,7 @@ public class SessionPreparer { try { prep.buildModels(); prep.makeResult(); - if (!params.isDryRun()) { + if ( ! params.isDryRun()) { prep.writeStateZK(); prep.writeRotZK(); prep.distribute(); @@ -112,6 +113,10 @@ public class SessionPreparer { final Optional<ApplicationSet> currentActiveApplicationSet; final Path tenantPath; final ApplicationId applicationId; + + /** The version of Vespa the application to be prepared specifies for its nodes */ + final com.yahoo.component.Version vespaVersion; + final Rotations rotations; final Set<Rotation> rotationsSet; final ModelContext.Properties properties; @@ -131,6 +136,7 @@ public class SessionPreparer { this.tenantPath = tenantPath; this.applicationId = params.getApplicationId(); + this.vespaVersion = params.vespaVersion().orElse(Vtag.currentVersion); this.rotations = new Rotations(curator, tenantPath); this.rotationsSet = getRotations(params.rotations()); this.properties = new ModelContextImpl.Properties(params.getApplicationId(), @@ -168,7 +174,7 @@ public class SessionPreparer { } void buildModels() { - this.modelResultList = preparedModelsBuilder.buildModels(applicationId, applicationPackage); + this.modelResultList = preparedModelsBuilder.buildModels(applicationId, vespaVersion, applicationPackage); checkTimeout("build models"); } @@ -179,8 +185,13 @@ public class SessionPreparer { void writeStateZK() { log.log(LogLevel.DEBUG, "Writing application package state to zookeeper"); - writeStateToZooKeeper(context.getSessionZooKeeperClient(), applicationPackage, params, logger, - prepareResult.getFileRegistries(), prepareResult.getProvisionInfos()); + writeStateToZooKeeper(context.getSessionZooKeeperClient(), + applicationPackage, + applicationId, + vespaVersion, + logger, + prepareResult.getFileRegistries(), + prepareResult.getProvisionInfos()); checkTimeout("write state to zookeeper"); } @@ -217,14 +228,16 @@ public class SessionPreparer { private void writeStateToZooKeeper(SessionZooKeeperClient zooKeeperClient, ApplicationPackage applicationPackage, - PrepareParams prepareParams, + ApplicationId applicationId, + com.yahoo.component.Version vespaVersion, DeployLogger deployLogger, Map<Version, FileRegistry> fileRegistryMap, Map<Version, ProvisionInfo> provisionInfoMap) { ZooKeeperDeployer zkDeployer = zooKeeperClient.createDeployer(deployLogger); try { zkDeployer.deploy(applicationPackage, fileRegistryMap, provisionInfoMap); - zooKeeperClient.writeApplicationId(prepareParams.getApplicationId()); + zooKeeperClient.writeApplicationId(applicationId); + zooKeeperClient.writeVespaVersion(vespaVersion); } catch (RuntimeException | IOException e) { zkDeployer.cleanup(); throw new RuntimeException("Error preparing session", e); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java index ecb5d0980bb..9fbd8d539fb 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java @@ -1,6 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.session; +import com.yahoo.component.Version; +import com.yahoo.component.Vtag; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.provision.ProvisionInfo; @@ -34,6 +36,7 @@ public class SessionZooKeeperClient { private static final java.util.logging.Logger log = java.util.logging.Logger.getLogger(SessionZooKeeperClient.class.getName()); static final String APPLICATION_ID_PATH = "applicationId"; + static final String VERSION_PATH = "version"; static final String CREATE_TIME_PATH = "createTime"; private final Curator curator; private final ConfigCurator configCurator; @@ -137,38 +140,36 @@ public class SessionZooKeeperClient { return cacheLoader.loadCache(); } + private String applicationIdPath() { + return rootPath.append(APPLICATION_ID_PATH).getAbsolute(); + } + public void writeApplicationId(ApplicationId id) { - String path = getApplicationIdPath(); - try { - configCurator.putData(path, id.serializedForm()); - } catch (RuntimeException e) { - throw new RuntimeException("Unable to write application id '" + id + "' to '" + path + "'", e); - } + configCurator.putData(applicationIdPath(), id.serializedForm()); } - private String getApplicationIdPath() { - return rootPath.append(APPLICATION_ID_PATH).getAbsolute(); + public ApplicationId readApplicationId() { + if ( ! configCurator.exists(applicationIdPath())) return ApplicationId.defaultId(); + return ApplicationId.fromSerializedForm(configCurator.getData(applicationIdPath())); } - public ApplicationId readApplicationId(TenantName tenant) { - String path = getApplicationIdPath(); - try { - // Fallback for cases where id never existed. - if ( ! configCurator.exists(path)) { - // TODO: DEBUG LOG - log.log(LogLevel.INFO, "Unable to locate application id at '" + path + "', returning default"); - return ApplicationId.defaultId(); - } - return ApplicationId.fromSerializedForm(configCurator.getData(path)); - } catch (RuntimeException e) { - throw new RuntimeException("Unable to read application id from '" + path + "'", e); - } + private String versionPath() { + return rootPath.append(VERSION_PATH).getAbsolute(); + } + + public void writeVespaVersion(Version version) { + configCurator.putData(versionPath(), version.toString()); + } + + public Version readVespaVersion() { + if ( ! configCurator.exists(versionPath())) return Vtag.currentVersion; + return new Version(configCurator.getData(versionPath())); } // in seconds public long readCreateTime() { String path = getCreateTimePath(); - if (!configCurator.exists(path)) return 0l; + if ( ! configCurator.exists(path)) return 0L; return Long.parseLong(configCurator.getData(path)); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java index 0352f2c9f3e..74f030919ec 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java @@ -1,6 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server; +import com.yahoo.component.Version; import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.config.model.application.provider.MockFileRegistry; @@ -48,7 +49,8 @@ public class ModelContextImplTest { Zone.defaultZone(), rotations), Optional.empty(), - Optional.empty()); + new Version(6), + new Version(6)); assertTrue(context.applicationPackage() instanceof MockApplicationPackage); assertFalse(context.hostProvisioner().isPresent()); assertFalse(context.permanentApplicationPackage().isPresent()); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java index 07fb3cd51c7..8e20f190c5b 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java @@ -115,14 +115,14 @@ public class DeployTester { /** * Do the initial "deploy" with the existing API-less code as the deploy API doesn't support first deploys yet. */ - public ApplicationId deployApp(String appName, Optional<String> dockerVespaImageVersion) { + public ApplicationId deployApp(String appName, Optional<String> vespaVersion) { final Tenant tenant = tenant(); LocalSession session = tenant.getSessionFactory().createSession(testApp, appName, new TimeoutBudget(Clock.systemUTC(), Duration.ofSeconds(60))); ApplicationId id = ApplicationId.from(tenant.getName(), ApplicationName.from(appName), InstanceName.defaultName()); PrepareParams.Builder paramsBuilder = new PrepareParams.Builder() .applicationId(id); - if (dockerVespaImageVersion.isPresent()) - paramsBuilder.dockerVespaImageVersion(dockerVespaImageVersion.get()); + if (vespaVersion.isPresent()) + paramsBuilder.vespaVersion(vespaVersion.get()); session.prepare(new SilentDeployLogger(), paramsBuilder.build(), Optional.empty(), diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/PrepareParamsTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/PrepareParamsTest.java index d91fff41f2e..b8ea8b5a4f9 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/PrepareParamsTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/PrepareParamsTest.java @@ -34,9 +34,9 @@ public class PrepareParamsTest { assertThat(prepareParams.getApplicationId(), is(ApplicationId.defaultId())); assertFalse(prepareParams.isDryRun()); assertFalse(prepareParams.ignoreValidationErrors()); - assertThat(prepareParams.getVespaVersion(), is(Optional.<String>empty())); + assertThat(prepareParams.vespaVersion(), is(Optional.<String>empty())); assertTrue(prepareParams.getTimeoutBudget().hasTimeLeft()); - assertThat(prepareParams.getRotations().size(), is(0)); + assertThat(prepareParams.rotations().size(), is(0)); } @@ -46,8 +46,7 @@ public class PrepareParamsTest { PrepareParams.DRY_RUN_PARAM_NAME + "=true&" + PrepareParams.IGNORE_VALIDATION_PARAM_NAME + "=false&" + PrepareParams.APPLICATION_NAME_PARAM_NAME + "=baz&" + - PrepareParams.VESPA_VERSION_PARAM_NAME + "=" + vespaVersion + "&" + - PrepareParams.DOCKER_VESPA_IMAGE_VERSION_PARAM_NAME+ "=" + vespaVersion; + PrepareParams.VESPA_VERSION_PARAM_NAME + "=" + vespaVersion; @Test public void testCorrectParsingWithRotation() { @@ -59,12 +58,11 @@ public class PrepareParamsTest { assertTrue(prepareParams.isDryRun()); assertFalse(prepareParams.ignoreValidationErrors()); final Version expectedVersion = Version.fromString(vespaVersion); - assertThat(prepareParams.getVespaVersion().get(), is(expectedVersion)); + assertThat(prepareParams.vespaVersion().get(), is(expectedVersion)); assertTrue(prepareParams.getTimeoutBudget().hasTimeLeft()); - final Set<Rotation> rotations = prepareParams.getRotations(); + final Set<Rotation> rotations = prepareParams.rotations(); assertThat(rotations.size(), is(1)); assertThat(rotations, contains(equalTo(new Rotation(rotation)))); - assertThat(prepareParams.getDockerVespaImageVersion().get(), is(expectedVersion)); } @Test @@ -74,7 +72,7 @@ public class PrepareParamsTest { PrepareParams prepareParams = createParams(request + "&" + PrepareParams.ROTATIONS_PARAM_NAME + "=" + twoRotations, TenantName.from("foo")); - final Set<Rotation> rotations = prepareParams.getRotations(); + final Set<Rotation> rotations = prepareParams.rotations(); assertThat(rotations, containsInAnyOrder(new Rotation(rotation), new Rotation(rotationTwo))); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java index ac25cba3b47..1e61e6e9e08 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java @@ -186,7 +186,7 @@ public class SessionPreparerTest extends TestWithCurator { preparer.prepare(getContext(getApplicationPackage(testApp)), getLogger(), params, Optional.empty(), tenantPath); SessionZooKeeperClient zkc = new SessionZooKeeperClient(curator, appPath); assertTrue(configCurator.exists(appPath.append(SessionZooKeeperClient.APPLICATION_ID_PATH).getAbsolute())); - assertThat(zkc.readApplicationId(tenant), is(origId)); + assertThat(zkc.readApplicationId(), is(origId)); } @Test diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClientTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClientTest.java index 4508d8c234f..d4f3665ac6f 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClientTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClientTest.java @@ -76,7 +76,7 @@ public class SessionZooKeeperClientTest extends TestWithCurator { @Test public void require_that_default_name_is_returned_if_node_does_not_exist() { - assertThat(createSessionZKClient("3").readApplicationId(TenantName.defaultName()).application().value(), is("default")); + assertThat(createSessionZKClient("3").readApplicationId().application().value(), is("default")); } @Test @@ -101,7 +101,7 @@ public class SessionZooKeeperClientTest extends TestWithCurator { SessionZooKeeperClient zkc = createSessionZKClient(sessionId); String path = "/" + sessionId + "/" + SessionZooKeeperClient.APPLICATION_ID_PATH; configCurator.putData(path, idString); - ApplicationId zkId = zkc.readApplicationId(TenantName.defaultName()); + ApplicationId zkId = zkc.readApplicationId(); assertThat(zkId.serializedForm(), is(expectedIdString)); } |