diff options
386 files changed, 3262 insertions, 1717 deletions
diff --git a/.travis.yml b/.travis.yml index 29aa44a4e9c..1144d01e5c6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,7 +22,6 @@ before_cache: - sudo rm -rf $HOME/.m2/repository/com/yahoo - du --summarize --human-readable $HOME/.m2/repository - du --summarize --human-readable $HOME/.ccache - - ccache --show-stats install: true diff --git a/README.md b/README.md index f934f16f011..866eb39bede 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ Code licensed under the Apache 2.0 license. See [LICENSE](LICENSE) for terms. ## Get started developing ### Setup build environment -C++ building is supported on CentOS 7. +C++ building is supported on CentOS 7. The Java source can be built on any platform having Java 8 and Maven installed. We recommend using the following environment: [Create C++ dev environment on CentOS using VirtualBox and Vagrant](vagrant/README.md). You can also setup CentOS 7 natively and install the following build dependencies: @@ -25,10 +25,9 @@ You can also setup CentOS 7 natively and install the following build dependencie yum-builddep -y <vespa-source>/dist/vespa.spec ### Build Java modules -Java modules can be built on any environment having Java 8 and Maven: sh bootstrap.sh - mvn install + mvn -T <num-threads> install ### Build C++ modules Replace `<build-dir>` with the name of the directory in which you'd like to build Vespa. diff --git a/application/src/test/resources/configdefinitions/mock-application.def b/application/src/test/resources/configdefinitions/mock-application.def index 41b50df1125..fa3a48bae96 100644 --- a/application/src/test/resources/configdefinitions/mock-application.def +++ b/application/src/test/resources/configdefinitions/mock-application.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=1 namespace=application mystruct.id string diff --git a/bundle-plugin-test/src/main/resources/configdefinitions/test.def b/bundle-plugin-test/src/main/resources/configdefinitions/test.def index 5d09d00b992..b4ba9ec518a 100644 --- a/bundle-plugin-test/src/main/resources/configdefinitions/test.def +++ b/bundle-plugin-test/src/main/resources/configdefinitions/test.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=1 namespace=example exampleKey string default="exampleValue" diff --git a/chain/src/main/resources/configdefinitions/chains.def b/chain/src/main/resources/configdefinitions/chains.def index 0ad72727438..8520a8d6332 100644 --- a/chain/src/main/resources/configdefinitions/chains.def +++ b/chain/src/main/resources/configdefinitions/chains.def @@ -1,6 +1,5 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. # Chains configuration -version=13 namespace=container.core components[].id string diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/ApplicationPackageXmlFilesValidator.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/ApplicationPackageXmlFilesValidator.java index 29002d8a685..8e9c5c0b509 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/ApplicationPackageXmlFilesValidator.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/ApplicationPackageXmlFilesValidator.java @@ -11,7 +11,6 @@ import java.io.File; import java.io.FilenameFilter; import java.io.IOException; import java.util.List; -import java.util.Optional; /** * Validation of xml files in application package against RELAX NG schemas. @@ -22,31 +21,26 @@ public class ApplicationPackageXmlFilesValidator { private final AppSubDirs appDirs; - /** The Vespa version this package should be validated against */ - private final Version vespaVersion; + private final SchemaValidators validators; private static final FilenameFilter xmlFilter = (dir, name) -> name.endsWith(".xml"); public ApplicationPackageXmlFilesValidator(AppSubDirs appDirs, Version vespaVersion) { this.appDirs = appDirs; - this.vespaVersion = vespaVersion; + this.validators = new SchemaValidators(vespaVersion, new BaseDeployLogger()); } - public static ApplicationPackageXmlFilesValidator createDefaultXMLValidator(File appDir, Version vespaVersion) { - return new ApplicationPackageXmlFilesValidator(new AppSubDirs(appDir), vespaVersion); - } - - public static ApplicationPackageXmlFilesValidator createTestXmlValidator(File appDir, Version vespaVersion) { + public static ApplicationPackageXmlFilesValidator create(File appDir, Version vespaVersion) { return new ApplicationPackageXmlFilesValidator(new AppSubDirs(appDir), vespaVersion); } @SuppressWarnings("deprecation") public void checkApplication() throws IOException { - validate(SchemaValidator.servicesXmlSchemaName, servicesFileName()); - validateOptional(SchemaValidator.hostsXmlSchemaName, FilesApplicationPackage.HOSTS); - validateOptional(SchemaValidator.deploymentXmlSchemaName, FilesApplicationPackage.DEPLOYMENT_FILE.getName()); - validateOptional(SchemaValidator.validationOverridesXmlSchemaName, FilesApplicationPackage.VALIDATION_OVERRIDES.getName()); + validate(validators.servicesXmlValidator(), servicesFileName()); + validateOptional(validators.hostsXmlValidator(), FilesApplicationPackage.HOSTS); + validateOptional(validators.deploymentXmlValidator(), FilesApplicationPackage.DEPLOYMENT_FILE.getName()); + validateOptional(validators.validationOverridesXmlValidator(), FilesApplicationPackage.VALIDATION_OVERRIDES.getName()); if (appDirs.searchdefinitions().exists()) { if (FilesApplicationPackage.getSearchDefinitionFiles(appDirs.root()).isEmpty()) { @@ -55,26 +49,26 @@ public class ApplicationPackageXmlFilesValidator { } } - validate(appDirs.routingtables, "routing-standalone.rnc"); + validateRouting(appDirs.routingtables); } // For testing - public static void checkIncludedDirs(ApplicationPackage app, Version vespaVersion) throws IOException { + public void checkIncludedDirs(ApplicationPackage app) throws IOException { for (String includedDir : app.getUserIncludeDirs()) { List<NamedReader> includedFiles = app.getFiles(Path.fromString(includedDir), ".xml", true); for (NamedReader file : includedFiles) { - createSchemaValidator("container-include.rnc", vespaVersion).validate(file); + validators.containerIncludeXmlValidator().validate(file); } } } - private void validateOptional(String schema, String file) throws IOException { + private void validateOptional(SchemaValidator validator, String file) throws IOException { if ( ! appDirs.file(file).exists()) return; - validate(schema, file); + validate(validator, file); } - private void validate(String schema, String file) throws IOException { - createSchemaValidator(schema, vespaVersion).validate(appDirs.file(file)); + private void validate(SchemaValidator validator, String filename) throws IOException { + validator.validate(appDirs.file(filename)); } @SuppressWarnings("deprecation") @@ -87,26 +81,22 @@ public class ApplicationPackageXmlFilesValidator { return servicesFile; } - private void validate(Tuple2<File, String> directory, String schemaFile) throws IOException { + private void validateRouting(Tuple2<File, String> directory) throws IOException { if ( ! directory.first.isDirectory()) return; - validate(directory, createSchemaValidator(schemaFile, vespaVersion)); + validateRouting(validators.routingStandaloneXmlValidator(), directory); } - private void validate(Tuple2<File, String> directory, SchemaValidator validator) throws IOException { + private void validateRouting(SchemaValidator validator, Tuple2<File, String> directory) throws IOException { File dir = directory.first; if ( ! dir.isDirectory()) return; String directoryName = directory.second; for (File f : dir.listFiles(xmlFilter)) { if (f.isDirectory()) - validate(new Tuple2<>(f, directoryName + File.separator + f.getName()),validator); + validateRouting(validator, new Tuple2<>(f, directoryName + File.separator + f.getName())); else validator.validate(f, directoryName + File.separator + f.getName()); } } - private static SchemaValidator createSchemaValidator(String schemaFile, Version vespaVersion) { - return new SchemaValidator(schemaFile, new BaseDeployLogger(), vespaVersion); - } - } diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/DeployData.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/DeployData.java index d9a784a2dc5..f48dffecb27 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/DeployData.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/DeployData.java @@ -5,7 +5,6 @@ package com.yahoo.config.model.application.provider; * A class for holding values generated or computed during deployment * * @author hmusum - * @since 5.1.11 */ public class DeployData { @@ -57,4 +56,5 @@ public class DeployData { public String getApplicationName() { return applicationName; } + } diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java index 8b8921f50a1..f1565d1fa4b 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java @@ -631,10 +631,11 @@ public class FilesApplicationPackage implements ApplicationPackage { @Override public void validateXML(Optional<Version> vespaVersion) throws IOException { - com.yahoo.component.Version modelVersion = vespaVersion.map(v -> new com.yahoo.component.Version(vespaVersion.toString())).orElse(Vtag.currentVersion); - ApplicationPackageXmlFilesValidator xmlFilesValidator = ApplicationPackageXmlFilesValidator.createDefaultXMLValidator(appDir, modelVersion); - xmlFilesValidator.checkApplication(); - ApplicationPackageXmlFilesValidator.checkIncludedDirs(this, modelVersion); + com.yahoo.component.Version modelVersion = + vespaVersion.map(v -> new com.yahoo.component.Version(vespaVersion.toString())).orElse(Vtag.currentVersion); + ApplicationPackageXmlFilesValidator validator = ApplicationPackageXmlFilesValidator.create(appDir, modelVersion); + validator.checkApplication(); + validator.checkIncludedDirs(this); } @Override diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/SchemaValidator.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/SchemaValidator.java index 70da2f2e92e..d0cca38b375 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/SchemaValidator.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/SchemaValidator.java @@ -6,214 +6,40 @@ import com.thaiopensource.util.PropertyMapBuilder; import com.thaiopensource.validate.ValidateProperty; import com.thaiopensource.validate.ValidationDriver; import com.thaiopensource.validate.rng.CompactSchemaReader; -import com.yahoo.component.Version; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.io.IOUtils; import com.yahoo.io.reader.NamedReader; -import com.yahoo.log.LogLevel; -import static com.yahoo.vespa.defaults.Defaults.getDefaults; import com.yahoo.yolean.Exceptions; -import org.osgi.framework.Bundle; -import org.osgi.framework.FrameworkUtil; import org.xml.sax.ErrorHandler; import org.xml.sax.InputSource; import org.xml.sax.SAXException; import org.xml.sax.SAXParseException; + import java.io.File; import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; import java.io.Reader; -import java.net.JarURLConnection; -import java.net.URL; -import java.util.Enumeration; -import java.util.jar.JarEntry; -import java.util.jar.JarFile; import java.util.logging.Level; -import java.util.logging.Logger; /** - * Validates xml files against one schema. + * Validates xml files against a schema. * * @author tonytv */ public class SchemaValidator { - public static final String schemaDirBase = System.getProperty("java.io.tmpdir", File.separator + "tmp" + File.separator + "vespa"); - static final String servicesXmlSchemaName = "services.rnc"; - static final String hostsXmlSchemaName = "hosts.rnc"; - static final String deploymentXmlSchemaName = "deployment.rnc"; - static final String validationOverridesXmlSchemaName = "validation-overrides.rnc"; private final CustomErrorHandler errorHandler = new CustomErrorHandler(); private final ValidationDriver driver; - private DeployLogger deployLogger; - private static final Logger log = Logger.getLogger(SchemaValidator.class.getName()); - - /** - * Initializes the validator by using the given file as schema file - * @param schema a schema file in RNC format - * @param logger a logger - * @param vespaVersion the version of Vespa we should validate against - */ - public SchemaValidator(String schema, DeployLogger logger, Version vespaVersion) { - this.deployLogger = logger; - driver = new ValidationDriver(PropertyMap.EMPTY, instanceProperties(), CompactSchemaReader.getInstance()); - File schemaDir = new File(schemaDirBase); - try { - schemaDir = saveSchemasFromJar(new File(SchemaValidator.schemaDirBase), vespaVersion); - } catch (IOException e) { - throw new RuntimeException(e); - } - loadSchema(new File(schemaDir + File.separator + "schema" + File.separator + schema)); - IOUtils.recursiveDeleteDir(schemaDir); - } + private final DeployLogger deployLogger; /** * Initializes the validator by using the given file as schema file - * @param schema a schema file in RNC format - * @param vespaVersion the version we should validate against - * @throws IOException if it is not possible to read schema files - */ - public SchemaValidator(String schema, Version vespaVersion) throws IOException { - this(schema, new BaseDeployLogger(), vespaVersion); - } - - /** - * Create a validator for services.xml for tests - * @throws IOException if it is not possible to read schema files - */ - public static SchemaValidator createTestValidatorServices(Version vespaVersion) throws IOException { - return new SchemaValidator(servicesXmlSchemaName, vespaVersion); - } - - /** - * Create a validator for hosts.xml for tests - * @throws IOException if it is not possible to read schema files - */ - public static SchemaValidator createTestValidatorHosts(Version vespaVersion) throws IOException { - return new SchemaValidator(hostsXmlSchemaName, vespaVersion); - } - - /** - * Create a validator for deployment.xml for tests * + * @param schemaFile schema file * @throws IOException if it is not possible to read schema files */ - public static SchemaValidator createTestValidatorDeployment(Version vespaVersion) throws IOException { - return new SchemaValidator(deploymentXmlSchemaName, vespaVersion); - } - - private class CustomErrorHandler implements ErrorHandler { - volatile String fileName; - - public void warning(SAXParseException e) throws SAXException { - deployLogger.log(Level.WARNING, message(e)); - } - - public void error(SAXParseException e) throws SAXException { - throw new IllegalArgumentException(message(e)); - } - - public void fatalError(SAXParseException e) throws SAXException { - throw new IllegalArgumentException(message(e)); - } - - private String message(SAXParseException e) { - return "XML error in " + fileName + ": " + - Exceptions.toMessageString(e) - + " [" + e.getLineNumber() + ":" + e.getColumnNumber() + "]"; - } - } - - /** - * Look for the schema files that should be in vespa-model.jar and saves them on temp dir. - * - * @return the directory the schema files are stored in - * @throws IOException if it is not possible to read schema files - */ - private File saveSchemasFromJar(File tmpBase, Version vespaVersion) throws IOException { - final Class<? extends SchemaValidator> schemaValidatorClass = this.getClass(); - final ClassLoader classLoader = schemaValidatorClass.getClassLoader(); - Enumeration<URL> uris = classLoader.getResources("schema"); - if (uris==null) return null; - File tmpDir = java.nio.file.Files.createTempDirectory(tmpBase.toPath(), "vespa").toFile(); - log.log(LogLevel.DEBUG, "Saving schemas to " + tmpDir); - while(uris.hasMoreElements()) { - URL u = uris.nextElement(); - log.log(LogLevel.DEBUG, "uri for resource 'schema'=" + u.toString()); - if ("jar".equals(u.getProtocol())) { - JarURLConnection jarConnection = (JarURLConnection) u.openConnection(); - JarFile jarFile = jarConnection.getJarFile(); - for (Enumeration<JarEntry> entries = jarFile.entries(); - entries.hasMoreElements();) { - - JarEntry je=entries.nextElement(); - if (je.getName().startsWith("schema/") && je.getName().endsWith(".rnc")) { - writeContentsToFile(tmpDir, je.getName(), jarFile.getInputStream(je)); - } - } - jarFile.close(); - } else if ("bundle".equals(u.getProtocol())) { - Bundle bundle = FrameworkUtil.getBundle(schemaValidatorClass); - log.log(LogLevel.DEBUG, classLoader.toString()); - log.log(LogLevel.DEBUG, "bundle=" + bundle); - // TODO: Hack to handle cases where bundle=null - if (bundle == null) { - File schemaPath; - if (vespaVersion.getMajor() == 5) { - schemaPath = new File(getDefaults().underVespaHome("share/vespa/schema/version/5.x/schema/")); - } else { - schemaPath = new File(getDefaults().underVespaHome("share/vespa/schema/")); - } - log.log(LogLevel.DEBUG, "Using schemas found in " + schemaPath); - copySchemas(schemaPath, tmpDir); - } else { - log.log(LogLevel.DEBUG, String.format("Saving schemas for model bundle %s:%s", bundle.getSymbolicName(), bundle - .getVersion())); - for (Enumeration<URL> entries = bundle.findEntries("schema", "*.rnc", true); - entries.hasMoreElements(); ) { - - URL url = entries.nextElement(); - writeContentsToFile(tmpDir, url.getFile(), url.openStream()); - } - } - } else if ("file".equals(u.getProtocol())) { - File schemaPath = new File(u.getPath()); - copySchemas(schemaPath, tmpDir); - } - } - return tmpDir; - } - - private static void copySchemas(File from, File to) throws IOException { - // TODO: only copy .rnc files. - if (! from.exists()) throw new IOException("Could not find schema source directory '" + from + "'"); - if (! from.isDirectory()) throw new IOException("Schema source '" + from + "' is not a directory"); - File sourceFile = new File(from, servicesXmlSchemaName); - if (! sourceFile.exists()) throw new IOException("Schema source file '" + sourceFile + "' not found"); - IOUtils.copyDirectoryInto(from, to); - } - - private static void writeContentsToFile(File outDir, String outFile, InputStream inputStream) throws IOException { - String contents = IOUtils.readAll(new InputStreamReader(inputStream)); - File out = new File(outDir, outFile); - IOUtils.writeFile(out, contents, false); - } - - private void loadSchema(File schemaFile) { - try { - driver.loadSchema(ValidationDriver.fileInputSource(schemaFile)); - } catch (SAXException e) { - throw new RuntimeException("Invalid schema '" + schemaFile + "'", e); - } catch (IOException e) { - throw new RuntimeException("IO error reading schema '" + schemaFile + "'", e); - } - } - - private PropertyMap instanceProperties() { - PropertyMapBuilder builder = new PropertyMapBuilder(); - builder.put(ValidateProperty.ERROR_HANDLER, errorHandler); - return builder.toPropertyMap(); + SchemaValidator(File schemaFile, DeployLogger deployLogger) throws IOException, SAXException { + this.deployLogger = deployLogger; + this.driver = new ValidationDriver(PropertyMap.EMPTY, instanceProperties(), CompactSchemaReader.getInstance()); + driver.loadSchema(ValidationDriver.fileInputSource(schemaFile)); } public void validate(File file) throws IOException { @@ -246,4 +72,33 @@ public class SchemaValidator { "XML error in " + (fileName == null ? " input" : fileName) + ": " + Exceptions.toMessageString(e)); } } + + private PropertyMap instanceProperties() { + PropertyMapBuilder builder = new PropertyMapBuilder(); + builder.put(ValidateProperty.ERROR_HANDLER, errorHandler); + return builder.toPropertyMap(); + } + + private class CustomErrorHandler implements ErrorHandler { + volatile String fileName; + + public void warning(SAXParseException e) throws SAXException { + deployLogger.log(Level.WARNING, message(e)); + } + + public void error(SAXParseException e) throws SAXException { + throw new IllegalArgumentException(message(e)); + } + + public void fatalError(SAXParseException e) throws SAXException { + throw new IllegalArgumentException(message(e)); + } + + private String message(SAXParseException e) { + return "XML error in " + fileName + ": " + + Exceptions.toMessageString(e) + + " [" + e.getLineNumber() + ":" + e.getColumnNumber() + "]"; + } + } + } diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/SchemaValidators.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/SchemaValidators.java new file mode 100644 index 00000000000..e7e65751ee8 --- /dev/null +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/SchemaValidators.java @@ -0,0 +1,195 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.application.provider; + +import com.yahoo.component.Version; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.io.IOUtils; +import com.yahoo.log.LogLevel; +import org.osgi.framework.Bundle; +import org.osgi.framework.FrameworkUtil; +import org.xml.sax.SAXException; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.JarURLConnection; +import java.net.URL; +import java.util.Enumeration; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; +import java.util.logging.Logger; + +import static com.yahoo.vespa.defaults.Defaults.getDefaults; + +/** + * Wrapper class for schema validators for application package xml files + * + * @author hmusum + */ +public class SchemaValidators { + + private static final String schemaDirBase = System.getProperty("java.io.tmpdir", File.separator + "tmp" + File.separator + "vespa"); + private static final Logger log = Logger.getLogger(SchemaValidators.class.getName()); + + private static final String servicesXmlSchemaName = "services.rnc"; + private static final String hostsXmlSchemaName = "hosts.rnc"; + private static final String deploymentXmlSchemaName = "deployment.rnc"; + private static final String validationOverridesXmlSchemaName = "validation-overrides.rnc"; + private static final String containerIncludeXmlSchemaName = "container-include.rnc"; + private static final String routingStandaloneXmlSchemaName = "routing-standalone.rnc"; + + + private final DeployLogger deployLogger; + + private final SchemaValidator servicesXmlValidator; + private final SchemaValidator hostsXmlValidator; + private final SchemaValidator deploymentXmlValidator; + private final SchemaValidator validationOverridesXmlValidator; + private final SchemaValidator containerIncludeXmlValidator; + private final SchemaValidator routingStandaloneXmlValidator; + + /** + * Initializes the validator by using the given file as schema file + * + * @param vespaVersion the version of Vespa we should validate against + */ + public SchemaValidators(Version vespaVersion, DeployLogger logger) { + this.deployLogger = logger; + File schemaDir; + try { + schemaDir = saveSchemasFromJar(new File(SchemaValidators.schemaDirBase), vespaVersion); + } catch (IOException e) { + throw new RuntimeException(e); + } + + servicesXmlValidator = createValidator(schemaDir, servicesXmlSchemaName); + hostsXmlValidator = createValidator(schemaDir, hostsXmlSchemaName); + deploymentXmlValidator = createValidator(schemaDir, deploymentXmlSchemaName); + validationOverridesXmlValidator = createValidator(schemaDir, validationOverridesXmlSchemaName); + containerIncludeXmlValidator = createValidator(schemaDir, containerIncludeXmlSchemaName); + routingStandaloneXmlValidator = createValidator(schemaDir, routingStandaloneXmlSchemaName); + IOUtils.recursiveDeleteDir(schemaDir); + } + + /** + * Initializes the validator by using the given file as schema file + * + * @param vespaVersion the version of Vespa we should validate against + */ + public SchemaValidators(Version vespaVersion) { + this(vespaVersion, new BaseDeployLogger()); + } + + public SchemaValidator servicesXmlValidator() throws IOException { + return servicesXmlValidator; + } + + public SchemaValidator hostsXmlValidator() throws IOException { + return hostsXmlValidator; + } + + public SchemaValidator deploymentXmlValidator() throws IOException { + return deploymentXmlValidator; + } + + SchemaValidator validationOverridesXmlValidator() throws IOException { + return validationOverridesXmlValidator; + } + + SchemaValidator containerIncludeXmlValidator() { + return containerIncludeXmlValidator; + } + + public SchemaValidator routingStandaloneXmlValidator() { + return routingStandaloneXmlValidator; + } + + /** + * Look for the schema files that should be in vespa-model.jar and saves them on temp dir. + * + * @return the directory the schema files are stored in + * @throws IOException if it is not possible to read schema files + */ + File saveSchemasFromJar(File tmpBase, Version vespaVersion) throws IOException { + final Class<? extends SchemaValidators> schemaValidatorClass = this.getClass(); + final ClassLoader classLoader = schemaValidatorClass.getClassLoader(); + Enumeration<URL> uris = classLoader.getResources("schema"); + if (uris == null) return null; + File tmpDir = java.nio.file.Files.createTempDirectory(tmpBase.toPath(), "vespa").toFile(); + log.log(LogLevel.DEBUG, "Will save all XML schemas to " + tmpDir); + while (uris.hasMoreElements()) { + URL u = uris.nextElement(); + log.log(LogLevel.DEBUG, "uri for resource 'schema'=" + u.toString()); + if ("jar".equals(u.getProtocol())) { + JarURLConnection jarConnection = (JarURLConnection) u.openConnection(); + JarFile jarFile = jarConnection.getJarFile(); + for (Enumeration<JarEntry> entries = jarFile.entries(); + entries.hasMoreElements(); ) { + + JarEntry je = entries.nextElement(); + if (je.getName().startsWith("schema/") && je.getName().endsWith(".rnc")) { + writeContentsToFile(tmpDir, je.getName(), jarFile.getInputStream(je)); + } + } + jarFile.close(); + } else if ("bundle".equals(u.getProtocol())) { + Bundle bundle = FrameworkUtil.getBundle(schemaValidatorClass); + log.log(LogLevel.DEBUG, classLoader.toString()); + log.log(LogLevel.DEBUG, "bundle=" + bundle); + // TODO: Hack to handle cases where bundle=null + if (bundle == null) { + File schemaPath; + if (vespaVersion.getMajor() == 5) { + schemaPath = new File(getDefaults().underVespaHome("share/vespa/schema/version/5.x/schema/")); + } else { + schemaPath = new File(getDefaults().underVespaHome("share/vespa/schema/")); + } + log.log(LogLevel.DEBUG, "Using schemas found in " + schemaPath); + copySchemas(schemaPath, tmpDir); + } else { + log.log(LogLevel.DEBUG, String.format("Saving schemas for model bundle %s:%s", bundle.getSymbolicName(), bundle + .getVersion())); + for (Enumeration<URL> entries = bundle.findEntries("schema", "*.rnc", true); + entries.hasMoreElements(); ) { + + URL url = entries.nextElement(); + writeContentsToFile(tmpDir, url.getFile(), url.openStream()); + } + } + } else if ("file".equals(u.getProtocol())) { + File schemaPath = new File(u.getPath()); + copySchemas(schemaPath, tmpDir); + } + } + return tmpDir; + } + + // TODO: This only copies schema for services.xml. Why? + private static void copySchemas(File from, File to) throws IOException { + // TODO: only copy .rnc files. + if (! from.exists()) throw new IOException("Could not find schema source directory '" + from + "'"); + if (! from.isDirectory()) throw new IOException("Schema source '" + from + "' is not a directory"); + File sourceFile = new File(from, servicesXmlSchemaName); + if (! sourceFile.exists()) throw new IOException("Schema source file '" + sourceFile + "' not found"); + IOUtils.copyDirectoryInto(from, to); + } + + private static void writeContentsToFile(File outDir, String outFile, InputStream inputStream) throws IOException { + String contents = IOUtils.readAll(new InputStreamReader(inputStream)); + File out = new File(outDir, outFile); + IOUtils.writeFile(out, contents, false); + } + + private SchemaValidator createValidator(File schemaDir, String schemaFile) { + try { + File file = new File(schemaDir + File.separator + "schema" + File.separator + schemaFile); + return new SchemaValidator(file, deployLogger); + } catch (SAXException e) { + throw new RuntimeException("Invalid schema '" + schemaFile + "'", e); + } catch (IOException e) { + throw new RuntimeException("IO error reading schema '" + schemaFile + "'", e); + } + } + +} diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/SimpleApplicationValidator.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/SimpleApplicationValidator.java index f94cf3e31a0..9db254bc742 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/SimpleApplicationValidator.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/SimpleApplicationValidator.java @@ -14,6 +14,6 @@ import java.io.Reader; public class SimpleApplicationValidator { public static void checkServices(Reader reader, Version version) throws IOException { - SchemaValidator.createTestValidatorServices(version).validate(reader); + new SchemaValidators(version, new BaseDeployLogger()).servicesXmlValidator().validate(reader); } } diff --git a/config-application-package/src/test/java/com/yahoo/config/application/OverrideProcessorTest.java b/config-application-package/src/test/java/com/yahoo/config/application/OverrideProcessorTest.java index b80372ec9ff..32a1c4f847f 100644 --- a/config-application-package/src/test/java/com/yahoo/config/application/OverrideProcessorTest.java +++ b/config-application-package/src/test/java/com/yahoo/config/application/OverrideProcessorTest.java @@ -15,8 +15,7 @@ import java.io.IOException; import java.io.StringReader; /** - * @author lulf - * @since 5.22 + * @author Ulf Lilleengen */ public class OverrideProcessorTest { diff --git a/config-lib/src/test/resources/configdefinitions/function-test.def b/config-lib/src/test/resources/configdefinitions/function-test.def index 63206fa7c3b..04d040a910b 100644 --- a/config-lib/src/test/resources/configdefinitions/function-test.def +++ b/config-lib/src/test/resources/configdefinitions/function-test.def @@ -18,7 +18,6 @@ # - Have an array within a struct, to verify that we correctly recurse. # - Reuse type name further within to ensure that this works. -version=8 # deprecated, remove in Vespa 7 namespace=test # Some random bool without a default value. These comments exist to check diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationFile.java b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationFile.java index bcc8d222ca5..0384a5c7a1c 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationFile.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationFile.java @@ -11,7 +11,7 @@ import java.util.List; * An application file represents a file within an application package. This class can be used to traverse the entire * application package file structure, as well as read and write files to it, and create directories. * - * @author Ulf Lillengen + * @author Ulf Lilleengen */ public abstract class ApplicationFile implements Comparable<ApplicationFile> { diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java index 795e87b7690..c1a786194a2 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.application.api; -import com.yahoo.config.provision.ProvisionInfo; +import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.Version; import com.yahoo.config.provision.Zone; import com.yahoo.path.Path; @@ -28,7 +28,7 @@ import java.util.jar.JarFile; * * Anyone wanting to access application data should use this interface. * - * @author vegardh + * @author Vegard Havdal */ public interface ApplicationPackage { @@ -228,10 +228,22 @@ public interface ApplicationPackage { throw new UnsupportedOperationException("This application package cannot write its metadata"); } - default Map<Version, ProvisionInfo> getProvisionInfoMap() { + /** + * Returns the single host allocation info of this, or an empty map if no allocation is available + * + * @deprecated please use #getAllocatedHosts + */ + // TODO: Remove on Vespa 7 + @Deprecated + default Map<Version, AllocatedHosts> getProvisionInfoMap() { return Collections.emptyMap(); } + /** Returns the host allocation info of this, or empty if no allocation is available */ + default Optional<AllocatedHosts> getAllocatedHosts() { + return Optional.empty(); + } + default Map<Version, FileRegistry> getFileRegistryMap() { return Collections.emptyMap(); } diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java index 2842ac5bd30..68c1a897dc3 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java @@ -346,11 +346,6 @@ public class DeploymentSpec { return true; } - // TODO: Remove when no version older than 6.111 is deployed anywhere - public boolean matches(Environment environment, Optional<RegionName> region) { - return deploysTo(environment, region); - } - @Override public int hashCode() { return Objects.hash(environment, region); diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/HostInfo.java b/config-model-api/src/main/java/com/yahoo/config/model/api/HostInfo.java index dcb875d02b5..589aee50d7c 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/HostInfo.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/HostInfo.java @@ -7,7 +7,6 @@ import java.util.Collection; * Contains information about a host and what services are running on it. * * @author lulf - * @since 5.37 */ public class HostInfo { diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/Model.java b/config-model-api/src/main/java/com/yahoo/config/model/api/Model.java index 0c038077fe4..f8f749ef070 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/Model.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/Model.java @@ -1,12 +1,11 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.model.api; -import com.yahoo.config.provision.ProvisionInfo; +import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.ConfigPayload; import com.yahoo.vespa.config.buildergen.ConfigDefinition; -import java.time.Clock; import java.time.Instant; import java.util.Optional; import java.util.Set; @@ -16,8 +15,7 @@ import java.util.Collection; * A {@link Model} represents the interface towards the model of an entire tenant, and defines methods * for querying this model. * - * @author lulf - * @since 5.1 + * @author Ulf Lilleengen */ public interface Model { @@ -60,9 +58,20 @@ public interface Model { /** * Get the provisioning info for this model. - * @return {@link ProvisionInfo} instance, if available. + * + * @return {@link AllocatedHosts} instance, if available. + * @deprecated use allocatedHosts */ - Optional<ProvisionInfo> getProvisionInfo(); + @Deprecated + // TODO: Remove this (and the implementation below) when no version older than 6.143 is deployed anywhere + default Optional<AllocatedHosts> getProvisionInfo() { + return Optional.of(allocatedHosts()); + } + + @SuppressWarnings("deprecation") + default AllocatedHosts allocatedHosts() { + return getProvisionInfo().get(); + } /** * Returns whether this application allows serving config request for a different version. @@ -71,11 +80,6 @@ public interface Model { */ default boolean allowModelVersionMismatch(Instant now) { return false; } - /** @deprecated pass now. */ - // TODO: Remove this when no version older than 6.115 is deployed anywhere - @Deprecated - default boolean allowModelVersionMismatch() { return allowModelVersionMismatch(Clock.systemUTC().instant()); } - /** * Returns whether old config models should be loaded (default) or not. * Skipping old config models is a validation override which is useful when the old model @@ -88,9 +92,4 @@ public interface Model { */ default boolean skipOldConfigModels(Instant now) { return false; } - /** @deprecated pass now. */ - // TODO: Remove this when no version older than 6.115 is deployed anywhere - @Deprecated - default boolean skipOldConfigModels() { return skipOldConfigModels(Clock.systemUTC().instant()); } - } diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 5afc570d81b..5b79415c132 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -17,7 +17,7 @@ import java.util.Set; /** * Model context containing state provided to model factories. * - * @author lulf + * @author Ulf Lilleengen */ public interface ModelContext { @@ -31,11 +31,6 @@ public interface ModelContext { Properties properties(); default Optional<File> appDir() { return Optional.empty();} - /** @deprecated TODO: Remove this when no config models older than 6.98 are used */ - @SuppressWarnings("unused") - @Deprecated - default Optional<com.yahoo.config.provision.Version> vespaVersion() { return Optional.empty(); } - /** The Vespa version this model is built for */ Version modelVespaVersion(); @@ -50,4 +45,5 @@ public interface ModelContext { Zone zone(); Set<Rotation> rotations(); } + } diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelState.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelState.java index 10153ca07df..c6ac913ad14 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelState.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelState.java @@ -5,5 +5,7 @@ package com.yahoo.config.model.api; * @author lulf */ public interface ModelState { + Model getModel(); + } diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/HostsXmlProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/HostsXmlProvisioner.java index 85f05116e5b..f909f3864da 100644 --- a/config-model/src/main/java/com/yahoo/config/model/provision/HostsXmlProvisioner.java +++ b/config-model/src/main/java/com/yahoo/config/model/provision/HostsXmlProvisioner.java @@ -14,7 +14,6 @@ import java.util.List; * application if one exists. Pre-condition: A valid hosts file. * * @author hmusum - * @since 5.11 */ public class HostsXmlProvisioner implements HostProvisioner { diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java index 133d94c745b..8b97eb2503e 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java @@ -25,6 +25,7 @@ import java.util.*; * @author tonytv */ public class MockApplicationPackage implements ApplicationPackage { + public static final String MUSIC_SEARCHDEFINITION = createSearchDefinition("music", "foo"); public static final String BOOK_SEARCHDEFINITION = createSearchDefinition("book", "bar"); diff --git a/config-model/src/main/java/com/yahoo/config/model/test/TestDriver.java b/config-model/src/main/java/com/yahoo/config/model/test/TestDriver.java index 55c522025fb..b538468d0bc 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/TestDriver.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/TestDriver.java @@ -5,7 +5,8 @@ import com.google.common.annotations.Beta; import com.yahoo.component.Version; import com.yahoo.config.model.MapConfigModelRegistry; import com.yahoo.config.application.api.ApplicationPackage; -import com.yahoo.config.model.application.provider.SchemaValidator; +import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.config.model.application.provider.SchemaValidators; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.builder.xml.ConfigModelBuilder; import com.yahoo.vespa.model.VespaModel; @@ -102,11 +103,10 @@ public class TestDriver { if (!validate) { return; } - SchemaValidator validator = SchemaValidator.createTestValidatorHosts(new Version(6)); + SchemaValidators schemaValidators = new SchemaValidators(new Version(6), new BaseDeployLogger()); if (appPkg.getHosts() != null) { - validator.validate(appPkg.getHosts()); + schemaValidators.hostsXmlValidator().validate(appPkg.getHosts()); } - validator = SchemaValidator.createTestValidatorServices(new Version(6)); - validator.validate(appPkg.getServices()); + schemaValidators.servicesXmlValidator().validate(appPkg.getServices()); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java b/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java index 6d4945f23ad..38e417aca9e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java @@ -45,7 +45,7 @@ public abstract class AbstractService extends AbstractConfigProducer<AbstractCon /** The optional PRELOAD libraries for this Service. */ // Please keep non-null, as passed to command line in service startup - private String preload = Defaults.getDefaults().underVespaHome("lib64/vespa/malloc/libvespamalloc.so"); + private String preload = Defaults.getDefaults().underVespaHome("lib64/vespa/malloc/libvespamallocd.so"); // If larger or equal to 0 it mean that explicit mmaps shall not be included in coredump. private long mmapNoCoreLimit = -1l; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java b/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java index bc890755ca9..5e74a2ebc8a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java @@ -22,7 +22,7 @@ import java.util.stream.Collectors; * TODO: Merge with {@link Host} * Host resources are ordered by their host order. * - * @author Ulf Lillengen + * @author Ulf Lilleengen */ public class HostResource implements Comparable<HostResource> { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java index c61435ca831..53cc8be9e96 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java @@ -20,7 +20,7 @@ import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.config.model.producer.AbstractConfigProducerRoot; import com.yahoo.config.model.producer.UserConfigRepo; -import com.yahoo.config.provision.ProvisionInfo; +import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.log.LogLevel; import com.yahoo.vespa.config.ConfigDefinitionKey; import com.yahoo.vespa.config.ConfigKey; @@ -83,7 +83,7 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri public static final Logger log = Logger.getLogger(VespaModel.class.getPackage().toString()); private ConfigModelRepo configModelRepo = new ConfigModelRepo(); - private final Optional<ProvisionInfo> info; + private final AllocatedHosts allocatedHosts; /** * The config id for the root config producer @@ -146,7 +146,7 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri if (complete) { // create a a completed, frozen model configModelRepo.readConfigModels(deployState, builder, root, configModelRegistry); addServiceClusters(deployState.getApplicationPackage(), builder); - this.info = Optional.of(createProvisionInfo()); // must happen after the two lines above + this.allocatedHosts = AllocatedHosts.withHosts(root.getHostSystem().getHostSpecs()); // must happen after the two lines above setupRouting(); this.fileDistributor = root.getFileDistributionConfigProducer().getFileDistributor(); getAdmin().addPerHostServices(getHostSystem().getHosts(), deployState.getProperties()); @@ -157,7 +157,7 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri this.deployState = null; } else { // create a model with no services instantiated and the given file distributor - this.info = Optional.of(createProvisionInfo()); + this.allocatedHosts = AllocatedHosts.withHosts(root.getHostSystem().getHostSpecs()); this.fileDistributor = fileDistributor; } } @@ -167,10 +167,6 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri return new VespaModel(new NullConfigModelRegistry(), deployState, false, new FileDistributor(deployState.getFileRegistry())); } - private ProvisionInfo createProvisionInfo() { - return ProvisionInfo.withHosts(root.getHostSystem().getHostSpecs()); - } - private void validateWrapExceptions() { try { validate(); @@ -421,8 +417,8 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri } @Override - public Optional<ProvisionInfo> getProvisionInfo() { - return info; + public AllocatedHosts allocatedHosts() { + return allocatedHosts; } private static Set<ConfigKey<?>> configsProduced(ConfigProducer cp) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java index 215aa6c2f7f..fc27f9e8dc7 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java @@ -34,7 +34,7 @@ import java.util.logging.Logger; /** * Factory for creating {@link VespaModel} instances. * - * @author lulf + * @author Ulf Lilleengen */ public class VespaModelFactory implements ModelFactory { @@ -42,9 +42,17 @@ public class VespaModelFactory implements ModelFactory { private final ConfigModelRegistry configModelRegistry; private final Zone zone; private final Clock clock; + private final Version version; + /** Creates a factory for vespa models for this version of the source */ @Inject public VespaModelFactory(ComponentRegistry<ConfigModelPlugin> pluginRegistry, Zone zone) { + this(Version.fromIntValues(VespaVersion.major, VespaVersion.minor, VespaVersion.micro), pluginRegistry, zone); + } + + /** Creates a factory for vespa models of a particular version */ + public VespaModelFactory(Version version, ComponentRegistry<ConfigModelPlugin> pluginRegistry, Zone zone) { + this.version = version; List<ConfigModelBuilder> modelBuilders = new ArrayList<>(); for (ConfigModelPlugin plugin : pluginRegistry.allComponents()) { if (plugin instanceof ConfigModelBuilder) { @@ -55,11 +63,15 @@ public class VespaModelFactory implements ModelFactory { this.zone = zone; this.clock = Clock.systemUTC(); } - + public VespaModelFactory(ConfigModelRegistry configModelRegistry) { this(configModelRegistry, Clock.systemUTC()); } public VespaModelFactory(ConfigModelRegistry configModelRegistry, Clock clock) { + this(Version.fromIntValues(VespaVersion.major, VespaVersion.minor, VespaVersion.micro), configModelRegistry, clock); + } + public VespaModelFactory(Version version, ConfigModelRegistry configModelRegistry, Clock clock) { + this.version = version; if (configModelRegistry == null) { this.configModelRegistry = new NullConfigModelRegistry(); log.info("Will not load config models from plugins, as no registry is available"); @@ -72,9 +84,7 @@ public class VespaModelFactory implements ModelFactory { /** Returns the version this model is build for */ @Override - public Version getVersion() { - return Version.fromIntValues(VespaVersion.major, VespaVersion.minor, VespaVersion.micro); - } + public Version getVersion() { return version; } @Override public Model createModel(ModelContext modelContext) { @@ -82,8 +92,7 @@ public class VespaModelFactory implements ModelFactory { } @Override - public ModelCreateResult createAndValidateModel(ModelContext modelContext, - boolean ignoreValidationErrors) { + public ModelCreateResult createAndValidateModel(ModelContext modelContext, boolean ignoreValidationErrors) { validateXml(modelContext, ignoreValidationErrors); DeployState deployState = createDeployState(modelContext); VespaModel model = buildModel(deployState); @@ -94,18 +103,16 @@ public class VespaModelFactory implements ModelFactory { private void validateXml(ModelContext modelContext, boolean ignoreValidationErrors) { if (modelContext.appDir().isPresent()) { ApplicationPackageXmlFilesValidator validator = - ApplicationPackageXmlFilesValidator.createDefaultXMLValidator(modelContext.appDir().get(), - modelContext.modelVespaVersion()); + ApplicationPackageXmlFilesValidator.create(modelContext.appDir().get(), + modelContext.modelVespaVersion()); try { validator.checkApplication(); - ApplicationPackageXmlFilesValidator.checkIncludedDirs(modelContext.applicationPackage(), - modelContext.modelVespaVersion()); + validator.checkIncludedDirs(modelContext.applicationPackage()); } catch (IllegalArgumentException e) { rethrowUnlessIgnoreErrors(e, ignoreValidationErrors); } catch (Exception e) { throw new RuntimeException(e); } - } else { validateXML(modelContext.applicationPackage(), ignoreValidationErrors); } @@ -164,7 +171,6 @@ public class VespaModelFactory implements ModelFactory { private List<ConfigChangeAction> validateModel(VespaModel model, DeployState deployState, boolean ignoreValidationErrors) { try { - deployState.getApplicationPackage().validateXML(); return Validation.validate(model, ignoreValidationErrors, deployState); } catch (IllegalArgumentException e) { rethrowUnlessIgnoreErrors(e, ignoreValidationErrors); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Slobrok.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Slobrok.java index 0867fc2a299..cefc08981a4 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/Slobrok.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Slobrok.java @@ -41,7 +41,7 @@ public class Slobrok extends AbstractService { } public String getStartupCommand() { - return "exec $ROOT/bin/vespa-slobrok -p " + getPort() + + return "exec $ROOT/sbin/vespa-slobrok -p " + getPort() + " -s " + getStatePort() + " -c " + getConfigId(); } diff --git a/config-model/src/test/cfg/application/configdeftest/configdefinitions/foo.def b/config-model/src/test/cfg/application/configdeftest/configdefinitions/foo.def index 89ce911ce10..c37aafd6fc4 100644 --- a/config-model/src/test/cfg/application/configdeftest/configdefinitions/foo.def +++ b/config-model/src/test/cfg/application/configdeftest/configdefinitions/foo.def @@ -1,4 +1,3 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=2 namespace=config bar int default=1 diff --git a/config-model/src/test/cfg/application/configdeftest/configdefinitions/xyzzy.def b/config-model/src/test/cfg/application/configdeftest/configdefinitions/xyzzy.def index e94d98b448e..c37aafd6fc4 100644 --- a/config-model/src/test/cfg/application/configdeftest/configdefinitions/xyzzy.def +++ b/config-model/src/test/cfg/application/configdeftest/configdefinitions/xyzzy.def @@ -1,4 +1,3 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=1 namespace=config bar int default=1 diff --git a/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java b/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java index ee2e6ffcc74..1e1b8cd2ac8 100644 --- a/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java @@ -242,9 +242,10 @@ public class ApplicationDeployTest { public FilesApplicationPackage createAppPkg(String appPkg, boolean validateXml) throws IOException { final FilesApplicationPackage filesApplicationPackage = FilesApplicationPackage.fromFile(new File(appPkg)); if (validateXml) { - ApplicationPackageXmlFilesValidator validator = ApplicationPackageXmlFilesValidator.createTestXmlValidator(new File(appPkg), new Version(6)); + ApplicationPackageXmlFilesValidator validator = + ApplicationPackageXmlFilesValidator.create(new File(appPkg), new Version(6)); validator.checkApplication(); - ApplicationPackageXmlFilesValidator.checkIncludedDirs(filesApplicationPackage, new Version(6)); + validator.checkIncludedDirs(filesApplicationPackage); } return filesApplicationPackage; } diff --git a/config-model/src/test/java/com/yahoo/config/model/application/provider/SchemaValidatorTest.java b/config-model/src/test/java/com/yahoo/config/model/application/provider/SchemaValidatorTest.java index 42659755186..fda230e22ab 100644 --- a/config-model/src/test/java/com/yahoo/config/model/application/provider/SchemaValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/application/provider/SchemaValidatorTest.java @@ -65,6 +65,6 @@ public class SchemaValidatorTest { } private SchemaValidator createValidator() throws IOException { - return SchemaValidator.createTestValidatorServices(new Version(6)); + return new SchemaValidators(new Version(6)).servicesXmlValidator(); } } diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/HostSpecTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/HostSpecTest.java index 6a17f314d26..51b039a7532 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/HostSpecTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/HostSpecTest.java @@ -11,10 +11,10 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; /** - * @author lulf - * @since 5.11 + * @author Ulf Lilleengen */ public class HostSpecTest { + @Test public void testEquals() { HostSpec h1 = new HostSpec("foo", Collections.<String>emptyList()); @@ -42,4 +42,5 @@ public class HostSpecTest { assertFalse(h4.equals(h3)); assertTrue(h4.equals(h4)); } + } diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java index 63d5d37598b..98f599769c0 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java @@ -110,9 +110,9 @@ public class ModelProvisioningTest { assertThat(model.getContainerClusters().get("mydisc").getContainers().get(0).getJvmArgs(), is("")); assertThat(model.getContainerClusters().get("mydisc").getContainers().get(1).getJvmArgs(), is("")); assertThat(model.getContainerClusters().get("mydisc").getContainers().get(2).getJvmArgs(), is("")); - assertThat(model.getContainerClusters().get("mydisc").getContainers().get(0).getPreLoad(), is(getDefaults().underVespaHome("lib64/vespa/malloc/libvespamalloc.so"))); - assertThat(model.getContainerClusters().get("mydisc").getContainers().get(1).getPreLoad(), is(getDefaults().underVespaHome("lib64/vespa/malloc/libvespamalloc.so"))); - assertThat(model.getContainerClusters().get("mydisc").getContainers().get(2).getPreLoad(), is(getDefaults().underVespaHome("lib64/vespa/malloc/libvespamalloc.so"))); + assertThat(model.getContainerClusters().get("mydisc").getContainers().get(0).getPreLoad(), is(getDefaults().underVespaHome("lib64/vespa/malloc/libvespamallocd.so"))); + assertThat(model.getContainerClusters().get("mydisc").getContainers().get(1).getPreLoad(), is(getDefaults().underVespaHome("lib64/vespa/malloc/libvespamallocd.so"))); + assertThat(model.getContainerClusters().get("mydisc").getContainers().get(2).getPreLoad(), is(getDefaults().underVespaHome("lib64/vespa/malloc/libvespamallocd.so"))); assertThat(model.getContainerClusters().get("mydisc").getMemoryPercentage(), is(Optional.empty())); assertThat(model.getContainerClusters().get("mydisc2").getContainers().get(0).getJvmArgs(), is("-verbosegc")); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java index 8102f358830..cc3f4a22966 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java @@ -18,7 +18,7 @@ import com.yahoo.config.model.provision.InMemoryProvisioner; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.config.model.test.TestDriver; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ProvisionInfo; +import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.messagebus.MessagebusConfig; import com.yahoo.net.HostName; @@ -286,7 +286,7 @@ public class VespaModelTestCase { .build()) .build(); VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); - ProvisionInfo info = model.getProvisionInfo().get(); + AllocatedHosts info = model.allocatedHosts(); assertEquals("Admin version 3 is ignored, and there are no other hosts to borrow for admin services", 0, info.getHosts().size()); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithFilePkg.java b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithFilePkg.java index 7802b6f51cd..c89d3098c4d 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithFilePkg.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithFilePkg.java @@ -45,8 +45,10 @@ public class VespaModelCreatorWithFilePkg { } public void validate() throws IOException { - ApplicationPackageXmlFilesValidator.createTestXmlValidator(applicationPkg.getAppDir(), new Version(6)).checkApplication(); - ApplicationPackageXmlFilesValidator.checkIncludedDirs(applicationPkg, new Version(6)); + ApplicationPackageXmlFilesValidator validator = + ApplicationPackageXmlFilesValidator.create(applicationPkg.getAppDir(), new Version(6)); + validator.checkApplication(); + validator.checkIncludedDirs(applicationPkg); } public VespaModel create(boolean validateApplicationWithSchema) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithMockPkg.java b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithMockPkg.java index 98e9fd7b166..8bb500906f3 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithMockPkg.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithMockPkg.java @@ -6,8 +6,8 @@ import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.ConfigModelRegistry; import com.yahoo.config.model.NullConfigModelRegistry; import com.yahoo.config.model.api.ConfigChangeAction; +import com.yahoo.config.model.application.provider.SchemaValidators; import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.config.model.application.provider.SchemaValidator; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.application.validation.Validation; @@ -56,14 +56,15 @@ public class VespaModelCreatorWithMockPkg { VespaModel model = new VespaModel(configModelRegistry, deployState); Version vespaVersion = new Version(6); if (validate) { + SchemaValidators validators = new SchemaValidators(vespaVersion); try { if (appPkg.getHosts() != null) { - SchemaValidator.createTestValidatorHosts(vespaVersion).validate(appPkg.getHosts()); + validators.hostsXmlValidator().validate(appPkg.getHosts()); } if (appPkg.getDeployment().isPresent()) { - SchemaValidator.createTestValidatorDeployment(vespaVersion).validate(appPkg.getDeployment().get()); + validators.deploymentXmlValidator().validate(appPkg.getDeployment().get()); } - SchemaValidator.createTestValidatorServices(vespaVersion).validate(appPkg.getServices()); + validators.servicesXmlValidator().validate(appPkg.getServices()); } catch (Exception e) { System.err.println(e.getClass()); throw e instanceof RuntimeException ? (RuntimeException) e : new RuntimeException(e); diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/AllocatedHosts.java b/config-provisioning/src/main/java/com/yahoo/config/provision/AllocatedHosts.java new file mode 100644 index 00000000000..13efc2b3337 --- /dev/null +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/AllocatedHosts.java @@ -0,0 +1,116 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.provision; + +import com.google.common.collect.ImmutableSet; +import com.yahoo.slime.ArrayTraverser; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.config.SlimeUtils; + +import java.io.IOException; +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.Optional; +import java.util.Set; + +/** + * The hosts allocated to an application. + * This can be serialized to/from JSON. + * This is immutable. + * + * @author Ulf Lilleengen + * @author bratseth + */ +public class AllocatedHosts { + + private static final String mappingKey = "mapping"; + private static final String hostSpecKey = "hostSpec"; + private static final String hostSpecHostName = "hostName"; + private static final String hostSpecMembership = "membership"; + private static final String hostSpecFlavor = "flavor"; + private static final String hostSpecVespaVersion = "vespaVersion"; + + private final ImmutableSet<HostSpec> hosts; + + AllocatedHosts(Set<HostSpec> hosts) { + this.hosts = ImmutableSet.copyOf(hosts); + } + + public static AllocatedHosts withHosts(Set<HostSpec> hosts) { + return new AllocatedHosts(hosts); + } + + private void toSlime(Cursor cursor) { + Cursor array = cursor.setArray(mappingKey); + for (HostSpec host : hosts) + toSlime(host, array.addObject().setObject(hostSpecKey)); + } + + private void toSlime(HostSpec host, Cursor cursor) { + cursor.setString(hostSpecHostName, host.hostname()); + if (host.membership().isPresent()) { + cursor.setString(hostSpecMembership, host.membership().get().stringValue()); + cursor.setString(hostSpecVespaVersion, host.membership().get().cluster().vespaVersion().toString()); + } + if (host.flavor().isPresent()) + cursor.setString(hostSpecFlavor, host.flavor().get().name()); + } + + /** Returns the hosts of this allocation */ + public Set<HostSpec> getHosts() { return hosts; } + + private static AllocatedHosts fromSlime(Inspector inspector, Optional<NodeFlavors> nodeFlavors) { + Inspector array = inspector.field(mappingKey); + Set<HostSpec> hosts = new LinkedHashSet<>(); + array.traverse(new ArrayTraverser() { + @Override + public void entry(int i, Inspector inspector) { + hosts.add(hostsFromSlime(inspector.field(hostSpecKey), nodeFlavors)); + } + }); + return new AllocatedHosts(hosts); + } + + static HostSpec hostsFromSlime(Inspector object, Optional<NodeFlavors> nodeFlavors) { + Optional<ClusterMembership> membership = + object.field(hostSpecMembership).valid() ? Optional.of(membershipFromSlime(object)) : Optional.empty(); + Optional<Flavor> flavor = + object.field(hostSpecFlavor).valid() ? flavorFromSlime(object, nodeFlavors) : Optional.empty(); + + return new HostSpec(object.field(hostSpecHostName).asString(),Collections.emptyList(), flavor, membership); + } + + private static ClusterMembership membershipFromSlime(Inspector object) { + return ClusterMembership.from(object.field(hostSpecMembership).asString(), + com.yahoo.component.Version.fromString(object.field(hostSpecVespaVersion).asString())); + } + + private static Optional<Flavor> flavorFromSlime(Inspector object, Optional<NodeFlavors> nodeFlavors) { + return nodeFlavors.map(flavorMapper -> flavorMapper.getFlavor(object.field(hostSpecFlavor).asString())) + .orElse(Optional.empty()); + } + + public byte[] toJson() throws IOException { + Slime slime = new Slime(); + toSlime(slime.setObject()); + return SlimeUtils.toJsonBytes(slime); + } + + public static AllocatedHosts fromJson(byte[] json, Optional<NodeFlavors> nodeFlavors) { + return fromSlime(SlimeUtils.jsonToSlime(json).get(), nodeFlavors); + } + + @Override + public boolean equals(Object other) { + if (other == this) return true; + if ( ! (other instanceof AllocatedHosts)) return false; + return ((AllocatedHosts) other).hosts.equals(this.hosts); + } + + @Override + public int hashCode() { + return hosts.hashCode(); + } + +} diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/HostSpec.java b/config-provisioning/src/main/java/com/yahoo/config/provision/HostSpec.java index 5d6b3fcaca4..dd8bc311939 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/HostSpec.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/HostSpec.java @@ -5,11 +5,12 @@ import com.google.common.collect.ImmutableList; import java.util.ArrayList; import java.util.List; +import java.util.Objects; import java.util.Optional; /** * A specification of a host and its role. - * The identity of a host is determined by its name. + * Equality and order is determined by the host name. * * @author hmusum */ @@ -19,7 +20,7 @@ public class HostSpec implements Comparable<HostSpec> { private final String hostname; /** Aliases of this host */ - private final List<String> aliases; + private final ImmutableList<String> aliases; /** The current membership role of this host in the cluster it belongs to */ private final Optional<ClusterMembership> membership; @@ -69,10 +70,11 @@ public class HostSpec implements Comparable<HostSpec> { } @Override - public boolean equals(Object o) { - if ( ! (o instanceof HostSpec)) return false; - HostSpec other = (HostSpec) o; - return this.hostname().equals(other.hostname()); + public boolean equals(Object other) { + if (other == this) return true; + if ( ! (other instanceof HostSpec)) return false; + + return ((HostSpec)other).hostname.equals(this.hostname); } @Override diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ProvisionInfo.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ProvisionInfo.java index 8bef1f7c9b7..dbb1b55aeb9 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/ProvisionInfo.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ProvisionInfo.java @@ -1,63 +1,35 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.provision; import com.yahoo.slime.ArrayTraverser; -import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; -import com.yahoo.slime.Slime; import com.yahoo.vespa.config.SlimeUtils; -import java.io.IOException; -import java.util.Collections; import java.util.LinkedHashSet; import java.util.Optional; import java.util.Set; /** - * Information about hosts provisioned for an application, and (de)serialization of this information to/from JSON. - * - * @author lulf - * @since 5.12 + * @author bratseth + * @deprecated use AllocatedHosts */ -public class ProvisionInfo { +// TODO: Remove when no version older than 6.143 is in production anywhere +@Deprecated +@SuppressWarnings("unused") +public class ProvisionInfo extends AllocatedHosts { private static final String mappingKey = "mapping"; private static final String hostSpecKey = "hostSpec"; - private static final String hostSpecHostName = "hostName"; - private static final String hostSpecMembership = "membership"; - private static final String hostSpecFlavor = "flavor"; - private static final String hostSpecVespaVersion = "vespaVersion"; - - private final Set<HostSpec> hosts = new LinkedHashSet<>(); private ProvisionInfo(Set<HostSpec> hosts) { - this.hosts.addAll(hosts); + super(hosts); } public static ProvisionInfo withHosts(Set<HostSpec> hosts) { return new ProvisionInfo(hosts); } - private void toSlime(Cursor cursor) { - Cursor array = cursor.setArray(mappingKey); - for (HostSpec host : hosts) { - Cursor object = array.addObject(); - serializeHostSpec(object.setObject(hostSpecKey), host); - } - } - - private void serializeHostSpec(Cursor cursor, HostSpec host) { - cursor.setString(hostSpecHostName, host.hostname()); - if (host.membership().isPresent()) { - cursor.setString(hostSpecMembership, host.membership().get().stringValue()); - cursor.setString(hostSpecVespaVersion, host.membership().get().cluster().vespaVersion().toString()); - } - if (host.flavor().isPresent()) - cursor.setString(hostSpecFlavor, host.flavor().get().name()); - } - - public Set<HostSpec> getHosts() { - return Collections.unmodifiableSet(hosts); + public static ProvisionInfo fromJson(byte[] json, Optional<NodeFlavors> nodeFlavors) { + return fromSlime(SlimeUtils.jsonToSlime(json).get(), nodeFlavors); } private static ProvisionInfo fromSlime(Inspector inspector, Optional<NodeFlavors> nodeFlavors) { @@ -66,46 +38,10 @@ public class ProvisionInfo { array.traverse(new ArrayTraverser() { @Override public void entry(int i, Inspector inspector) { - hosts.add(deserializeHostSpec(inspector.field(hostSpecKey), nodeFlavors)); + hosts.add(hostsFromSlime(inspector.field(hostSpecKey), nodeFlavors)); } }); return new ProvisionInfo(hosts); } - private static HostSpec deserializeHostSpec(Inspector object, Optional<NodeFlavors> nodeFlavors) { - Optional<ClusterMembership> membership = - object.field(hostSpecMembership).valid() ? Optional.of(readMembership(object)) : Optional.empty(); - Optional<Flavor> flavor = - object.field(hostSpecFlavor).valid() ? readFlavor(object, nodeFlavors) : Optional.empty(); - - return new HostSpec(object.field(hostSpecHostName).asString(),Collections.emptyList(), flavor, membership); - } - - private static ClusterMembership readMembership(Inspector object) { - return ClusterMembership.from(object.field(hostSpecMembership).asString(), - com.yahoo.component.Version.fromString(object.field(hostSpecVespaVersion).asString())); - } - - private static Optional<Flavor> readFlavor(Inspector object, Optional<NodeFlavors> nodeFlavors) { - return nodeFlavors.map(flavorMapper -> flavorMapper.getFlavor(object.field(hostSpecFlavor).asString())) - .orElse(Optional.empty()); - } - - public byte[] toJson() throws IOException { - Slime slime = new Slime(); - toSlime(slime.setObject()); - return SlimeUtils.toJsonBytes(slime); - } - - public static ProvisionInfo fromJson(byte[] json, Optional<NodeFlavors> nodeFlavors) { - return fromSlime(SlimeUtils.jsonToSlime(json).get(), nodeFlavors); - } - - public ProvisionInfo merge(ProvisionInfo provisionInfo) { - Set<HostSpec> mergedSet = new LinkedHashSet<>(); - mergedSet.addAll(this.hosts); - mergedSet.addAll(provisionInfo.getHosts()); - return ProvisionInfo.withHosts(mergedSet); - } - } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/Provisioner.java b/config-provisioning/src/main/java/com/yahoo/config/provision/Provisioner.java index 980cd4a00b9..6be1d49ebd3 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/Provisioner.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/Provisioner.java @@ -9,8 +9,7 @@ import java.util.List; /** * Interface used by the config system to acquire hosts. * - * @author lulf - * @since 5.11 + * @author Ulf Lilleengen */ public interface Provisioner { diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/AllocatedHostsTest.java b/config-provisioning/src/test/java/com/yahoo/config/provision/AllocatedHostsTest.java new file mode 100644 index 00000000000..675af88596a --- /dev/null +++ b/config-provisioning/src/test/java/com/yahoo/config/provision/AllocatedHostsTest.java @@ -0,0 +1,52 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.provision; + +import org.junit.Test; + +import java.io.IOException; +import java.util.LinkedHashSet; +import java.util.Optional; +import java.util.Set; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * @author Ulf Lilleengen + */ +public class AllocatedHostsTest { + + private final HostSpec h1 = new HostSpec("host1", Optional.empty()); + private final HostSpec h2 = new HostSpec("host2", Optional.empty()); + private final HostSpec h3 = new HostSpec("host3", Optional.of(ClusterMembership.from("container/test/0", com.yahoo.component.Version.fromString("6.73.1")))); + + @Test + public void testAllocatedHostsSerialization() throws IOException { + Set<HostSpec> hosts = new LinkedHashSet<>(); + hosts.add(h1); + hosts.add(h2); + hosts.add(h3); + AllocatedHosts info = AllocatedHosts.withHosts(hosts); + assertAllocatedHosts(info); + } + + private void assertAllocatedHosts(AllocatedHosts info) throws IOException { + AllocatedHosts serializedAllocatedHosts = AllocatedHosts.fromJson(info.toJson(), Optional.empty()); + assertEquals(info.getHosts().size(), serializedAllocatedHosts.getHosts().size()); + assertTrue(serializedAllocatedHosts.getHosts().contains(h1)); + assertTrue(serializedAllocatedHosts.getHosts().contains(h2)); + assertTrue(serializedAllocatedHosts.getHosts().contains(h3)); + assertTrue(!getHost(h1.hostname(), serializedAllocatedHosts.getHosts()).membership().isPresent()); + assertEquals("container/test/0", getHost(h3.hostname(), serializedAllocatedHosts.getHosts()).membership().get().stringValue()); + assertEquals(h3.membership().get().cluster().vespaVersion(), getHost(h3.hostname(), + serializedAllocatedHosts.getHosts()).membership().get().cluster().vespaVersion()); + } + + private HostSpec getHost(String hostname, Set<HostSpec> hosts) { + for (HostSpec host : hosts) + if (host.hostname().equals(hostname)) + return host; + throw new IllegalArgumentException("No host " + hostname + " is present"); + } + +} diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/ProvisionInfoTest.java b/config-provisioning/src/test/java/com/yahoo/config/provision/ProvisionInfoTest.java deleted file mode 100644 index 4fa69eb77e0..00000000000 --- a/config-provisioning/src/test/java/com/yahoo/config/provision/ProvisionInfoTest.java +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.config.provision; - -import org.junit.Test; - -import java.io.IOException; -import java.util.Collections; -import java.util.LinkedHashSet; -import java.util.Optional; -import java.util.Set; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -/** - * @author lulf - * @since 5.12 - */ -public class ProvisionInfoTest { - - private final HostSpec h1 = new HostSpec("host1", Optional.empty()); - private final HostSpec h2 = new HostSpec("host2", Optional.empty()); - private final HostSpec h3 = new HostSpec("host3", Optional.of(ClusterMembership.from("container/test/0", com.yahoo.component.Version.fromString("6.73.1")))); - - @Test - public void testProvisionInfoSerialization() throws IOException { - Set<HostSpec> hosts = new LinkedHashSet<>(); - hosts.add(h1); - hosts.add(h2); - hosts.add(h3); - ProvisionInfo info = ProvisionInfo.withHosts(hosts); - assertProvisionInfo(info); - } - - @Test - public void testProvisionInfoMerging() throws IOException { - Set<HostSpec> hostsA = new LinkedHashSet<>(Collections.singleton(h1)); - Set<HostSpec> hostsB = new LinkedHashSet<>(); - hostsB.add(h2); - hostsB.add(h3); - - ProvisionInfo infoA = ProvisionInfo.withHosts(hostsA); - ProvisionInfo infoB = ProvisionInfo.withHosts(hostsB); - assertProvisionInfo(infoA.merge(infoB)); - assertProvisionInfo(infoB.merge(infoA)); - } - - private void assertProvisionInfo(ProvisionInfo info) throws IOException { - ProvisionInfo serializedInfo = ProvisionInfo.fromJson(info.toJson(), Optional.empty()); - assertEquals(info.getHosts().size(), serializedInfo.getHosts().size()); - assertTrue(serializedInfo.getHosts().contains(h1)); - assertTrue(serializedInfo.getHosts().contains(h2)); - assertTrue(serializedInfo.getHosts().contains(h3)); - assertTrue(!getHost(h1.hostname(), serializedInfo.getHosts()).membership().isPresent()); - assertEquals("container/test/0", getHost(h3.hostname(), serializedInfo.getHosts()).membership().get().stringValue()); - assertEquals(h3.membership().get().cluster().vespaVersion(), getHost(h3.hostname(), serializedInfo.getHosts()).membership().get().cluster().vespaVersion()); - } - - private HostSpec getHost(String hostname, Set<HostSpec> hosts) { - for (HostSpec host : hosts) - if (host.hostname().equals(hostname)) - return host; - throw new IllegalArgumentException("No host " + hostname + " is present"); - } - -} diff --git a/config/src/test/java/com/yahoo/vespa/config/classes/app.1.def b/config/src/test/java/com/yahoo/vespa/config/classes/app.1.def index 7a8f33e075c..df2a57bad04 100644 --- a/config/src/test/java/com/yahoo/vespa/config/classes/app.1.def +++ b/config/src/test/java/com/yahoo/vespa/config/classes/app.1.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=1 message string default="Hello!" diff --git a/config/src/test/java/com/yahoo/vespa/config/classes/qr-logging.def b/config/src/test/java/com/yahoo/vespa/config/classes/qr-logging.def index f2325a5d13f..3fcab08a63b 100644 --- a/config/src/test/java/com/yahoo/vespa/config/classes/qr-logging.def +++ b/config/src/test/java/com/yahoo/vespa/config/classes/qr-logging.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=3 logger string default="com.yahoo" # Either QueryAccessLog for a regular Vespa access log, or YApacheAccessLog for a log on yApache format speciallog[].name string diff --git a/config/src/test/java/com/yahoo/vespa/config/classes/qr-templates.3.def b/config/src/test/java/com/yahoo/vespa/config/classes/qr-templates.3.def index 1b1538150a8..d3d7ff87cbd 100644 --- a/config/src/test/java/com/yahoo/vespa/config/classes/qr-templates.3.def +++ b/config/src/test/java/com/yahoo/vespa/config/classes/qr-templates.3.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=3 ## Directory for temporary files directory string default="tmp/templates" diff --git a/config/src/test/java/com/yahoo/vespa/config/classes/ranges.1.def b/config/src/test/java/com/yahoo/vespa/config/classes/ranges.1.def index b165f37a898..5377b143d77 100644 --- a/config/src/test/java/com/yahoo/vespa/config/classes/ranges.1.def +++ b/config/src/test/java/com/yahoo/vespa/config/classes/ranges.1.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=1 quux int default=5 range=[,] xyzzy double default=5 range=[,] longVal long default=5 range=[,] diff --git a/config/src/test/java/com/yahoo/vespa/config/classes/testfoobar.12.def b/config/src/test/java/com/yahoo/vespa/config/classes/testfoobar.12.def index acd825a7c54..ba008565f8d 100644 --- a/config/src/test/java/com/yahoo/vespa/config/classes/testfoobar.12.def +++ b/config/src/test/java/com/yahoo/vespa/config/classes/testfoobar.12.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=12-1-2 longVal long longWithDefault long default=8589934592 diff --git a/config/src/test/resources/configs/def-files/app.def b/config/src/test/resources/configs/def-files/app.def index 1e91c9c665c..37c3dd75e26 100644 --- a/config/src/test/resources/configs/def-files/app.def +++ b/config/src/test/resources/configs/def-files/app.def @@ -1,6 +1,5 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. namespace=foo -version=1 message string default="Hello!" diff --git a/config/src/test/resources/configs/def-files/arraytypes.def b/config/src/test/resources/configs/def-files/arraytypes.def index da0ace98df6..ad18c4e5386 100644 --- a/config/src/test/resources/configs/def-files/arraytypes.def +++ b/config/src/test/resources/configs/def-files/arraytypes.def @@ -2,7 +2,6 @@ # Config containing only simple array types that can be used for testing # individual types in detail. namespace=foo -version=1 boolarr[] bool doublearr[] double diff --git a/config/src/test/resources/configs/def-files/chains-test.def b/config/src/test/resources/configs/def-files/chains-test.def index 7e9444970c3..b7da79931d5 100644 --- a/config/src/test/resources/configs/def-files/chains-test.def +++ b/config/src/test/resources/configs/def-files/chains-test.def @@ -1,7 +1,6 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. # Chains configuration namespace=foo -version=12 component[].id string diff --git a/config/src/test/resources/configs/def-files/datastructures.def b/config/src/test/resources/configs/def-files/datastructures.def index 52bd9ca965a..cbcbc4b8bed 100644 --- a/config/src/test/resources/configs/def-files/datastructures.def +++ b/config/src/test/resources/configs/def-files/datastructures.def @@ -1,6 +1,5 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. namespace=foo -version=4 date[] string diff --git a/config/src/test/resources/configs/def-files/defaulttest.def b/config/src/test/resources/configs/def-files/defaulttest.def index b122718d7c3..fa8005b77c3 100644 --- a/config/src/test/resources/configs/def-files/defaulttest.def +++ b/config/src/test/resources/configs/def-files/defaulttest.def @@ -1,6 +1,5 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. namespace=foo -version=3 nondefaultstring string defaultstring string default="thedefault" diff --git a/config/src/test/resources/configs/def-files/function-test.def b/config/src/test/resources/configs/def-files/function-test.def index b2185bf55b2..24eeb81ab7a 100644 --- a/config/src/test/resources/configs/def-files/function-test.def +++ b/config/src/test/resources/configs/def-files/function-test.def @@ -18,7 +18,6 @@ # - Have an array within a struct, to verify that we correctly recurse. # - Reuse type name further within to ensure that this works. -version=8 namespace=foo diff --git a/config/src/test/resources/configs/def-files/int.def b/config/src/test/resources/configs/def-files/int.def index 91efe21556f..c1a1241c413 100755 --- a/config/src/test/resources/configs/def-files/int.def +++ b/config/src/test/resources/configs/def-files/int.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. namespace=foo -version=1 intVal int default=1 diff --git a/config/src/test/resources/configs/def-files/md5test.def b/config/src/test/resources/configs/def-files/md5test.def index 868e62f674a..da79d022648 100644 --- a/config/src/test/resources/configs/def-files/md5test.def +++ b/config/src/test/resources/configs/def-files/md5test.def @@ -1,10 +1,8 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. namespace=foo -# version=4 , version in comment does not count. # Added empty line to see if we can confuse # the server's md5 calculation -version=3 #even adding a variable name starting with 'version' versiontag int default=3 diff --git a/config/src/test/resources/configs/def-files/namespace.def b/config/src/test/resources/configs/def-files/namespace.def index 23bb794f198..a8b55fb315e 100644 --- a/config/src/test/resources/configs/def-files/namespace.def +++ b/config/src/test/resources/configs/def-files/namespace.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=1 namespace=myproject.config diff --git a/config/src/test/resources/configs/def-files/simpletypes.def b/config/src/test/resources/configs/def-files/simpletypes.def index 4fe74c23763..2603e2f4100 100644 --- a/config/src/test/resources/configs/def-files/simpletypes.def +++ b/config/src/test/resources/configs/def-files/simpletypes.def @@ -2,7 +2,6 @@ namespace=foo # Config containing only simple leaf types with default values, that can be used # for testing individual types in detail. -version=1 boolval bool default=false doubleval double default=0.0 diff --git a/config/src/test/resources/configs/def-files/standard.def b/config/src/test/resources/configs/def-files/standard.def index 4955bea9139..f4c6ced1e00 100644 --- a/config/src/test/resources/configs/def-files/standard.def +++ b/config/src/test/resources/configs/def-files/standard.def @@ -2,7 +2,6 @@ # Config containing only simple leaf types with default values, that can be used # for testing individual types in detail. namespace=foo -version=1 basicStruct.intVal int default=0 basicStruct.stringVal string default="s" diff --git a/config/src/test/resources/configs/def-files/string.def b/config/src/test/resources/configs/def-files/string.def index 44d94c7b8a5..78596515b7c 100755 --- a/config/src/test/resources/configs/def-files/string.def +++ b/config/src/test/resources/configs/def-files/string.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. namespace=foo -version=1 stringVal string default="_default_" diff --git a/config/src/test/resources/configs/def-files/structtypes.def b/config/src/test/resources/configs/def-files/structtypes.def index 4f8260030a5..fe9d879fb64 100644 --- a/config/src/test/resources/configs/def-files/structtypes.def +++ b/config/src/test/resources/configs/def-files/structtypes.def @@ -1,7 +1,6 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. # Config containing only structs in various forms namespace=foo -version=2 simple.name string default="_default_" simple.gender enum { MALE, FEMALE } default=MALE diff --git a/config/src/test/resources/configs/def-files/test-nodefs.def b/config/src/test/resources/configs/def-files/test-nodefs.def index 2126d4461c3..e3b700c5732 100644 --- a/config/src/test/resources/configs/def-files/test-nodefs.def +++ b/config/src/test/resources/configs/def-files/test-nodefs.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=2 namespace=foo # test config vars with no defaults diff --git a/config/src/test/resources/configs/def-files/test-nonstring.def b/config/src/test/resources/configs/def-files/test-nonstring.def index b19b5aad1d9..eaa6fafe87d 100644 --- a/config/src/test/resources/configs/def-files/test-nonstring.def +++ b/config/src/test/resources/configs/def-files/test-nonstring.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=1 namespace=foo # Test non-string config vars with defaults diff --git a/config/src/test/resources/configs/def-files/test-reference.def b/config/src/test/resources/configs/def-files/test-reference.def index 449f1eda557..b5bbb5f4462 100644 --- a/config/src/test/resources/configs/def-files/test-reference.def +++ b/config/src/test/resources/configs/def-files/test-reference.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=1 namespace=foo configId reference default=":parent:" diff --git a/config/src/test/resources/configs/def-files/testnamespace.def b/config/src/test/resources/configs/def-files/testnamespace.def index 5a946f44c27..6e58c691097 100644 --- a/config/src/test/resources/configs/def-files/testnamespace.def +++ b/config/src/test/resources/configs/def-files/testnamespace.def @@ -1,4 +1,3 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=1 namespace=foo basicStruct.stringVal string diff --git a/config/src/test/resources/configs/def-files/unicode.def b/config/src/test/resources/configs/def-files/unicode.def index fa0a89d177d..493e3e37630 100644 --- a/config/src/test/resources/configs/def-files/unicode.def +++ b/config/src/test/resources/configs/def-files/unicode.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=2 namespace=foo unicodestring1 string diff --git a/config_test/src/main/resources/configdefinitions/greeting.def b/config_test/src/main/resources/configdefinitions/greeting.def index 64a14016f90..924c1331e2c 100644 --- a/config_test/src/main/resources/configdefinitions/greeting.def +++ b/config_test/src/main/resources/configdefinitions/greeting.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=1 namespace=configtest greeting string default="Hello, world." diff --git a/configdefinitions/src/vespa/ilscripts.def b/configdefinitions/src/vespa/ilscripts.def index 2fe34f678cc..901e87dbd04 100644 --- a/configdefinitions/src/vespa/ilscripts.def +++ b/configdefinitions/src/vespa/ilscripts.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=5 namespace=vespa.configdefinition ## The maximum number of occurrences of a given term to index per field diff --git a/configdefinitions/src/vespa/indexschema.def b/configdefinitions/src/vespa/indexschema.def index dfc13b463c7..5352e4d9009 100644 --- a/configdefinitions/src/vespa/indexschema.def +++ b/configdefinitions/src/vespa/indexschema.def @@ -1,6 +1,5 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. ## Config specifying the index fields and field collections that are part of an index schema. -version=7 namespace=vespa.config.search ## The name of the index field. diff --git a/configdefinitions/src/vespa/rank-profiles.def b/configdefinitions/src/vespa/rank-profiles.def index d0f19b9ac0d..525930c990b 100644 --- a/configdefinitions/src/vespa/rank-profiles.def +++ b/configdefinitions/src/vespa/rank-profiles.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=13 namespace=vespa.config.search ## name of this rank profile. maps to table index for internal use. diff --git a/configdefinitions/src/vespa/specialtokens.def b/configdefinitions/src/vespa/specialtokens.def index 2a84a0c7875..1e6addcf358 100644 --- a/configdefinitions/src/vespa/specialtokens.def +++ b/configdefinitions/src/vespa/specialtokens.def @@ -13,7 +13,6 @@ ## prefix of another token, Vespa will prefer to return the first ## matching token in the list when encountering the longest ## special token. -version=3 namespace=vespa.configdefinition ## Path to makefsa binary, needed to create specialtokens-dictionary diff --git a/configdefinitions/src/vespa/summary.def b/configdefinitions/src/vespa/summary.def index 90103da3c42..20ca6b10450 100644 --- a/configdefinitions/src/vespa/summary.def +++ b/configdefinitions/src/vespa/summary.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=2 namespace=vespa.config.search defaultsummaryid int default=-1 diff --git a/configgen/src/test/resources/allfeatures.def b/configgen/src/test/resources/allfeatures.def index bedb8cb6dbf..e13e14c9e36 100644 --- a/configgen/src/test/resources/allfeatures.def +++ b/configgen/src/test/resources/allfeatures.def @@ -18,7 +18,6 @@ # - Have an array within a struct, to verify that we correctly recurse. # - Reuse type name further within to ensure that this works. -version=11 namespace=configgen # Some random bool without a default value. These comments exist to check # that comment parsing works.e diff --git a/configgen/src/test/resources/bar.foo.def b/configgen/src/test/resources/bar.foo.def index b22be0c62dc..e0084b792a5 100644 --- a/configgen/src/test/resources/bar.foo.def +++ b/configgen/src/test/resources/bar.foo.def @@ -1,4 +1,3 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=1 namespace=baz xyzzy int default=10 diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/GlobalComponentRegistry.java b/configserver/src/main/java/com/yahoo/vespa/config/server/GlobalComponentRegistry.java index 1bda8dcb69a..4f26cfa265b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/GlobalComponentRegistry.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/GlobalComponentRegistry.java @@ -19,8 +19,7 @@ import java.util.Optional; /** * Interface representing all global config server components used within the config server. * - * @author lulf - * @since 5.1 + * @author Ulf Lilleengen */ public interface GlobalComponentRegistry { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistry.java b/configserver/src/main/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistry.java index f88a0ef1a2d..fa5224732f6 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistry.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistry.java @@ -22,8 +22,7 @@ import java.util.Optional; /** * Registry containing all the "static"/"global" components in a config server in one place. * - * @author lulf - * @since 5.1 + * @author Ulf Lilleengen */ public class InjectedGlobalComponentRegistry implements GlobalComponentRegistry { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java index 94abbc10046..0435c8e59db 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java @@ -9,8 +9,7 @@ import java.util.List; /** * The applications of a tenant * - * @author lulf - * @since 5.1 + * @author Ulf Lilleengen */ public interface TenantApplications { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ZKTenantApplications.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ZKTenantApplications.java index ac318aba8e8..5260dd9228c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ZKTenantApplications.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ZKTenantApplications.java @@ -29,8 +29,7 @@ import java.util.logging.Logger; * Each application is stored as a single file, named the same as the application id and containing the id * of the session storing the content of the application. * - * @author lulf - * @since 5.1 + * @author Ulf Lilleengen */ // TODO: Merge into interface and separate out curator layer instead public class ZKTenantApplications implements TenantApplications, PathChildrenCacheListener { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java index 61382af6a30..51995eb98cf 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java @@ -136,7 +136,7 @@ public class Deployment implements com.yahoo.config.provision.Deployment { transaction.add(deactivateCurrentActivateNew(localSessionRepo.getActiveSession(session.getApplicationId()), session, ignoreSessionStaleFailure)); if (hostProvisioner.isPresent()) { - hostProvisioner.get().activate(transaction, session.getApplicationId(), session.getProvisionInfo().getHosts()); + hostProvisioner.get().activate(transaction, session.getApplicationId(), session.getAllocatedHosts().getHosts()); } transaction.commit(); session.waitUntilActivated(timeoutBudget); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 820ad8e530c..70b677b4057 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -18,7 +18,7 @@ import java.util.Set; /** * Implementation of {@link ModelContext} for configserver. * - * @author lulf + * @author Ulf Lilleengen */ public class ModelContextImpl implements ModelContext { @@ -83,6 +83,11 @@ public class ModelContextImpl implements ModelContext { return permanentApplicationPackage; } + /** + * Returns the host provisioner to use, or empty to use the default provisioner, + * creating hosts from the application package defined hosts + */ + // TODO: Don't allow empty here but create the right provisioner when this is set up instead @Override public Optional<HostProvisioner> hostProvisioner() { return hostProvisioner; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java index ea278596e80..69266620e45 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java @@ -6,7 +6,7 @@ import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.UnparsedConfigDefinition; -import com.yahoo.config.provision.ProvisionInfo; +import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.Version; import com.yahoo.io.reader.NamedReader; import com.yahoo.log.LogLevel; @@ -25,13 +25,12 @@ import java.util.*; * A class used for reading and writing application data to zookeeper. * * @author hmusum - * @since 5.1 */ public class ZooKeeperClient { private final ConfigCurator configCurator; private final DeployLogger logger; - private final boolean trace; + private final boolean logFine; /* This is the generation that will be used for reading and writing application data. (1 more than last deployed application) */ private final Path rootPath; @@ -42,10 +41,10 @@ public class ZooKeeperClient { } }; - public ZooKeeperClient(ConfigCurator configCurator, DeployLogger logger, boolean trace, Path rootPath) { + public ZooKeeperClient(ConfigCurator configCurator, DeployLogger logger, boolean logFine, Path rootPath) { this.configCurator = configCurator; this.logger = logger; - this.trace = trace; + this.logFine = logFine; this.rootPath = rootPath; } @@ -62,7 +61,7 @@ public class ZooKeeperClient { try { while (retries > 0) { try { - trace("Setting up ZooKeeper nodes for this application"); + logFine("Setting up ZooKeeper nodes for this application"); createZooKeeperNodes(); break; } catch (RuntimeException e) { @@ -105,28 +104,28 @@ public class ZooKeeperClient { * * @param app the application package to feed to zookeeper */ - void feedZooKeeper(ApplicationPackage app) { - trace("Feeding application config into ZooKeeper"); + void write(ApplicationPackage app) { + logFine("Feeding application config into ZooKeeper"); // gives lots and lots of debug output: // BasicConfigurator.configure(); try { - trace("zk operations: " + configCurator.getNumberOfOperations()); - trace("zk operations: " + configCurator.getNumberOfOperations()); - trace("Feeding user def files into ZooKeeper"); - feedZKUserDefs(app); - trace("zk operations: " + configCurator.getNumberOfOperations()); - trace("Feeding application package into ZooKeeper"); + logFine("zk operations: " + configCurator.getNumberOfOperations()); + logFine("zk operations: " + configCurator.getNumberOfOperations()); + logFine("Feeding user def files into ZooKeeper"); + writeUserDefs(app); + logFine("zk operations: " + configCurator.getNumberOfOperations()); + logFine("Feeding application package into ZooKeeper"); // TODO 1200 zk operations done in the below method - feedZKAppPkg(app); - feedSearchDefinitions(app); - feedZKUserIncludeDirs(app, app.getUserIncludeDirs()); - trace("zk operations: " + configCurator.getNumberOfOperations()); - trace("zk read operations: " + configCurator.getNumberOfReadOperations()); - trace("zk write operations: " + configCurator.getNumberOfWriteOperations()); - trace("Feeding sd from docproc bundle into ZooKeeper"); - trace("zk operations: " + configCurator.getNumberOfOperations()); - trace("Write application metadata into ZooKeeper"); - feedZKAppMetaData(app.getMetaData()); - trace("zk operations: " + configCurator.getNumberOfOperations()); + writeSomeOf(app); + writeSearchDefinitions(app); + writeUserIncludeDirs(app, app.getUserIncludeDirs()); + logFine("zk operations: " + configCurator.getNumberOfOperations()); + logFine("zk read operations: " + configCurator.getNumberOfReadOperations()); + logFine("zk write operations: " + configCurator.getNumberOfWriteOperations()); + logFine("Feeding sd from docproc bundle into ZooKeeper"); + logFine("zk operations: " + configCurator.getNumberOfOperations()); + logFine("Write application metadata into ZooKeeper"); + write(app.getMetaData()); + logFine("zk operations: " + configCurator.getNumberOfOperations()); } catch (Exception e) { throw new IllegalStateException("Unable to write vespa model to config server(s) " + System.getProperty("configsources") + "\n" + "Please ensure that cloudconfig_server is started on the config server node(s), " + @@ -134,7 +133,7 @@ public class ZooKeeperClient { } } - private void feedSearchDefinitions(ApplicationPackage app) throws IOException { + private void writeSearchDefinitions(ApplicationPackage app) throws IOException { Collection<NamedReader> sds = app.getSearchDefinitions(); if (sds.isEmpty()) { return; @@ -142,7 +141,7 @@ public class ZooKeeperClient { Path zkPath = getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH).append(ApplicationPackage.SEARCH_DEFINITIONS_DIR); configCurator.createNode(zkPath.getAbsolute()); // Ensures that ranking expressions and other files are also fed. - feedDirZooKeeper(app.getFile(ApplicationPackage.SEARCH_DEFINITIONS_DIR), zkPath, false); + writeDir(app.getFile(ApplicationPackage.SEARCH_DEFINITIONS_DIR), zkPath, false); for (NamedReader sd : sds) { String name = sd.getName(); Reader reader = sd.getReader(); @@ -153,12 +152,12 @@ public class ZooKeeperClient { } /** - * Puts the application package files into ZK + * Puts some of the application package files into ZK - see write(app). * * @param app The application package to use as input. * @throws java.io.IOException if not able to write to Zookeeper */ - void feedZKAppPkg(ApplicationPackage app) throws IOException { + void writeSomeOf(ApplicationPackage app) throws IOException { ApplicationFile.PathFilter srFilter = new ApplicationFile.PathFilter() { @Override public boolean accept(Path path) { @@ -167,40 +166,40 @@ public class ZooKeeperClient { }; // Copy app package files and subdirs into zk // TODO: We should have a way of doing this which doesn't require repeating all the content - feedFileZooKeeper(app.getFile(Path.fromString(ApplicationPackage.SERVICES)), - getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH)); - feedFileZooKeeper(app.getFile(Path.fromString(ApplicationPackage.HOSTS)), - getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH)); - feedFileZooKeeper(app.getFile(Path.fromString(ApplicationPackage.DEPLOYMENT_FILE.getName())), - getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH)); - feedFileZooKeeper(app.getFile(Path.fromString(ApplicationPackage.VALIDATION_OVERRIDES.getName())), - getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH)); + writeFile(app.getFile(Path.fromString(ApplicationPackage.SERVICES)), + getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH)); + writeFile(app.getFile(Path.fromString(ApplicationPackage.HOSTS)), + getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH)); + writeFile(app.getFile(Path.fromString(ApplicationPackage.DEPLOYMENT_FILE.getName())), + getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH)); + writeFile(app.getFile(Path.fromString(ApplicationPackage.VALIDATION_OVERRIDES.getName())), + getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH)); - feedDirZooKeeper(app.getFile(Path.fromString(ApplicationPackage.TEMPLATES_DIR)), - getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH), - true); - feedDirZooKeeper(app.getFile(ApplicationPackage.RULES_DIR), - getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH).append(ApplicationPackage.RULES_DIR), - srFilter, true); - feedDirZooKeeper(app.getFile(ApplicationPackage.QUERY_PROFILES_DIR), - getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH).append(ApplicationPackage.QUERY_PROFILES_DIR), - xmlFilter, true); - feedDirZooKeeper(app.getFile(ApplicationPackage.PAGE_TEMPLATES_DIR), - getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH).append(ApplicationPackage.PAGE_TEMPLATES_DIR), - xmlFilter, true); - feedDirZooKeeper(app.getFile(Path.fromString(ApplicationPackage.SEARCHCHAINS_DIR)), - getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH).append(ApplicationPackage.SEARCHCHAINS_DIR), - xmlFilter, true); - feedDirZooKeeper(app.getFile(Path.fromString(ApplicationPackage.DOCPROCCHAINS_DIR)), - getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH).append(ApplicationPackage.DOCPROCCHAINS_DIR), - xmlFilter, true); - feedDirZooKeeper(app.getFile(Path.fromString(ApplicationPackage.ROUTINGTABLES_DIR)), - getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH).append(ApplicationPackage.ROUTINGTABLES_DIR), - xmlFilter, true); + writeDir(app.getFile(Path.fromString(ApplicationPackage.TEMPLATES_DIR)), + getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH), + true); + writeDir(app.getFile(ApplicationPackage.RULES_DIR), + getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH).append(ApplicationPackage.RULES_DIR), + srFilter, true); + writeDir(app.getFile(ApplicationPackage.QUERY_PROFILES_DIR), + getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH).append(ApplicationPackage.QUERY_PROFILES_DIR), + xmlFilter, true); + writeDir(app.getFile(ApplicationPackage.PAGE_TEMPLATES_DIR), + getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH).append(ApplicationPackage.PAGE_TEMPLATES_DIR), + xmlFilter, true); + writeDir(app.getFile(Path.fromString(ApplicationPackage.SEARCHCHAINS_DIR)), + getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH).append(ApplicationPackage.SEARCHCHAINS_DIR), + xmlFilter, true); + writeDir(app.getFile(Path.fromString(ApplicationPackage.DOCPROCCHAINS_DIR)), + getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH).append(ApplicationPackage.DOCPROCCHAINS_DIR), + xmlFilter, true); + writeDir(app.getFile(Path.fromString(ApplicationPackage.ROUTINGTABLES_DIR)), + getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH).append(ApplicationPackage.ROUTINGTABLES_DIR), + xmlFilter, true); } - private void feedDirZooKeeper(ApplicationFile file, Path zooKeeperAppPath, boolean recurse) throws IOException { - feedDirZooKeeper(file, zooKeeperAppPath, new ApplicationFile.PathFilter() { + private void writeDir(ApplicationFile file, Path zooKeeperAppPath, boolean recurse) throws IOException { + writeDir(file, zooKeeperAppPath, new ApplicationFile.PathFilter() { @Override public boolean accept(Path path) { return true; @@ -208,7 +207,7 @@ public class ZooKeeperClient { }, recurse); } - private void feedDirZooKeeper(ApplicationFile dir, Path path, ApplicationFile.PathFilter filenameFilter, boolean recurse) throws IOException { + private void writeDir(ApplicationFile dir, Path path, ApplicationFile.PathFilter filenameFilter, boolean recurse) throws IOException { if (!dir.isDirectory()) { logger.log(LogLevel.FINE, dir.getPath().getAbsolute()+" is not a directory. Not feeding the files into ZooKeeper."); return; @@ -220,10 +219,10 @@ public class ZooKeeperClient { if (file.isDirectory()) { configCurator.createNode(path.append(name).getAbsolute()); if (recurse) { - feedDirZooKeeper(file, path.append(name), filenameFilter, recurse); + writeDir(file, path.append(name), filenameFilter, recurse); } } else { - feedFileZooKeeper(file, path); + writeFile(file, path); } } } @@ -248,7 +247,7 @@ public class ZooKeeperClient { return ret; } - private void feedFileZooKeeper(ApplicationFile file, Path zkPath) throws IOException { + private void writeFile(ApplicationFile file, Path zkPath) throws IOException { if (!file.exists()) { return; } @@ -260,7 +259,7 @@ public class ZooKeeperClient { } } - private void feedZKUserIncludeDirs(ApplicationPackage applicationPackage, List<String> userIncludeDirs) throws IOException { + private void writeUserIncludeDirs(ApplicationPackage applicationPackage, List<String> userIncludeDirs) throws IOException { // User defined include directories for (String userInclude : userIncludeDirs) { ApplicationFile dir = applicationPackage.getFile(Path.fromString(userInclude)); @@ -268,9 +267,9 @@ public class ZooKeeperClient { if (files == null || files.isEmpty()) { configCurator.createNode(getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + userInclude).getAbsolute()); } - feedDirZooKeeper(dir, - getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + userInclude), - xmlFilter, true); + writeDir(dir, + getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + userInclude), + xmlFilter, true); } } @@ -278,22 +277,22 @@ public class ZooKeeperClient { * Feeds all user-defined .def file from the application package into ZooKeeper (both into * /defconfigs and /userdefconfigs */ - private void feedZKUserDefs(ApplicationPackage applicationPackage) { + private void writeUserDefs(ApplicationPackage applicationPackage) { Map<ConfigDefinitionKey, UnparsedConfigDefinition> configDefs = applicationPackage.getAllExistingConfigDefs(); for (Map.Entry<ConfigDefinitionKey, UnparsedConfigDefinition> entry : configDefs.entrySet()) { ConfigDefinitionKey key = entry.getKey(); String contents = entry.getValue().getUnparsedContent(); - feedDefToZookeeper(key.getName(), key.getNamespace(), getZooKeeperAppPath(ConfigCurator.USER_DEFCONFIGS_ZK_SUBPATH).getAbsolute(), contents); - feedDefToZookeeper(key.getName(), key.getNamespace(), getZooKeeperAppPath(ConfigCurator.DEFCONFIGS_ZK_SUBPATH).getAbsolute(), contents); + write(key.getName(), key.getNamespace(), getZooKeeperAppPath(ConfigCurator.USER_DEFCONFIGS_ZK_SUBPATH).getAbsolute(), contents); + write(key.getName(), key.getNamespace(), getZooKeeperAppPath(ConfigCurator.DEFCONFIGS_ZK_SUBPATH).getAbsolute(), contents); } logger.log(LogLevel.FINE, configDefs.size() + " user config definitions"); } - private void feedDefToZookeeper(String name, String namespace, String path, String data) { - feedDefToZookeeper(name, namespace, "", path, com.yahoo.text.Utf8.toBytes(data)); + private void write(String name, String namespace, String path, String data) { + write(name, namespace, "", path, com.yahoo.text.Utf8.toBytes(data)); } - private void feedDefToZookeeper(String name, String namespace, String version, String path, byte[] data) { + private void write(String name, String namespace, String version, String path, byte[] data) { configCurator.putDefData( ("".equals(namespace)) ? name : (namespace + "." + name), version, @@ -301,8 +300,8 @@ public class ZooKeeperClient { data); } - private void feedZKFileRegistry(Version vespaVersion, FileRegistry fileRegistry) { - trace("Feeding file registry data into ZooKeeper"); + private void write(Version vespaVersion, FileRegistry fileRegistry) { + logFine("Feeding file registry data into ZooKeeper"); String exportedRegistry = PreGeneratedFileRegistry.exportRegistry(fileRegistry); configCurator.putData(getZooKeeperAppPath(null).append(ZKApplicationPackage.fileRegistryNode).getAbsolute(), @@ -316,12 +315,12 @@ public class ZooKeeperClient { * * @param metaData The application metadata. */ - private void feedZKAppMetaData(ApplicationMetaData metaData) { + private void write(ApplicationMetaData metaData) { configCurator.putData(getZooKeeperAppPath(ConfigCurator.META_ZK_PATH).getAbsolute(), metaData.asJsonString()); } void cleanupZooKeeper() { - trace("Exception occurred. Cleaning up ZooKeeper"); + logFine("Exception occurred. Cleaning up ZooKeeper"); try { for (String subPath : Arrays.asList( ConfigCurator.DEFCONFIGS_ZK_SUBPATH, @@ -350,26 +349,20 @@ public class ZooKeeperClient { } } - void trace(String msg) { - if (trace) { + void logFine(String msg) { + if (logFine) { logger.log(LogLevel.FINE, msg); } } - private void feedProvisionInfo(Version version, ProvisionInfo info) throws IOException { - byte[] json = info.toJson(); - configCurator.putData(rootPath.append(ZKApplicationPackage.allocatedHostsNode).append(version.toSerializedForm()).getAbsolute(), json); + public void write(AllocatedHosts info) throws IOException { + configCurator.putData(rootPath.append(ZKApplicationPackage.allocatedHostsNode).getAbsolute(), info.toJson()); } - public void feedZKFileRegistries(Map<Version, FileRegistry> fileRegistryMap) { + public void write(Map<Version, FileRegistry> fileRegistryMap) { for (Map.Entry<Version, FileRegistry> versionFileRegistryEntry : fileRegistryMap.entrySet()) { - feedZKFileRegistry(versionFileRegistryEntry.getKey(), versionFileRegistryEntry.getValue()); + write(versionFileRegistryEntry.getKey(), versionFileRegistryEntry.getValue()); } } - public void feedProvisionInfos(Map<Version, ProvisionInfo> provisionInfoMap) throws IOException { - for (Map.Entry<Version, ProvisionInfo> versionProvisionInfoEntry : provisionInfoMap.entrySet()) { - feedProvisionInfo(versionProvisionInfoEntry.getKey(), versionProvisionInfoEntry.getValue()); - } - } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperDeployer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperDeployer.java index 246d7226cfd..22ce952481d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperDeployer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperDeployer.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.config.server.deploy; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.FileRegistry; -import com.yahoo.config.provision.ProvisionInfo; +import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.Version; import java.io.IOException; @@ -14,7 +14,6 @@ import java.util.Map; * Initialize must be called before each deploy. * * @author lulf - * @since 5.1 */ public class ZooKeeperDeployer { @@ -28,15 +27,16 @@ public class ZooKeeperDeployer { * Deploys an application package to zookeeper. initialize() must be called before calling this method. * * @param applicationPackage The application package to persist. - * @param fileRegistryMap The file registries to persist. - * @param provisionInfoMap The provisioning infos to persist. + * @param fileRegistryMap the file registries to persist. + * @param allocatedHosts the provisioning info to persist. * @throws IOException if deploying fails */ - public void deploy(ApplicationPackage applicationPackage, Map<Version, FileRegistry> fileRegistryMap, Map<Version, ProvisionInfo> provisionInfoMap) throws IOException { + public void deploy(ApplicationPackage applicationPackage, Map<Version, FileRegistry> fileRegistryMap, + AllocatedHosts allocatedHosts) throws IOException { zooKeeperClient.setupZooKeeper(); - zooKeeperClient.feedZooKeeper(applicationPackage); - zooKeeperClient.feedZKFileRegistries(fileRegistryMap); - zooKeeperClient.feedProvisionInfos(provisionInfoMap); + zooKeeperClient.write(applicationPackage); + zooKeeperClient.write(fileRegistryMap); + zooKeeperClient.write(allocatedHosts); } public void cleanup() { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/host/HostRegistry.java b/configserver/src/main/java/com/yahoo/vespa/config/server/host/HostRegistry.java index 36e7737163a..e61fc124d53 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/host/HostRegistry.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/host/HostRegistry.java @@ -16,7 +16,6 @@ import com.yahoo.log.LogLevel; * TODO: Is there a generalized version of this pattern? Need some sort mix of Bimap and Multimap * * @author lulf - * @since 5.3 */ public class HostRegistry<T> implements HostValidator<T> { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java index bbd1e189e29..1301f24788f 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java @@ -5,12 +5,11 @@ import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.api.ConfigDefinitionRepo; -import com.yahoo.config.model.api.HostProvisioner; import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.api.ModelFactory; import com.yahoo.config.model.application.provider.MockFileRegistry; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ProvisionInfo; +import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Version; import com.yahoo.config.provision.Zone; @@ -24,7 +23,6 @@ import com.yahoo.vespa.config.server.application.PermanentApplicationPackage; import com.yahoo.vespa.config.server.deploy.ModelContextImpl; import com.yahoo.vespa.config.server.monitoring.MetricUpdater; import com.yahoo.vespa.config.server.monitoring.Metrics; -import com.yahoo.vespa.config.server.provision.StaticProvisioner; import com.yahoo.vespa.config.server.session.SessionZooKeeperClient; import com.yahoo.vespa.config.server.session.SilentDeployLogger; import com.yahoo.vespa.curator.Curator; @@ -47,7 +45,6 @@ public class ActivatedModelsBuilder extends ModelsBuilder<Application> { private final long appGeneration; private final SessionZooKeeperClient zkClient; private final Optional<PermanentApplicationPackage> permanentApplicationPackage; - private final Optional<com.yahoo.config.provision.Provisioner> hostProvisioner; private final ConfigserverConfig configserverConfig; private final ConfigDefinitionRepo configDefinitionRepo; private final Metrics metrics; @@ -56,7 +53,8 @@ public class ActivatedModelsBuilder extends ModelsBuilder<Application> { private final DeployLogger logger; public ActivatedModelsBuilder(TenantName tenant, long appGeneration, SessionZooKeeperClient zkClient, GlobalComponentRegistry globalComponentRegistry) { - super(globalComponentRegistry.getModelFactoryRegistry()); + super(globalComponentRegistry.getModelFactoryRegistry(), + globalComponentRegistry.getHostProvisioner().isPresent()); this.tenant = tenant; this.appGeneration = appGeneration; this.zkClient = zkClient; @@ -64,17 +62,17 @@ public class ActivatedModelsBuilder extends ModelsBuilder<Application> { this.configserverConfig = globalComponentRegistry.getConfigserverConfig(); this.configDefinitionRepo = globalComponentRegistry.getConfigDefinitionRepo(); this.metrics = globalComponentRegistry.getMetrics(); - this.hostProvisioner = globalComponentRegistry.getHostProvisioner(); this.curator = globalComponentRegistry.getCurator(); this.zone = globalComponentRegistry.getZone(); this.logger = new SilentDeployLogger(); } @Override - protected Application buildModelVersion(ModelFactory modelFactory, + protected Application buildModelVersion(ModelFactory modelFactory, ApplicationPackage applicationPackage, - ApplicationId applicationId, + ApplicationId applicationId, com.yahoo.component.Version wantedNodeVespaVersion, + Optional<AllocatedHosts> ignored, // Ignored since we have this in the app package for activated models Instant now) { log.log(LogLevel.DEBUG, String.format("Loading model version %s for session %s application %s", modelFactory.getVersion(), appGeneration, applicationId)); @@ -86,7 +84,7 @@ public class ActivatedModelsBuilder extends ModelsBuilder<Application> { logger, configDefinitionRepo, getForVersionOrLatest(applicationPackage.getFileRegistryMap(), modelFactory.getVersion()).orElse(new MockFileRegistry()), - createHostProvisioner(getForVersionOrLatest(applicationPackage.getProvisionInfoMap(), modelFactory.getVersion())), + createStaticProvisioner(applicationPackage.getAllocatedHosts()), createModelContextProperties(applicationId), Optional.empty(), new com.yahoo.component.Version(modelFactory.getVersion().toString()), @@ -96,17 +94,6 @@ public class ActivatedModelsBuilder extends ModelsBuilder<Application> { applicationMetricUpdater, applicationId); } - private Optional<HostProvisioner> createHostProvisioner(Optional<ProvisionInfo> provisionInfo) { - if (hostProvisioner.isPresent() && provisionInfo.isPresent()) { - return Optional.of(createStaticProvisioner(provisionInfo.get())); - } - return Optional.empty(); - } - - private HostProvisioner createStaticProvisioner(ProvisionInfo provisionInfo) { - return new StaticProvisioner(provisionInfo); - } - private static <T> Optional<T> getForVersionOrLatest(Map<Version, T> map, Version version) { if (map.isEmpty()) { return Optional.empty(); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelFactoryRegistry.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelFactoryRegistry.java index ab1c84eb5dd..1c2c9c731d1 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelFactoryRegistry.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelFactoryRegistry.java @@ -13,7 +13,7 @@ import java.util.*; * A registry of model factories. Allows querying for a specific version of a {@link ModelFactory} or * simply returning all of them. Keeps track of the latest {@link com.yahoo.config.provision.Version} supported. * - * @author lulf + * @author Ulf Lilleengen */ public class ModelFactoryRegistry { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java index cc7a12801d3..6a4ab40d843 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java @@ -3,16 +3,20 @@ package com.yahoo.vespa.config.server.modelfactory; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.api.HostProvisioner; import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.api.ModelFactory; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.OutOfCapacityException; +import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.Rotation; import com.yahoo.config.provision.Version; import com.yahoo.config.provision.Zone; +import com.yahoo.lang.SettableOptional; import com.yahoo.vespa.config.server.ConfigServerSpec; import com.yahoo.vespa.config.server.deploy.ModelContextImpl; import com.yahoo.vespa.config.server.http.UnknownVespaVersionException; +import com.yahoo.vespa.config.server.provision.StaticProvisioner; import java.time.Instant; import java.util.ArrayList; @@ -37,13 +41,24 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> { private final ModelFactoryRegistry modelFactoryRegistry; - protected ModelsBuilder(ModelFactoryRegistry modelFactoryRegistry) { + /** True if we are running in hosted mode */ + private final boolean hosted; + + protected ModelsBuilder(ModelFactoryRegistry modelFactoryRegistry, boolean hosted) { this.modelFactoryRegistry = modelFactoryRegistry; + this.hosted = hosted; } + /** + * Builds all applicable model versions + * + * @param allocatedHosts the newest version (major and minor) (which is loaded first) decides the allocated hosts + * and assigns to this SettableOptional such that it can be used after this method returns + */ public List<MODELRESULT> buildModels(ApplicationId applicationId, com.yahoo.component.Version wantedNodeVespaVersion, ApplicationPackage applicationPackage, + SettableOptional<AllocatedHosts> allocatedHosts, Instant now) { Set<Version> versions = modelFactoryRegistry.allVersions(); @@ -52,7 +67,7 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> { if (requestedMajorVersion.isPresent()) versions = filterByMajorVersion(requestedMajorVersion.get(), versions); - // Load models by one major version at the time as new major versions are allowed to be unloadable + // Load models by one major version at the time as new major versions are allowed to be non-loadable // in the case where an existing application is incompatible with a new major version // (which is possible by the definition of major) List<Integer> majorVersions = versions.stream() @@ -65,7 +80,8 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> { for (int i = 0; i < majorVersions.size(); i++) { try { allApplicationModels.addAll(buildModelVersion(filterByMajorVersion(majorVersions.get(i), versions), - applicationId, wantedNodeVespaVersion, applicationPackage, now)); + applicationId, wantedNodeVespaVersion, applicationPackage, + allocatedHosts, now)); // skip old config models if requested after we have found a major version which works if (allApplicationModels.size() > 0 && allApplicationModels.get(0).getModel().skipOldConfigModels(now)) @@ -91,30 +107,43 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> { private List<MODELRESULT> buildModelVersion(Set<Version> versions, ApplicationId applicationId, com.yahoo.component.Version wantedNodeVespaVersion, ApplicationPackage applicationPackage, + SettableOptional<AllocatedHosts> allocatedHosts, Instant now) { Version latest = findLatest(versions); // load latest application version - MODELRESULT latestApplicationVersion = buildModelVersion(modelFactoryRegistry.getFactory(latest), + MODELRESULT latestModelVersion = buildModelVersion(modelFactoryRegistry.getFactory(latest), applicationPackage, applicationId, wantedNodeVespaVersion, + allocatedHosts.asOptional(), now); - if (latestApplicationVersion.getModel().skipOldConfigModels(now)) { - return Collections.singletonList(latestApplicationVersion); - } - else { // load old model versions - List<MODELRESULT> allApplicationVersions = new ArrayList<>(); - allApplicationVersions.add(latestApplicationVersion); - for (Version version : versions) { - if (version.equals(latest)) continue; // already loaded - allApplicationVersions.add(buildModelVersion(modelFactoryRegistry.getFactory(version), - applicationPackage, - applicationId, - wantedNodeVespaVersion, - now)); - } - return allApplicationVersions; + allocatedHosts.set(latestModelVersion.getModel().allocatedHosts()); // Update with additional clusters allocated + + if (latestModelVersion.getModel().skipOldConfigModels(now)) + return Collections.singletonList(latestModelVersion); + + // load old model versions + List<MODELRESULT> allApplicationVersions = new ArrayList<>(); + allApplicationVersions.add(latestModelVersion); + + // TODO: We use the allocated hosts from the newest version when building older model versions. + // This is correct except for the case where an old model specifies a cluster which the new version + // does not. In that case we really want to extend the set of allocated hosts to include those of that + // cluster as well. To do that, create a new provisioner which uses static provisioning for known + // clusters and the node repository provisioner as fallback. + for (Version version : versions) { + if (version.equals(latest)) continue; // already loaded + + MODELRESULT modelVersion = buildModelVersion(modelFactoryRegistry.getFactory(version), + applicationPackage, + applicationId, + wantedNodeVespaVersion, + allocatedHosts.asOptional(), + now); + allocatedHosts.set(modelVersion.getModel().allocatedHosts()); // Update with additional clusters allocated + allApplicationVersions.add(modelVersion); } + return allApplicationVersions; } private Set<Version> filterByMajorVersion(int majorVersion, Set<Version> versions) { @@ -133,6 +162,7 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> { protected abstract MODELRESULT buildModelVersion(ModelFactory modelFactory, ApplicationPackage applicationPackage, ApplicationId applicationId, com.yahoo.component.Version wantedNodeVespaVersion, + Optional<AllocatedHosts> allocatedHosts, Instant now); protected ModelContext.Properties createModelContextProperties(ApplicationId applicationId, @@ -147,4 +177,15 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> { rotations); } + /** + * Returns a host provisioner returning the previously allocated hosts if available and when on hosted Vespa, + * returns empty otherwise, which may either mean that no hosts are allocated or that we are running + * non-hosted and should default to use hosts defined in the application package, depending on context + */ + protected Optional<HostProvisioner> createStaticProvisioner(Optional<AllocatedHosts> allocatedHosts) { + if (hosted && allocatedHosts.isPresent()) + return Optional.of(new StaticProvisioner(allocatedHosts.get())); + return Optional.empty(); + } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java index b2dca075c4f..78d660b347e 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java @@ -12,6 +12,7 @@ import com.yahoo.config.model.api.ModelCreateResult; import com.yahoo.config.model.api.ModelFactory; import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.Version; import com.yahoo.log.LogLevel; import com.yahoo.vespa.config.server.application.ApplicationSet; @@ -21,6 +22,7 @@ import com.yahoo.vespa.config.server.deploy.ModelContextImpl; import com.yahoo.vespa.config.server.filedistribution.FileDistributionProvider; import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; import com.yahoo.vespa.config.server.provision.ProvisionerAdapter; +import com.yahoo.vespa.config.server.provision.StaticProvisioner; import com.yahoo.vespa.config.server.session.FileDistributionFactory; import com.yahoo.vespa.config.server.session.PrepareParams; import com.yahoo.vespa.config.server.session.SessionContext; @@ -60,7 +62,7 @@ public class PreparedModelsBuilder extends ModelsBuilder<PreparedModelsBuilder.P PrepareParams params, Optional<ApplicationSet> currentActiveApplicationSet, ModelContext.Properties properties) { - super(modelFactoryRegistry); + super(modelFactoryRegistry, properties.hostedVespa()); this.permanentApplicationPackage = permanentApplicationPackage; this.configDefinitionRepo = configDefinitionRepo; @@ -79,14 +81,17 @@ public class PreparedModelsBuilder extends ModelsBuilder<PreparedModelsBuilder.P protected PreparedModelResult buildModelVersion(ModelFactory modelFactory, ApplicationPackage applicationPackage, ApplicationId applicationId, - com.yahoo.component.Version wantedNodeVespaVersion, Instant now) { + com.yahoo.component.Version wantedNodeVespaVersion, + Optional<AllocatedHosts> allocatedHosts, + Instant now) { Version modelVersion = modelFactory.getVersion(); log.log(LogLevel.DEBUG, "Building model " + modelVersion + " for " + applicationId); FileDistributionProvider fileDistributionProvider = fileDistributionFactory.createProvider( context.getServerDBSessionDir(), applicationId); - Optional<HostProvisioner> hostProvisioner = createHostProvisionerAdapter(properties); + // Use empty on non-hosted systems, use already allocated hosts if available, create connection to a host provisioner otherwise + Optional<HostProvisioner> hostProvisioner = createHostProvisioner(allocatedHosts); Optional<Model> previousModel = currentActiveApplicationSet .map(set -> set.getForVersionOrLatest(Optional.of(modelVersion), now).getModel()); ModelContext modelContext = new ModelContextImpl( @@ -109,6 +114,24 @@ public class PreparedModelsBuilder extends ModelsBuilder<PreparedModelsBuilder.P return new PreparedModelsBuilder.PreparedModelResult(modelVersion, result.getModel(), fileDistributionProvider, result.getConfigChangeActions()); } + // This method is an excellent demonstration of what happens when one is too liberal with Optional + // -bratseth, who had to write the below :-\ + private Optional<HostProvisioner> createHostProvisioner(Optional<AllocatedHosts> allocatedHosts) { + Optional<HostProvisioner> nodeRepositoryProvisioner = createNodeRepositoryProvisioner(properties); + if ( ! allocatedHosts.isPresent()) return nodeRepositoryProvisioner; + + Optional<HostProvisioner> staticProvisioner = createStaticProvisioner(allocatedHosts); + if ( ! staticProvisioner.isPresent()) return Optional.empty(); // Since we have hosts allocated this means we are on non-hosted + + // The following option should not be possible, but since there is a right action for it we can take it + if ( ! nodeRepositoryProvisioner.isPresent()) + return Optional.of(new StaticProvisioner(allocatedHosts.get())); + + // Nodes are already allocated by a model and we should use them unless this model requests hosts from a + // previously unallocated cluster. This allows future models to stop allocate certain clusters. + return Optional.of(new StaticProvisioner(allocatedHosts.get(), nodeRepositoryProvisioner.get())); + } + private Optional<File> getAppDir(ApplicationPackage applicationPackage) { try { return applicationPackage instanceof FilesApplicationPackage ? @@ -124,7 +147,7 @@ public class PreparedModelsBuilder extends ModelsBuilder<PreparedModelsBuilder.P .collect(Collectors.toList())); } - private Optional<HostProvisioner> createHostProvisionerAdapter(ModelContext.Properties properties) { + private Optional<HostProvisioner> createNodeRepositoryProvisioner(ModelContext.Properties properties) { return hostProvisionerProvider.getHostProvisioner().map( provisioner -> new ProvisionerAdapter(provisioner, properties.applicationId())); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/provision/HostProvisionerProvider.java b/configserver/src/main/java/com/yahoo/vespa/config/server/provision/HostProvisionerProvider.java index 619ee7499e2..32a641a3bc7 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/provision/HostProvisionerProvider.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/provision/HostProvisionerProvider.java @@ -13,21 +13,18 @@ import java.util.logging.Logger; /** * This class is necessary to support both having and not having a host provisioner. We inject * a component registry here, which then enables us to check whether or not we have a provisioner available. + * We only have a provisioner if we are running in hosted mode. * - * @author lulf - * @since 5.15 + * @author Ulf Lilleengen */ public class HostProvisionerProvider { - private static final Logger log = Logger.getLogger(HostProvisionerProvider.class.getName()); private final Optional<Provisioner> hostProvisioner; public HostProvisionerProvider(ComponentRegistry<Provisioner> hostProvisionerRegistry, ConfigserverConfig configserverConfig) { if (hostProvisionerRegistry.allComponents().isEmpty() || ! configserverConfig.hostedVespa()) { - log.info("Host provisioner is missing, provisioner component count: " + hostProvisionerRegistry.allComponents().size() + ", is hosted Vespa: " + configserverConfig.hostedVespa()); hostProvisioner = Optional.empty(); } else { - log.log(LogLevel.DEBUG, "Host provisioner injected. Will be used for all deployments"); hostProvisioner = Optional.of(hostProvisionerRegistry.allComponents().get(0)); } } @@ -36,6 +33,7 @@ public class HostProvisionerProvider { this(componentRegistry, new ConfigserverConfig(new ConfigserverConfig.Builder())); } + /** Returns the host provisioner, or empty if we are not in hosted mode */ public Optional<Provisioner> getHostProvisioner() { return hostProvisioner; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/provision/ProvisionerAdapter.java b/configserver/src/main/java/com/yahoo/vespa/config/server/provision/ProvisionerAdapter.java index ad0ee3b3eb9..32380b296dd 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/provision/ProvisionerAdapter.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/provision/ProvisionerAdapter.java @@ -16,8 +16,7 @@ import java.util.*; * behavior to the config model. Adapts interface from a {@link HostProvisioner} to a * {@link Provisioner}. * - * @author lulf - * @since 5.11 + * @author Ulf Lilleengen */ public class ProvisionerAdapter implements HostProvisioner { @@ -31,6 +30,7 @@ public class ProvisionerAdapter implements HostProvisioner { @Override public HostSpec allocateHost(String alias) { + // Wow. Such mess. TODO: Actually support polymorphy or stop pretending to, see also ModelContextImpl.getHostProvisioner throw new UnsupportedOperationException("Allocating a single host in a hosted environment is not possible"); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/provision/StaticProvisioner.java b/configserver/src/main/java/com/yahoo/vespa/config/server/provision/StaticProvisioner.java index 1cad735879a..7e97690331f 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/provision/StaticProvisioner.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/provision/StaticProvisioner.java @@ -5,19 +5,35 @@ import com.yahoo.config.model.api.HostProvisioner; import com.yahoo.config.provision.*; import java.util.List; +import java.util.Optional; import java.util.stream.Collectors; /** - * Host provisioning from an existing {@link ProvisionInfo} instance. + * Host provisioning from an existing {@link AllocatedHosts} instance. * * @author bratseth */ public class StaticProvisioner implements HostProvisioner { - private final ProvisionInfo provisionInfo; + private final AllocatedHosts allocatedHosts; + + /** The fallback provisioner to use for unknown clusters, or null to not fall back */ + private final HostProvisioner fallback; - public StaticProvisioner(ProvisionInfo provisionInfo) { - this.provisionInfo = provisionInfo; + /** + * Creates a static host provisioner with no fallback + */ + public StaticProvisioner(AllocatedHosts allocatedHosts) { + this(allocatedHosts, null); + } + + /** + * Creates a static host provisioner which will fall back to using the given provisioner + * if a request is made for nodes in a cluster which is not present in this allocation. + */ + public StaticProvisioner(AllocatedHosts allocatedHosts, HostProvisioner fallback) { + this.allocatedHosts = allocatedHosts; + this.fallback = fallback; } @Override @@ -27,9 +43,14 @@ public class StaticProvisioner implements HostProvisioner { @Override public List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, int groups, ProvisionLogger logger) { - return provisionInfo.getHosts().stream() - .filter(host -> host.membership().isPresent() && matches(host.membership().get().cluster(), cluster)) - .collect(Collectors.toList()); + List<HostSpec> hostsAlreadyAllocatedToCluster = + allocatedHosts.getHosts().stream() + .filter(host -> host.membership().isPresent() && matches(host.membership().get().cluster(), cluster)) + .collect(Collectors.toList()); + if ( ! hostsAlreadyAllocatedToCluster.isEmpty()) + return hostsAlreadyAllocatedToCluster; + else + return fallback.prepare(cluster, capacity, groups, logger); } private boolean matches(ClusterSpec nodeCluster, ClusterSpec requestedCluster) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java index a222182f87a..308ca31f278 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java @@ -6,7 +6,7 @@ import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.ApplicationMetaData; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.config.provision.ProvisionInfo; +import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.transaction.AbstractTransaction; import com.yahoo.transaction.NestedTransaction; import com.yahoo.transaction.Transaction; @@ -30,11 +30,10 @@ import java.util.Optional; * prepared. Deleting a local session will ensure that the local filesystem state and global zookeeper state is * cleaned for this session. * - * @author lulf - * @since 5.1 + * @author Ulf Lilleengen */ // This is really the store of an application, whether it is active or in an edit session -// TODO: Separate the "application store" and "session" aspects - the latter belongs in the HTTP layer +// TODO: Separate the "application store" and "session" aspects - the latter belongs in the HTTP layer -bratseth public class LocalSession extends Session implements Comparable<LocalSession> { private final ApplicationPackage applicationPackage; @@ -172,8 +171,8 @@ public class LocalSession extends Session implements Comparable<LocalSession> { public Version getVespaVersion() { return zooKeeperClient.readVespaVersion(); } - public ProvisionInfo getProvisionInfo() { - return zooKeeperClient.getProvisionInfo(); + public AllocatedHosts getAllocatedHosts() { + return zooKeeperClient.getAllocatedHosts(); } @Override diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionRepo.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionRepo.java index 7fff132dd7d..ae15be83062 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionRepo.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionRepo.java @@ -18,7 +18,6 @@ import java.util.logging.Logger; * File-based session repository for LocalSessions. Contains state for the local instance of the configserver. * * @author lulf - * @since 5.1 */ public class LocalSessionRepo extends SessionRepo<LocalSession> { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java index c20f6e0b853..5f3cf46e0de 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.config.server.session; import com.yahoo.config.provision.*; +import com.yahoo.lang.SettableOptional; import com.yahoo.vespa.config.server.*; import com.yahoo.log.LogLevel; import com.yahoo.vespa.config.server.application.ApplicationSet; @@ -19,8 +20,7 @@ import java.util.logging.Logger; * A RemoteSession represents a session created on another config server. This session can * be regarded as read only, and this interface only allows reading information about a session. * - * @author lulf - * @since 5.1 + * @author Ulf Lilleengen */ public class RemoteSession extends Session { @@ -57,6 +57,7 @@ public class RemoteSession extends Session { return ApplicationSet.fromList(applicationLoader.buildModels(zooKeeperClient.readApplicationId(), zooKeeperClient.readVespaVersion(), zooKeeperClient.loadApplicationPackage(), + new SettableOptional<>(), clock.instant())); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionFactory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionFactory.java index 0f1e65aaf2b..298acaca901 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionFactory.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionFactory.java @@ -12,8 +12,7 @@ import com.yahoo.vespa.curator.Curator; import java.time.Clock; /** - * @author lulf - * @since 5.1.24 + * @author Ulf Lilleengen */ public class RemoteSessionFactory { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionContext.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionContext.java index a840e5d5a97..323c2667d30 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionContext.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionContext.java @@ -12,8 +12,7 @@ import java.io.File; /** * The dependencies needed for a local session to be edited and prepared. * - * @author lulf - * @since 5.1 + * @author Ulf Lilleengen */ public class SessionContext { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java index 19651041ea4..c2655d1a1b3 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java @@ -170,7 +170,8 @@ public class SessionFactoryImpl implements SessionFactory, LocalSessionLoader { defRepo, serverId, nodeFlavors); - SessionContext context = new SessionContext(applicationPackage, sessionZKClient, sessionDir, applicationRepo, hostRegistry, superModelGenerationCounter); + SessionContext context = new SessionContext(applicationPackage, sessionZKClient, sessionDir, applicationRepo, + hostRegistry, superModelGenerationCounter); return new LocalSession(tenant, sessionId, sessionPreparer, context); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java index 0b76c57e142..beb62cf3ac9 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java @@ -10,6 +10,7 @@ import com.yahoo.config.application.api.FileRegistry; import com.yahoo.config.model.api.ConfigDefinitionRepo; import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.provision.*; +import com.yahoo.lang.SettableOptional; import com.yahoo.log.LogLevel; import com.yahoo.path.Path; import com.yahoo.vespa.config.server.application.ApplicationSet; @@ -42,8 +43,7 @@ import javax.xml.transform.TransformerException; /** * A SessionPreparer is responsible for preparing a session given an application package. * - * @author lulf - * @since 5.1 + * @author Ulf Lilleengen */ public class SessionPreparer { @@ -92,8 +92,8 @@ public class SessionPreparer { Preparation preparation = new Preparation(context, logger, params, currentActiveApplicationSet, tenantPath); preparation.preprocess(); try { - preparation.buildModels(now); - preparation.makeResult(); + AllocatedHosts allocatedHosts = preparation.buildModels(now); + preparation.makeResult(allocatedHosts); if ( ! params.isDryRun()) { preparation.writeStateZK(); preparation.writeRotZK(); @@ -177,13 +177,16 @@ public class SessionPreparer { checkTimeout("preprocess"); } - void buildModels(Instant now) { - this.modelResultList = preparedModelsBuilder.buildModels(applicationId, vespaVersion, applicationPackage, now); + AllocatedHosts buildModels(Instant now) { + SettableOptional<AllocatedHosts> allocatedHosts = new SettableOptional<>(); + this.modelResultList = preparedModelsBuilder.buildModels(applicationId, vespaVersion, + applicationPackage, allocatedHosts, now); checkTimeout("build models"); + return allocatedHosts.get(); } - void makeResult() { - this.prepareResult = new PrepareResult(modelResultList); + void makeResult(AllocatedHosts allocatedHosts) { + this.prepareResult = new PrepareResult(allocatedHosts, modelResultList); checkTimeout("making result from models"); } @@ -195,7 +198,7 @@ public class SessionPreparer { vespaVersion, logger, prepareResult.getFileRegistries(), - prepareResult.getProvisionInfos()); + prepareResult.allocatedHosts()); checkTimeout("write state to zookeeper"); } @@ -236,10 +239,10 @@ public class SessionPreparer { com.yahoo.component.Version vespaVersion, DeployLogger deployLogger, Map<Version, FileRegistry> fileRegistryMap, - Map<Version, ProvisionInfo> provisionInfoMap) { + AllocatedHosts allocatedHosts) { ZooKeeperDeployer zkDeployer = zooKeeperClient.createDeployer(deployLogger); try { - zkDeployer.deploy(applicationPackage, fileRegistryMap, provisionInfoMap); + zkDeployer.deploy(applicationPackage, fileRegistryMap, allocatedHosts); zooKeeperClient.writeApplicationId(applicationId); zooKeeperClient.writeVespaVersion(vespaVersion); } catch (RuntimeException | IOException e) { @@ -251,21 +254,19 @@ public class SessionPreparer { /** The result of preparation over all model versions */ private static class PrepareResult { + private final AllocatedHosts allocatedHosts; private final ImmutableList<PreparedModelsBuilder.PreparedModelResult> results; - - public PrepareResult(List<PreparedModelsBuilder.PreparedModelResult> results) { + + public PrepareResult(AllocatedHosts allocatedHosts, List<PreparedModelsBuilder.PreparedModelResult> results) { + this.allocatedHosts = allocatedHosts; this.results = ImmutableList.copyOf(results); } /** Returns the results for each model as an immutable list */ public List<PreparedModelsBuilder.PreparedModelResult> asList() { return results; } - public Map<Version, ProvisionInfo> getProvisionInfos() { - return results.stream() - .filter(result -> result.model.getProvisionInfo().isPresent()) - .collect(Collectors.toMap((prepareResult -> prepareResult.version), - (prepareResult -> prepareResult.model.getProvisionInfo().get()))); - } + /** Returns the host allocations resulting from this preparation. */ + public AllocatedHosts allocatedHosts() { return allocatedHosts; } public Map<Version, FileRegistry> getFileRegistries() { return results.stream() @@ -290,4 +291,31 @@ public class SessionPreparer { } + /** + * During model building each model version will request nodes allocated (from the node allocator) + * for each cluster specified by that model. As allocations are stable this should usually + * result in the same allocations for the same clusters across all model versions, + * otherwise we should fail this preparation as such inconsistencies lead to undefined behavior, + * and there is really just one true allocation (for a given cluster) to be activated in the node repository. + * + * However, these disagreements between allocations in each model version are allowed: + * - A node may be retired in some model version but not another. This allows model versions to change cluster sizes, + * and is ok because the system will converge on the latest version's opinion + * - Clusters may be present on some version but not on another. This does not lead to inconsistency + * and allows new model versions to introduce new clusters. + * + * For each cluster, the newest model version which has that cluster decides the correct retirement status of nodes + * (and all model versions having the cluster must have the same nodes). + * + * This class ensures these constraints and returns a reconciliated set of nodes which should be activated, + * given a set of model activation results. + */ + private static final class ReconciliatedHostAllocations { + + public ReconciliatedHostAllocations(List<PreparedModelsBuilder.PreparedModelResult> results) { + + } + + } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java index a677c5cb7f9..09fc83e225d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java @@ -6,8 +6,7 @@ import com.yahoo.component.Vtag; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.provision.NodeFlavors; -import com.yahoo.config.provision.ProvisionInfo; -import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.transaction.Transaction; import com.yahoo.log.LogLevel; import com.yahoo.path.Path; @@ -31,8 +30,7 @@ import java.util.concurrent.TimeUnit; * Zookeeper client for a specific session. Can be used to read and write session status * and create and get prepare and active barrier. * - * @author lulf - * @since 5.1 + * @author Ulf Lilleengen */ public class SessionZooKeeperClient { @@ -189,10 +187,9 @@ public class SessionZooKeeperClient { return rootPath.append(CREATE_TIME_PATH).getAbsolute(); } - ProvisionInfo getProvisionInfo() { - return loadApplicationPackage().getProvisionInfoMap().values().stream() - .reduce((infoA, infoB) -> infoA.merge(infoB)) - .orElseThrow(() -> new IllegalStateException("Trying to read provision info, but no provision info exists")); + AllocatedHosts getAllocatedHosts() { + return loadApplicationPackage().getAllocatedHosts() + .orElseThrow(() -> new IllegalStateException("Allocated hosts does not exists")); } public ZooKeeperDeployer createDeployer(DeployLogger logger) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantBuilder.java index 144bbf10dc6..f4a5e941ac2 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantBuilder.java @@ -27,8 +27,7 @@ import java.util.concurrent.Executors; /** * Builder for helping out with tenant creation. Each of a tenants dependencies may be overridden for testing. * - * @author lulf - * @since 5.1 + * @author Ulf Lilleengen */ public class TenantBuilder { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenants.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenants.java index 28bae587e2a..377b3997e43 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenants.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenants.java @@ -48,12 +48,12 @@ import java.util.logging.Logger; * Once a tenant is deleted from zookeeper, the zookeeper watcher thread will get notified on all configservers, and * shutdown and delete any per-configserver state. * - * @author vegardh - * @author lulf - * @since 5.1.26 + * @author Vegard Havdal + * @author Ulf Lilleengen */ //TODO Rename to TenantRepository public class Tenants implements ConnectionStateListener, PathChildrenCacheListener { + public static final TenantName HOSTED_VESPA_TENANT = TenantName.from("hosted-vespa"); private static final TenantName DEFAULT_TENANT = TenantName.defaultName(); private static final List<TenantName> SYSTEM_TENANT_NAMES = Arrays.asList( diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ConfigCurator.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ConfigCurator.java index 80c1c44546f..c09fe5ae4bb 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ConfigCurator.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ConfigCurator.java @@ -146,7 +146,7 @@ public class ConfigCurator { /** * Returns the data at a path, or null if the path does not exist. * - * @param path a String with a pathname. + * @param path a String with a pathname. * @return a byte array with data. */ public byte[] getBytes(String path) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java index 4f591278a38..05c301ddba8 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java @@ -2,6 +2,8 @@ package com.yahoo.vespa.config.server.zookeeper; import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableSet; +import com.yahoo.component.Version; import com.yahoo.config.application.api.ApplicationMetaData; import com.yahoo.config.application.api.ComponentInfo; import com.yahoo.config.application.api.FileRegistry; @@ -10,9 +12,9 @@ import com.yahoo.config.codegen.DefParser; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.application.provider.*; +import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeFlavors; -import com.yahoo.config.provision.ProvisionInfo; -import com.yahoo.config.provision.Version; +import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.io.IOUtils; import com.yahoo.path.Path; import com.yahoo.io.reader.NamedReader; @@ -24,20 +26,26 @@ import com.yahoo.vespa.config.util.ConfigUtils; import java.io.File; import java.io.Reader; import java.io.StringReader; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; /** * Represents an application residing in zookeeper. * - * @author tonytv + * @author Tony Vaagenes */ public class ZKApplicationPackage implements ApplicationPackage { private ZKLiveApp liveApp; - private final Map<Version, PreGeneratedFileRegistry> fileRegistryMap = new HashMap<>(); - private final Map<Version, ProvisionInfo> provisionInfoMap = new HashMap<>(); - private static final Version legacyVersion = Version.fromIntValues(0, 0, 0); + private final Map<com.yahoo.config.provision.Version, PreGeneratedFileRegistry> fileRegistryMap = new HashMap<>(); + private final Optional<AllocatedHosts> allocatedHosts; + private static final com.yahoo.config.provision.Version legacyVersion = com.yahoo.config.provision.Version.fromIntValues(0, 0, 0); public static final String fileRegistryNode = "fileregistry"; public static final String allocatedHostsNode = "allocatedHosts"; @@ -48,34 +56,56 @@ public class ZKApplicationPackage implements ApplicationPackage { liveApp = new ZKLiveApp(zk, appPath); metaData = readMetaDataFromLiveApp(liveApp); importFileRegistries(fileRegistryNode); - importProvisionInfos(allocatedHostsNode, nodeFlavors); + allocatedHosts = importAllocatedHosts(allocatedHostsNode, nodeFlavors); } - private void importProvisionInfos(String allocatedHostsNode, Optional<NodeFlavors> nodeFlavors) { - List<String> provisionInfoNodes = liveApp.getChildren(allocatedHostsNode); - if (provisionInfoNodes.isEmpty()) { - Optional<ProvisionInfo> provisionInfo = importProvisionInfo(allocatedHostsNode, nodeFlavors); - provisionInfo.ifPresent(info -> provisionInfoMap.put(legacyVersion, info)); - } else { - provisionInfoNodes.stream() - .forEach(versionStr -> { - Version version = Version.fromString(versionStr); - Optional<ProvisionInfo> provisionInfo = importProvisionInfo(Joiner.on("/").join(allocatedHostsNode, versionStr), - nodeFlavors); - provisionInfo.ifPresent(info -> provisionInfoMap.put(version, info)); - }); + private Optional<AllocatedHosts> importAllocatedHosts(String allocatedHostsPath, Optional<NodeFlavors> nodeFlavors) { + if ( ! liveApp.exists(allocatedHostsPath)) return Optional.empty(); + Optional<AllocatedHosts> allocatedHosts = readAllocatedHosts(allocatedHostsPath, nodeFlavors); + if ( ! allocatedHosts.isPresent()) { // Read from legacy location. TODO: Remove when 6.143 is in production everywhere + List<String> allocatedHostsByVersionNodes = liveApp.getChildren(allocatedHostsPath); + allocatedHosts = merge(readAllocatedHostsByVersion(allocatedHostsByVersionNodes, nodeFlavors)); + } + return allocatedHosts; + } + + private Map<Version, AllocatedHosts> readAllocatedHostsByVersion(List<String> allocatedHostsByVersionNodes, + Optional<NodeFlavors> nodeFlavors) { + Map<Version, AllocatedHosts> allocatedHostsByVersion = new HashMap<>(); + allocatedHostsByVersionNodes.stream() + .forEach(versionStr -> { + Version version = Version.fromString(versionStr); + Optional<AllocatedHosts> allocatedHosts = readAllocatedHosts(Joiner.on("/").join(allocatedHostsNode, versionStr), + nodeFlavors); + allocatedHosts.ifPresent(info -> allocatedHostsByVersion.put(version, info)); + }); + return allocatedHostsByVersion; + } + + private Optional<AllocatedHosts> merge(Map<Version, AllocatedHosts> allocatedHostsByVersion) { + // Merge the allocated hosts in any order. This is wrong but preserves current behavior (modulo order differences) + if (allocatedHostsByVersion.isEmpty()) return Optional.empty(); + + Map<String, HostSpec> merged = new HashMap<>(); + for (Map.Entry<Version, AllocatedHosts> entry : allocatedHostsByVersion.entrySet()) { + for (HostSpec host : entry.getValue().getHosts()) + merged.put(host.hostname(), host); } + return Optional.of(AllocatedHosts.withHosts(ImmutableSet.copyOf(merged.values()))); } - private Optional<ProvisionInfo> importProvisionInfo(String provisionInfoNode, Optional<NodeFlavors> nodeFlavors) { + /** + * Reads allocated hosts at the given node. + * + * @return the allocated hosts at this node or empty if there is no data at this path + */ + private Optional<AllocatedHosts> readAllocatedHosts(String allocatedHostsPath, Optional<NodeFlavors> nodeFlavors) { try { - if (liveApp.exists(provisionInfoNode)) { - return Optional.of(ProvisionInfo.fromJson(liveApp.getBytes(provisionInfoNode), nodeFlavors)); - } else { - return Optional.empty(); - } + byte[] data = liveApp.getBytes(allocatedHostsPath); + if (data.length == 0) return Optional.empty(); // TODO: Remove this line (and make return non-optional) when 6.143 is in production everywhere + return Optional.of(AllocatedHosts.fromJson(data, nodeFlavors)); } catch (Exception e) { - throw new RuntimeException("Unable to read provision info", e); + throw new RuntimeException("Unable to read allocated hosts", e); } } @@ -85,9 +115,9 @@ public class ZKApplicationPackage implements ApplicationPackage { fileRegistryMap.put(legacyVersion, importFileRegistry(fileRegistryNode)); } else { fileRegistryNodes.stream() - .forEach(versionStr -> { - Version version = Version.fromString(versionStr); - fileRegistryMap.put(version, importFileRegistry(Joiner.on("/").join(fileRegistryNode, versionStr))); + .forEach(version -> { + fileRegistryMap.put(com.yahoo.config.provision.Version.fromString(version), + importFileRegistry(Joiner.on("/").join(fileRegistryNode, version))); }); } } @@ -147,16 +177,17 @@ public class ZKApplicationPackage implements ApplicationPackage { return ret; } - public Map<Version, ProvisionInfo> getProvisionInfoMap() { - return Collections.unmodifiableMap(provisionInfoMap); + @Override + public Optional<AllocatedHosts> getAllocatedHosts() { + return allocatedHosts; } @Override - public Map<Version, FileRegistry> getFileRegistryMap() { + public Map<com.yahoo.config.provision.Version, FileRegistry> getFileRegistryMap() { return Collections.unmodifiableMap(fileRegistryMap); } - private Optional<PreGeneratedFileRegistry> getPreGeneratedFileRegistry(Version vespaVersion) { + private Optional<PreGeneratedFileRegistry> getPreGeneratedFileRegistry(com.yahoo.config.provision.Version vespaVersion) { // Assumes at least one file registry, which we always have. Optional<PreGeneratedFileRegistry> fileRegistry = Optional.ofNullable(fileRegistryMap.get(vespaVersion)); if (!fileRegistry.isPresent()) { @@ -243,7 +274,7 @@ public class ZKApplicationPackage implements ApplicationPackage { } @Override - public List<ComponentInfo> getComponentsInfo(Version vespaVersion) { + public List<ComponentInfo> getComponentsInfo(com.yahoo.config.provision.Version vespaVersion) { List<ComponentInfo> components = new ArrayList<>(); PreGeneratedFileRegistry fileRegistry = getPreGeneratedFileRegistry(vespaVersion).get(); for (String path : fileRegistry.getPaths()) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKLiveApp.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKLiveApp.java index 88748cb7689..8084be1cefa 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKLiveApp.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKLiveApp.java @@ -173,14 +173,11 @@ public class ZKLiveApp { * Returns the full list of children (file names) in the given path. * * @param path a path relative to the currently active application - * @return a list of file names + * @return a list of file names, which is empty (never null) if the path does not exist */ public List<String> getChildren(String path) { String fullPath = getFullPath(path); - if (! zk.exists(fullPath)) { - log.fine("ZKApplicationPackage: " + fullPath + " is not a valid dir"); - return Collections.emptyList(); - } + if (! zk.exists(fullPath)) return Collections.emptyList(); return zk.getChildren(fullPath); } diff --git a/configserver/src/test/apps/serverdb/serverdefs/attributes.def b/configserver/src/test/apps/serverdb/serverdefs/attributes.def index 7943091ae6e..89671b57046 100644 --- a/configserver/src/test/apps/serverdb/serverdefs/attributes.def +++ b/configserver/src/test/apps/serverdb/serverdefs/attributes.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=2 namespace=config attribute[].name string attribute[].datatype string diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ModelStub.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ModelStub.java index b892abc67af..2ccce9727a3 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ModelStub.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ModelStub.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.config.server; import com.yahoo.config.model.api.FileDistribution; import com.yahoo.config.model.api.HostInfo; import com.yahoo.config.model.api.Model; -import com.yahoo.config.provision.ProvisionInfo; +import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.ConfigPayload; import com.yahoo.vespa.config.buildergen.ConfigDefinition; @@ -44,7 +44,7 @@ public class ModelStub implements Model { } @Override - public Optional<ProvisionInfo> getProvisionInfo() { + public AllocatedHosts allocatedHosts() { return null; } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/MockModel.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/MockModel.java index 5806c7991fc..cf5463b7f4c 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/MockModel.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/MockModel.java @@ -7,7 +7,7 @@ import com.yahoo.config.model.api.Model; import com.yahoo.config.model.api.PortInfo; import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.ProvisionInfo; +import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.ConfigPayload; import com.yahoo.vespa.config.buildergen.ConfigDefinition; @@ -93,7 +93,7 @@ class MockModel implements Model { } @Override - public Optional<ProvisionInfo> getProvisionInfo() { + public AllocatedHosts allocatedHosts() { throw new UnsupportedOperationException(); } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/configdefs/a.def b/configserver/src/test/java/com/yahoo/vespa/config/server/configdefs/a.def index d9791909f69..bf99c65ec14 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/configdefs/a.def +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/configdefs/a.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=1 storage[].feeder[] string search[].feeder[] string storage[].id reference diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/configdefs/b.def b/configserver/src/test/java/com/yahoo/vespa/config/server/configdefs/b.def index 81d9f1d30bd..42e58eb4f9e 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/configdefs/b.def +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/configdefs/b.def @@ -1,4 +1,3 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=1 gaff int default=0 usercfgwithid int diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/configdefs/c.def b/configserver/src/test/java/com/yahoo/vespa/config/server/configdefs/c.def index fb0fb64172e..3621e2eeb9a 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/configdefs/c.def +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/configdefs/c.def @@ -1,4 +1,3 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=1 foo string gaz int diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/configdefs/compositeinclude.def b/configserver/src/test/java/com/yahoo/vespa/config/server/configdefs/compositeinclude.def index 39d16c616e8..195acc54718 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/configdefs/compositeinclude.def +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/configdefs/compositeinclude.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=1 classes[].id int classes[].name string classes[].fields[].name string diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/configdefs/d.def b/configserver/src/test/java/com/yahoo/vespa/config/server/configdefs/d.def index 4b65de94e36..eb19c8a95ba 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/configdefs/d.def +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/configdefs/d.def @@ -1,4 +1,3 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=1 thestring string default="g" theint int default=6 diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/configdefs/e.def b/configserver/src/test/java/com/yahoo/vespa/config/server/configdefs/e.def index dd5ffa7ed16..20d311db6e0 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/configdefs/e.def +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/configdefs/e.def @@ -1,4 +1,3 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=1 # this one will be implicit, no cfg fo int default=-45 diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/configdefs/recursiveinclude.def b/configserver/src/test/java/com/yahoo/vespa/config/server/configdefs/recursiveinclude.def index 0d5a81f0a63..aca6789c179 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/configdefs/recursiveinclude.def +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/configdefs/recursiveinclude.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=4 rec int ursive string national int diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/configdefs/spooler.def b/configserver/src/test/java/com/yahoo/vespa/config/server/configdefs/spooler.def index 675d2601873..2721af7e278 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/configdefs/spooler.def +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/configdefs/spooler.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=2 # Which directory to find spool files in. directory string default="/home/vespa/var/spool/vespa" diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java index c9556425dda..5fea15a7b10 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java @@ -20,7 +20,7 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.InstanceName; -import com.yahoo.config.provision.ProvisionInfo; +import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.ProvisionLogger; import com.yahoo.config.provision.Provisioner; import com.yahoo.config.provision.Version; @@ -69,7 +69,7 @@ public class DeployTester { private ApplicationId id; public DeployTester(String appPath) { - this(appPath, Collections.singletonList(createDefaultModelFactory(Clock.systemUTC()))); + this(appPath, Collections.singletonList(createModelFactory(Clock.systemUTC()))); } public DeployTester(String appPath, List<ModelFactory> modelFactories) { @@ -80,7 +80,7 @@ public class DeployTester { public DeployTester(String appPath, ConfigserverConfig configserverConfig) { this(appPath, - Collections.singletonList(createDefaultModelFactory(Clock.systemUTC())), + Collections.singletonList(createModelFactory(Clock.systemUTC())), configserverConfig); } @@ -99,9 +99,16 @@ public class DeployTester { public Tenant tenant() { return tenants.defaultTenant(); } - /** Create the model factory which will be used in production */ - public static ModelFactory createDefaultModelFactory(Clock clock) { return new VespaModelFactory(new NullConfigModelRegistry(), clock); } - + /** Create a model factory for the version of this source*/ + public static ModelFactory createModelFactory(Clock clock) { + return new VespaModelFactory(new NullConfigModelRegistry(), clock); + } + + /** Create a model factory for a particular version */ + public static ModelFactory createModelFactory(Version version, Clock clock) { + return new VespaModelFactory(version, new NullConfigModelRegistry(), clock); + } + /** Create a model factory which always fails validation */ public static ModelFactory createFailingModelFactory(Version version) { return new FailingModelFactory(version); } @@ -109,20 +116,19 @@ public class DeployTester { * Do the initial "deploy" with the existing API-less code as the deploy API doesn't support first deploys yet. */ public ApplicationId deployApp(String appName, Instant now) { - return deployApp(appName, Optional.empty(), now); + return deployApp(appName, null, now); } /** * Do the initial "deploy" with the existing API-less code as the deploy API doesn't support first deploys yet. */ - public ApplicationId deployApp(String appName, Optional<String> vespaVersion, Instant now) { + public ApplicationId deployApp(String appName, String vespaVersion, Instant now) { Tenant tenant = tenant(); LocalSession session = tenant.getSessionFactory().createSession(testApp, appName, new TimeoutBudget(Clock.systemUTC(), Duration.ofSeconds(60))); ApplicationId id = ApplicationId.from(tenant.getName(), ApplicationName.from(appName), InstanceName.defaultName()); - PrepareParams.Builder paramsBuilder = new PrepareParams.Builder() - .applicationId(id); - if (vespaVersion.isPresent()) - paramsBuilder.vespaVersion(vespaVersion.get()); + PrepareParams.Builder paramsBuilder = new PrepareParams.Builder().applicationId(id); + if (vespaVersion != null) + paramsBuilder.vespaVersion(vespaVersion); session.prepare(new SilentDeployLogger(), paramsBuilder.build(), Optional.empty(), @@ -134,11 +140,11 @@ public class DeployTester { return id; } - public ProvisionInfo getProvisionInfoFromDeployedApp(ApplicationId applicationId) { + public AllocatedHosts getAllocatedHostsOf(ApplicationId applicationId) { Tenant tenant = tenant(); LocalSession session = tenant.getLocalSessionRepo().getSession(tenant.getApplicationRepo() .getSessionIdForApplication(applicationId)); - return session.getProvisionInfo(); + return session.getAllocatedHosts(); } public ApplicationId applicationId() { return id; } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java index 31e92cc9f93..301ae63fb8c 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java @@ -32,7 +32,7 @@ public class HostedDeployTest { @Test public void testRedeployWithVersion() throws InterruptedException, IOException { DeployTester tester = new DeployTester("src/test/apps/hosted/", createConfigserverConfig()); - tester.deployApp("myApp", Optional.of("4.5.6"), Instant.now()); + tester.deployApp("myApp", "4.5.6", Instant.now()); Optional<com.yahoo.config.provision.Deployment> deployment = tester.redeployFromLocalActive(); assertTrue(deployment.isPresent()); @@ -53,11 +53,23 @@ public class HostedDeployTest { } @Test + public void testDeployMultipleVersions() throws InterruptedException, IOException { + ManualClock clock = new ManualClock("2016-10-09T00:00:00"); + List<ModelFactory> modelFactories = new ArrayList<>(); + modelFactories.add(DeployTester.createModelFactory(Version.fromString("6.1.0"), clock)); + modelFactories.add(DeployTester.createModelFactory(Version.fromString("6.2.0"), clock)); + modelFactories.add(DeployTester.createModelFactory(Version.fromString("7.0.0"), clock)); + DeployTester tester = new DeployTester("src/test/apps/hosted/", modelFactories, createConfigserverConfig()); + ApplicationId app = tester.deployApp("myApp", Instant.now()); + assertEquals(3, tester.getAllocatedHostsOf(app).getHosts().size()); + } + + @Test public void testRedeployAfterExpiredValidationOverride() throws InterruptedException, IOException { // Old version of model fails, but application disables loading old models until 2016-10-10, so deployment works ManualClock clock = new ManualClock("2016-10-09T00:00:00"); List<ModelFactory> modelFactories = new ArrayList<>(); - modelFactories.add(DeployTester.createDefaultModelFactory(clock)); + modelFactories.add(DeployTester.createModelFactory(clock)); modelFactories.add(DeployTester.createFailingModelFactory(Version.fromIntValues(1, 0, 0))); // older than default DeployTester tester = new DeployTester("src/test/apps/validationOverride/", modelFactories, createConfigserverConfig()); tester.deployApp("myApp", clock.instant()); @@ -97,19 +109,19 @@ public class HostedDeployTest { public void testDeployWithDockerImage() throws InterruptedException, IOException { final String vespaVersion = "6.51.1"; DeployTester tester = new DeployTester("src/test/apps/hosted/", createConfigserverConfig()); - ApplicationId applicationId = tester.deployApp("myApp", Optional.of(vespaVersion), Instant.now()); - assertProvisionInfo(vespaVersion, tester, applicationId); + ApplicationId applicationId = tester.deployApp("myApp", vespaVersion, Instant.now()); + assertAllocatedHosts(vespaVersion, tester, applicationId); System.out.println("Redeploy"); Optional<com.yahoo.config.provision.Deployment> deployment = tester.redeployFromLocalActive(); assertTrue(deployment.isPresent()); deployment.get().prepare(); deployment.get().activate(); - //assertProvisionInfo(vespaVersion, tester, applicationId); + //assertAllocatedHosts(vespaVersion, tester, applicationId); } - private void assertProvisionInfo(String vespaVersion, DeployTester tester, ApplicationId applicationId) { - tester.getProvisionInfoFromDeployedApp(applicationId).getHosts().stream() + private void assertAllocatedHosts(String vespaVersion, DeployTester tester, ApplicationId applicationId) { + tester.getAllocatedHostsOf(applicationId).getHosts().stream() .forEach(h -> assertEquals(vespaVersion, h.membership().get().cluster().vespaVersion())); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/RedeployTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/RedeployTest.java index 5658e0fb2aa..49e40321321 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/RedeployTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/RedeployTest.java @@ -47,7 +47,7 @@ public class RedeployTest { @Test public void testNoRedeploy() { List<ModelFactory> modelFactories = new ArrayList<>(); - modelFactories.add(DeployTester.createDefaultModelFactory(Clock.systemUTC())); + modelFactories.add(DeployTester.createModelFactory(Clock.systemUTC())); modelFactories.add(DeployTester.createFailingModelFactory(Version.fromIntValues(1, 0, 0))); DeployTester tester = new DeployTester("ignored/app/path", modelFactories); ApplicationId id = ApplicationId.from(TenantName.from("default"), diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java index ac029c12b17..bf7f7038c1a 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.deploy; +import com.google.common.collect.ImmutableSet; import com.yahoo.config.application.api.ApplicationMetaData; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.FileRegistry; @@ -46,8 +47,8 @@ public class ZooKeeperClientTest extends TestWithCurator { Map<Version, FileRegistry> fileRegistries = createFileRegistries(); app.writeMetaData(); zkc.setupZooKeeper(); - zkc.feedZooKeeper(app); - zkc.feedZKFileRegistries(fileRegistries); + zkc.write(app); + zkc.write(fileRegistries); } private Map<Version, FileRegistry> createFileRegistries() { @@ -174,23 +175,15 @@ public class ZooKeeperClientTest extends TestWithCurator { Path app = Path.fromString("/1"); ZooKeeperClient zooKeeperClient = new ZooKeeperClient(zk, logger, true, app); zooKeeperClient.setupZooKeeper(); - zooKeeperClient.feedProvisionInfos(createProvisionInfos()); + HostSpec host1 = new HostSpec("host1.yahoo.com", Collections.emptyList()); + HostSpec host2 = new HostSpec("host2.yahoo.com", Collections.emptyList()); + ImmutableSet<HostSpec> hosts = ImmutableSet.of(host1, host2); + zooKeeperClient.write(AllocatedHosts.withHosts(hosts)); Path hostsPath = app.append(ZKApplicationPackage.allocatedHostsNode); assertTrue(zk.exists(hostsPath.getAbsolute())); - assertEquals(0, zk.getBytes(hostsPath.getAbsolute()).length); // Changed from null - assertTrue(zk.exists(hostsPath.append("1.2.3").getAbsolute())); - assertTrue(zk.exists(hostsPath.append("3.2.1").getAbsolute())); - assertTrue(zk.getBytes(hostsPath.append("1.2.3").getAbsolute()).length > 0); - assertTrue(zk.getBytes(hostsPath.append("3.2.1").getAbsolute()).length > 0); - } - - private Map<Version, ProvisionInfo> createProvisionInfos() { - Map<Version, ProvisionInfo> provisionInfoMap = new HashMap<>(); - ProvisionInfo a = ProvisionInfo.withHosts(Collections.singleton(new HostSpec("host.yahoo.com", Collections.emptyList()))); - ProvisionInfo b = ProvisionInfo.withHosts(Collections.singleton(new HostSpec("host2.yahoo.com", Collections.emptyList()))); - provisionInfoMap.put(Version.fromIntValues(1, 2, 3), a); - provisionInfoMap.put(Version.fromIntValues(3, 2, 1), b); - return provisionInfoMap; + + AllocatedHosts deserialized = AllocatedHosts.fromJson(zk.getBytes(hostsPath.getAbsolute()), Optional.empty()); + assertEquals(hosts, deserialized.getHosts()); } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperDeployerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperDeployerTest.java index 9c7f12c2147..b256079d259 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperDeployerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperDeployerTest.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.config.server.deploy; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.application.provider.*; +import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.Version; import com.yahoo.io.IOUtils; import com.yahoo.path.Path; @@ -52,7 +53,7 @@ public class ZooKeeperDeployerTest { ZooKeeperClient client = new ZooKeeperClient(configCurator, logger, true, appPath); ZooKeeperDeployer deployer = new ZooKeeperDeployer(client); - deployer.deploy(applicationPackage, Collections.singletonMap(Version.fromIntValues(1, 0, 0), new MockFileRegistry()), Collections.emptyMap()); + deployer.deploy(applicationPackage, Collections.singletonMap(Version.fromIntValues(1, 0, 0), new MockFileRegistry()), AllocatedHosts.withHosts(Collections.emptySet())); assertTrue(configCurator.exists(appPath.getAbsolute())); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java index 5e10f364aeb..7fca78b087b 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java @@ -13,7 +13,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.InstanceName; -import com.yahoo.config.provision.ProvisionInfo; +import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; import com.yahoo.container.jdisc.HttpResponse; @@ -217,8 +217,8 @@ public class SessionActiveHandlerTest extends SessionHandlerTest { zkClient.writeStatus(status); ZooKeeperClient zkC = new ZooKeeperClient(configCurator, new BaseDeployLogger(), false, pathProvider.getSessionDirs().append(String.valueOf(sessionId))); VespaModelFactory modelFactory = new VespaModelFactory(new NullConfigModelRegistry()); - zkC.feedZKFileRegistries(Collections.singletonMap(modelFactory.getVersion(), new MockFileRegistry())); - zkC.feedProvisionInfos(Collections.singletonMap(modelFactory.getVersion(), ProvisionInfo.withHosts(Collections.emptySet()))); + zkC.write(Collections.singletonMap(modelFactory.getVersion(), new MockFileRegistry())); + zkC.write(AllocatedHosts.withHosts(Collections.emptySet())); TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder() .curator(curator) .configCurator(configCurator) @@ -318,7 +318,7 @@ public class SessionActiveHandlerTest extends SessionHandlerTest { ActivateRequest invoke(boolean createLocalSession) throws Exception { SessionZooKeeperClient zkClient = new MockSessionZKClient(curator, pathProvider.getSessionDirs().append(String.valueOf(sessionId)), - Optional.of(ProvisionInfo.withHosts(Collections.singleton(new HostSpec("bar", Collections.emptyList()))))); + Optional.of(AllocatedHosts.withHosts(Collections.singleton(new HostSpec("bar", Collections.emptyList()))))); session = createRemoteSession(sessionId, initialStatus, zkClient, clock); if (createLocalSession) { LocalSessionRepo repo = addLocalSession(sessionId, deployData, zkClient); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/provision/StaticProvisionerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/provision/StaticProvisionerTest.java index c54514eb097..badcdf53b77 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/provision/StaticProvisionerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/provision/StaticProvisionerTest.java @@ -20,7 +20,7 @@ import java.io.IOException; import static org.junit.Assert.assertEquals; /** - * @author lulf + * @author Ulf Lilleengen */ public class StaticProvisionerTest { @@ -30,7 +30,7 @@ public class StaticProvisionerTest { InMemoryProvisioner inMemoryHostProvisioner = new InMemoryProvisioner(false, "host1.yahoo.com", "host2.yahoo.com", "host3.yahoo.com", "host4.yahoo.com"); VespaModel firstModel = createModel(app, inMemoryHostProvisioner); - StaticProvisioner staticProvisioner = new StaticProvisioner(firstModel.getProvisionInfo().get()); + StaticProvisioner staticProvisioner = new StaticProvisioner(firstModel.allocatedHosts()); VespaModel secondModel = createModel(app, staticProvisioner); assertModelConfig(firstModel, secondModel); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java index f0086eabd26..be98f41c82a 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java @@ -120,19 +120,19 @@ public class LocalSessionTest { @Test(expected = IllegalStateException.class) public void require_that_no_provision_info_throws_exception() throws Exception { - createSession(TenantName.defaultName(), 3).getProvisionInfo(); + createSession(TenantName.defaultName(), 3).getAllocatedHosts(); } @Test public void require_that_provision_info_can_be_read() throws Exception { - ProvisionInfo input = ProvisionInfo.withHosts(Collections.singleton(new HostSpec("myhost", Collections.<String>emptyList()))); + AllocatedHosts input = AllocatedHosts.withHosts(Collections.singleton(new HostSpec("myhost", Collections.<String>emptyList()))); LocalSession session = createSession(TenantName.defaultName(), 3, new SessionTest.MockSessionPreparer(), Optional.of(input)); ApplicationId origId = new ApplicationId.Builder() .tenant("tenant") .applicationName("foo").instanceName("quux").build(); doPrepare(session, new PrepareParams.Builder().applicationId(origId).build(), Instant.now()); - ProvisionInfo info = session.getProvisionInfo(); + AllocatedHosts info = session.getAllocatedHosts(); assertNotNull(info); assertThat(info.getHosts().size(), is(1)); assertTrue(info.getHosts().contains(new HostSpec("myhost", Collections.emptyList()))); @@ -151,18 +151,18 @@ public class LocalSessionTest { } private LocalSession createSession(TenantName tenant, long sessionId, SessionTest.MockSessionPreparer preparer) throws Exception { - return createSession(tenant, sessionId, preparer, Optional.<ProvisionInfo>empty()); + return createSession(tenant, sessionId, preparer, Optional.<AllocatedHosts>empty()); } - private LocalSession createSession(TenantName tenant, long sessionId, SessionTest.MockSessionPreparer preparer, Optional<ProvisionInfo> provisionInfo) throws Exception { + private LocalSession createSession(TenantName tenant, long sessionId, SessionTest.MockSessionPreparer preparer, Optional<AllocatedHosts> allocatedHosts) throws Exception { Path appPath = Path.fromString("/" + sessionId); - SessionZooKeeperClient zkc = new MockSessionZKClient(curator, appPath, provisionInfo); + SessionZooKeeperClient zkc = new MockSessionZKClient(curator, appPath, allocatedHosts); zkc.createWriteStatusTransaction(Session.Status.NEW).commit(); ZooKeeperClient zkClient = new ZooKeeperClient(configCurator, new BaseDeployLogger(), false, appPath); - if (provisionInfo.isPresent()) { - zkClient.feedProvisionInfos(Collections.singletonMap(Version.fromIntValues(0, 0, 0), provisionInfo.get())); + if (allocatedHosts.isPresent()) { + zkClient.write(allocatedHosts.get()); } - zkClient.feedZKFileRegistries(Collections.singletonMap(Version.fromIntValues(0, 0, 0), new MockFileRegistry())); + zkClient.write(Collections.singletonMap(Version.fromIntValues(0, 0, 0), new MockFileRegistry())); File sessionDir = new File(tenantFileSystemDirs.path(), String.valueOf(sessionId)); sessionDir.createNewFile(); return new LocalSession(tenant, sessionId, preparer, new SessionContext(FilesApplicationPackage.fromFile(testApp), zkc, sessionDir, new MemoryTenantApplications(), new HostRegistry<>(), superModelGenerationCounter)); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/MockSessionZKClient.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/MockSessionZKClient.java index 9b658a807b9..412e7881a26 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/MockSessionZKClient.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/MockSessionZKClient.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.config.server.session; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.test.MockApplicationPackage; -import com.yahoo.config.provision.ProvisionInfo; +import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.transaction.Transaction; import com.yahoo.path.Path; import com.yahoo.vespa.curator.Curator; @@ -14,22 +14,21 @@ import java.util.Optional; /** * Overrides application package fetching, because this part is hard to do without feeding a full app. * - * @author lulf - * @since 5.1 + * @author Ulf Lilleengen */ public class MockSessionZKClient extends SessionZooKeeperClient { private ApplicationPackage app = null; - private Optional<ProvisionInfo> info = null; + private Optional<AllocatedHosts> info = null; private Session.Status sessionStatus; public MockSessionZKClient(Curator curator, Path rootPath) { this(curator, rootPath, (ApplicationPackage)null); } - public MockSessionZKClient(Curator curator, Path rootPath, Optional<ProvisionInfo> provisionInfo) { + public MockSessionZKClient(Curator curator, Path rootPath, Optional<AllocatedHosts> allocatedHosts) { this(curator, rootPath); - this.info = provisionInfo; + this.info = allocatedHosts; } public MockSessionZKClient(Curator curator, Path rootPath, ApplicationPackage application) { @@ -49,8 +48,8 @@ public class MockSessionZKClient extends SessionZooKeeperClient { } @Override - ProvisionInfo getProvisionInfo() { - return info.orElseThrow(() -> new IllegalStateException("Trying to read provision info, but no provision info exists")); + AllocatedHosts getAllocatedHosts() { + return info.orElseThrow(() -> new IllegalStateException("Could not find allocated hosts")); } @Override diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandlerTest.java index 65f546e149a..3b67597c43c 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandlerTest.java @@ -8,6 +8,7 @@ import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.config.model.application.provider.MockFileRegistry; import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.Version; import com.yahoo.io.IOUtils; import com.yahoo.path.Path; @@ -94,7 +95,7 @@ public class TenantRequestHandlerTest extends TestWithCurator { File app = tempFolder.newFolder(); IOUtils.copyDirectory(appDir, app); ZooKeeperDeployer deployer = zkc.createDeployer(new BaseDeployLogger()); - deployer.deploy(FilesApplicationPackage.fromFile(appDir), Collections.singletonMap(vespaVersion, new MockFileRegistry()), Collections.emptyMap()); + deployer.deploy(FilesApplicationPackage.fromFile(appDir), Collections.singletonMap(vespaVersion, new MockFileRegistry()), AllocatedHosts.withHosts(Collections.emptySet())); } private ApplicationSet reloadConfig(long id, Clock clock) { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java index abc77a91c51..adf26dbfa32 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java @@ -16,7 +16,7 @@ import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeFlavors; -import com.yahoo.config.provision.ProvisionInfo; +import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.Version; import com.yahoo.config.provisioning.FlavorsConfig; import com.yahoo.path.Path; @@ -33,7 +33,7 @@ public class ZKApplicationPackageTest extends TestWithCurator { private static final String APP = "src/test/apps/zkapp"; private static final String TEST_FLAVOR_NAME = "test-flavor"; private static final Optional<Flavor> TEST_FLAVOR = new MockNodeFlavors().getFlavor(TEST_FLAVOR_NAME); - private static final ProvisionInfo provisionInfo = ProvisionInfo.withHosts( + private static final AllocatedHosts ALLOCATED_HOSTS = AllocatedHosts.withHosts( Collections.singleton(new HostSpec("foo.yahoo.com", Collections.emptyList(), TEST_FLAVOR, Optional.empty()))); @Rule @@ -64,9 +64,8 @@ public class ZKApplicationPackageTest extends TestWithCurator { assertTrue(zkApp.getFileRegistryMap().containsKey(goodVersion)); assertFalse(zkApp.getFileRegistryMap().containsKey(Version.fromIntValues(0, 0, 0))); assertThat(zkApp.getFileRegistryMap().get(goodVersion).fileSourceHost(), is("dummyfiles")); - assertTrue(zkApp.getProvisionInfoMap().containsKey(goodVersion)); - ProvisionInfo readInfo = zkApp.getProvisionInfoMap().get(goodVersion); - assertThat(Utf8.toString(readInfo.toJson()), is(Utf8.toString(provisionInfo.toJson()))); + AllocatedHosts readInfo = zkApp.getAllocatedHosts().get(); + assertThat(Utf8.toString(readInfo.toJson()), is(Utf8.toString(ALLOCATED_HOSTS.toJson()))); assertThat(readInfo.getHosts().iterator().next().flavor(), is(TEST_FLAVOR)); assertTrue(zkApp.getDeployment().isPresent()); assertThat(DeploymentSpec.fromXml(zkApp.getDeployment().get()).globalServiceId().get(), is("mydisc")); @@ -78,7 +77,7 @@ public class ZKApplicationPackageTest extends TestWithCurator { String metaData = "{\"deploy\":{\"user\":\"foo\",\"from\":\"bar\",\"timestamp\":1},\"application\":{\"name\":\"foo\",\"checksum\":\"abc\",\"generation\":4,\"previousActiveGeneration\":3}}"; zk.putData("/0", ConfigCurator.META_ZK_PATH, metaData); zk.putData("/0/" + ZKApplicationPackage.fileRegistryNode + "/3.0.0", "dummyfiles"); - zk.putData("/0/" + ZKApplicationPackage.allocatedHostsNode + "/3.0.0", provisionInfo.toJson()); + zk.putData("/0/" + ZKApplicationPackage.allocatedHostsNode + "/3.0.0", ALLOCATED_HOSTS.toJson()); } private static class MockNodeFlavors extends NodeFlavors{ diff --git a/configserver/src/test/resources/configdefinitions/app.def b/configserver/src/test/resources/configdefinitions/app.def index 02250339d81..7c7bb58b3a6 100644 --- a/configserver/src/test/resources/configdefinitions/app.def +++ b/configserver/src/test/resources/configdefinitions/app.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=1 namespace=config message string default="Hello!" diff --git a/configserver/src/test/resources/configdefinitions/datastructures.def b/configserver/src/test/resources/configdefinitions/datastructures.def index 8d5582daeb4..79b3a8e77a0 100644 --- a/configserver/src/test/resources/configdefinitions/datastructures.def +++ b/configserver/src/test/resources/configdefinitions/datastructures.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=3 namespace=config date[] string diff --git a/configserver/src/test/resources/configdefinitions/function-test.def b/configserver/src/test/resources/configdefinitions/function-test.def index da024b7fb02..b2a27c42285 100644 --- a/configserver/src/test/resources/configdefinitions/function-test.def +++ b/configserver/src/test/resources/configdefinitions/function-test.def @@ -18,7 +18,6 @@ # - Have an array within a struct, to verify that we correctly recurse. # - Reuse type name further within to ensure that this works. -version=4 namespace=config # Some random bool without a default value. These comments exist to check diff --git a/configserver/src/test/resources/configdefinitions/md5test.def b/configserver/src/test/resources/configdefinitions/md5test.def index a9483f417ab..1c6eae0e587 100644 --- a/configserver/src/test/resources/configdefinitions/md5test.def +++ b/configserver/src/test/resources/configdefinitions/md5test.def @@ -1,9 +1,7 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -# version=4 , version in comment does not count. # Added empty line to see if we can confuse # the server's md5 calculation -version=3 namespace=config #even adding a variable name starting with 'version' diff --git a/configserver/src/test/resources/configdefinitions/simpletypes.def b/configserver/src/test/resources/configdefinitions/simpletypes.def index c5296e5a25e..569df41c97d 100644 --- a/configserver/src/test/resources/configdefinitions/simpletypes.def +++ b/configserver/src/test/resources/configdefinitions/simpletypes.def @@ -1,7 +1,6 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. # Config containing only simple leaf types with default values, that can be used # for testing individual types in detail. -version=1 namespace=config boolval bool default=false diff --git a/configserver/src/test/resources/configdefinitions/unicode.def b/configserver/src/test/resources/configdefinitions/unicode.def index 389db0f502e..e3b93671e99 100644 --- a/configserver/src/test/resources/configdefinitions/unicode.def +++ b/configserver/src/test/resources/configdefinitions/unicode.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=2 namespace=config unicodestring1 string diff --git a/container-accesslogging/src/main/resources/configdefinitions/access-log.def b/container-accesslogging/src/main/resources/configdefinitions/access-log.def index 099b9cd7a26..c0b6c0ffd40 100644 --- a/container-accesslogging/src/main/resources/configdefinitions/access-log.def +++ b/container-accesslogging/src/main/resources/configdefinitions/access-log.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=2 namespace=container.core diff --git a/container-core/src/main/resources/configdefinitions/application-metadata.def b/container-core/src/main/resources/configdefinitions/application-metadata.def index 802db6c36da..a3ccf1b55cf 100644 --- a/container-core/src/main/resources/configdefinitions/application-metadata.def +++ b/container-core/src/main/resources/configdefinitions/application-metadata.def @@ -1,7 +1,6 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. # Contains meta info about one deployed application -version=4 namespace=container.core # The name of the directory that contained the application package diff --git a/container-core/src/main/resources/configdefinitions/container-document.def b/container-core/src/main/resources/configdefinitions/container-document.def index 9d1d41d2b48..e8db64b011a 100644 --- a/container-core/src/main/resources/configdefinitions/container-document.def +++ b/container-core/src/main/resources/configdefinitions/container-document.def @@ -2,7 +2,6 @@ # # Container settings for document type management # -version=1 namespace=container.core.document # A document type name to use a concrete document type for diff --git a/container-core/src/main/resources/configdefinitions/diagnostics.def b/container-core/src/main/resources/configdefinitions/diagnostics.def index cab58b0df97..33f733d4386 100644 --- a/container-core/src/main/resources/configdefinitions/diagnostics.def +++ b/container-core/src/main/resources/configdefinitions/diagnostics.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=2 namespace=container.core ## The fraction of queries to time out over a period of 10s to consider diff --git a/container-core/src/main/resources/configdefinitions/health-monitor.def b/container-core/src/main/resources/configdefinitions/health-monitor.def index adc8c9f9487..dc5cdbc6ca4 100644 --- a/container-core/src/main/resources/configdefinitions/health-monitor.def +++ b/container-core/src/main/resources/configdefinitions/health-monitor.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=1 namespace=container.jdisc.config diff --git a/container-core/src/main/resources/configdefinitions/metrics-presentation.def b/container-core/src/main/resources/configdefinitions/metrics-presentation.def index f1dbeeb43fd..b6c40993ef5 100644 --- a/container-core/src/main/resources/configdefinitions/metrics-presentation.def +++ b/container-core/src/main/resources/configdefinitions/metrics-presentation.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=1 namespace=metrics diff --git a/container-core/src/main/resources/configdefinitions/qr-logging.def b/container-core/src/main/resources/configdefinitions/qr-logging.def index 8db776eafbf..f828f321ae7 100644 --- a/container-core/src/main/resources/configdefinitions/qr-logging.def +++ b/container-core/src/main/resources/configdefinitions/qr-logging.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=4 namespace=container.core logger string default="com.yahoo" # Either QueryAccessLog for a regular Vespa access log, or YApacheAccessLog for a log on yApache format diff --git a/container-core/src/main/resources/configdefinitions/qr-searchers.def b/container-core/src/main/resources/configdefinitions/qr-searchers.def index e9c7d0ff06f..a688749da6d 100644 --- a/container-core/src/main/resources/configdefinitions/qr-searchers.def +++ b/container-core/src/main/resources/configdefinitions/qr-searchers.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=49 namespace=container # this file needs more comments diff --git a/container-core/src/main/resources/configdefinitions/qr-templates.def b/container-core/src/main/resources/configdefinitions/qr-templates.def index 4d419e12bc2..46b1a5cbd65 100644 --- a/container-core/src/main/resources/configdefinitions/qr-templates.def +++ b/container-core/src/main/resources/configdefinitions/qr-templates.def @@ -1,7 +1,6 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. # Not used # TODO: Remove on Vespa 7 -version=8 namespace=container.core ## Prefix to use in queries to choose a given template diff --git a/container-core/src/main/resources/configdefinitions/qr.def b/container-core/src/main/resources/configdefinitions/qr.def index 6efd2d5646c..f06cd2061c1 100644 --- a/container-core/src/main/resources/configdefinitions/qr.def +++ b/container-core/src/main/resources/configdefinitions/qr.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=37 namespace=container ### All params must be flagged as 'restart' because this config is manually diff --git a/container-core/src/main/resources/configdefinitions/threadpool.def b/container-core/src/main/resources/configdefinitions/threadpool.def index 36d15ca9bca..5b5e7e2f4a2 100644 --- a/container-core/src/main/resources/configdefinitions/threadpool.def +++ b/container-core/src/main/resources/configdefinitions/threadpool.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=1 namespace=container.handler diff --git a/container-core/src/main/resources/configdefinitions/vip-status.def b/container-core/src/main/resources/configdefinitions/vip-status.def index abed5194b10..44da7292f05 100644 --- a/container-core/src/main/resources/configdefinitions/vip-status.def +++ b/container-core/src/main/resources/configdefinitions/vip-status.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=1 namespace=container.core ## If there is a Vespa search backend connected to this container, and that diff --git a/container-core/src/test/vespa-configdef/int.def b/container-core/src/test/vespa-configdef/int.def index 45431fb5571..48a94e6bfcd 100644 --- a/container-core/src/test/vespa-configdef/int.def +++ b/container-core/src/test/vespa-configdef/int.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=1 namespace=config.core diff --git a/container-core/src/test/vespa-configdef/string.def b/container-core/src/test/vespa-configdef/string.def index f08dad5baee..250dec3955e 100644 --- a/container-core/src/test/vespa-configdef/string.def +++ b/container-core/src/test/vespa-configdef/string.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=1 namespace=config.core diff --git a/container-di/src/main/resources/configdefinitions/bundles.def b/container-di/src/main/resources/configdefinitions/bundles.def index 089afb6a086..9e10d863106 100644 --- a/container-di/src/main/resources/configdefinitions/bundles.def +++ b/container-di/src/main/resources/configdefinitions/bundles.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=1 namespace=container # References to all 3rd-party bundles to be installed. diff --git a/container-di/src/main/resources/configdefinitions/components.def b/container-di/src/main/resources/configdefinitions/components.def index d35d5086d55..f27abc2fa5a 100644 --- a/container-di/src/main/resources/configdefinitions/components.def +++ b/container-di/src/main/resources/configdefinitions/components.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=3 namespace=container ## A list of components. Components depending on other components may use this to diff --git a/container-di/src/test/vespa-configdef/bootstrap1.def b/container-di/src/test/vespa-configdef/bootstrap1.def index 10e920c8b6e..bdee16d99ea 100644 --- a/container-di/src/test/vespa-configdef/bootstrap1.def +++ b/container-di/src/test/vespa-configdef/bootstrap1.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=1 namespace=config.test dummy string default="" diff --git a/container-di/src/test/vespa-configdef/bootstrap2.def b/container-di/src/test/vespa-configdef/bootstrap2.def index 5571b9de3c2..b4fbffd8ae6 100644 --- a/container-di/src/test/vespa-configdef/bootstrap2.def +++ b/container-di/src/test/vespa-configdef/bootstrap2.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=1 namespace=config.test dummy string default="" diff --git a/container-di/src/test/vespa-configdef/components1.def b/container-di/src/test/vespa-configdef/components1.def index 10e920c8b6e..bdee16d99ea 100644 --- a/container-di/src/test/vespa-configdef/components1.def +++ b/container-di/src/test/vespa-configdef/components1.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=1 namespace=config.test dummy string default="" diff --git a/container-di/src/test/vespa-configdef/int.def b/container-di/src/test/vespa-configdef/int.def index 9f461321ba5..a34539c4a0f 100644 --- a/container-di/src/test/vespa-configdef/int.def +++ b/container-di/src/test/vespa-configdef/int.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=1 namespace=config.di diff --git a/container-di/src/test/vespa-configdef/string.def b/container-di/src/test/vespa-configdef/string.def index eaaba80d74e..396afe54f3f 100644 --- a/container-di/src/test/vespa-configdef/string.def +++ b/container-di/src/test/vespa-configdef/string.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=1 namespace=config.di diff --git a/container-di/src/test/vespa-configdef/test.def b/container-di/src/test/vespa-configdef/test.def index 53b38c579cb..d3e0ed17748 100644 --- a/container-di/src/test/vespa-configdef/test.def +++ b/container-di/src/test/vespa-configdef/test.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=1 namespace=config.test diff --git a/container-di/src/test/vespa-configdef/test2.def b/container-di/src/test/vespa-configdef/test2.def index 53b38c579cb..d3e0ed17748 100644 --- a/container-di/src/test/vespa-configdef/test2.def +++ b/container-di/src/test/vespa-configdef/test2.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=1 namespace=config.test diff --git a/container-di/src/test/vespa-configdef/thread-pool.def b/container-di/src/test/vespa-configdef/thread-pool.def index a38c024fad7..9e6b6694e84 100644 --- a/container-di/src/test/vespa-configdef/thread-pool.def +++ b/container-di/src/test/vespa-configdef/thread-pool.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=1 namespace=config.test diff --git a/container-disc/src/main/resources/configdefinitions/jdisc-bindings.def b/container-disc/src/main/resources/configdefinitions/jdisc-bindings.def index 70017be423b..c1b65d39d7a 100644 --- a/container-disc/src/main/resources/configdefinitions/jdisc-bindings.def +++ b/container-disc/src/main/resources/configdefinitions/jdisc-bindings.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=1 namespace=container.jdisc handlers{}.serverBindings[] string diff --git a/container-messagebus/src/main/resources/configdefinitions/container-mbus.def b/container-messagebus/src/main/resources/configdefinitions/container-mbus.def index ed6ce8437ef..b18bec66959 100644 --- a/container-messagebus/src/main/resources/configdefinitions/container-mbus.def +++ b/container-messagebus/src/main/resources/configdefinitions/container-mbus.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=2 namespace=container.jdisc #settings for message bus in container diff --git a/container-messagebus/src/main/resources/configdefinitions/session.def b/container-messagebus/src/main/resources/configdefinitions/session.def index 9551a6c1ecc..93e71449e98 100644 --- a/container-messagebus/src/main/resources/configdefinitions/session.def +++ b/container-messagebus/src/main/resources/configdefinitions/session.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=1 namespace=container.jdisc.config diff --git a/container-search-and-docproc/src/main/resources/configdefinitions/application-userdata.def b/container-search-and-docproc/src/main/resources/configdefinitions/application-userdata.def index 0f3884b222f..0285d3d2103 100644 --- a/container-search-and-docproc/src/main/resources/configdefinitions/application-userdata.def +++ b/container-search-and-docproc/src/main/resources/configdefinitions/application-userdata.def @@ -2,7 +2,6 @@ # Contains user generated info about one deployed application # The values in this config are set by config overrides in vespa-services -version=1 namespace=container.handler.observability # The user-defined version of this application. diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java index e0d18f2b571..999846d1755 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java @@ -15,7 +15,7 @@ import com.yahoo.data.access.simple.Value.StringValue; * A regular hit from a Vespa backend * * @author bratseth - * @author steinar + * @author Steinar Knutsen */ public class FastHit extends Hit { @@ -322,10 +322,6 @@ public class FastHit extends Hit { needXmlEscape = ! (fieldType instanceof XMLField); this.contents = contents; } - public RawField(byte [] contents) { - needXmlEscape = true; - this.contents = contents; - } public byte [] getUtf8() { return contents; } public boolean needXmlEscape() { return needXmlEscape; } @@ -359,10 +355,6 @@ public class FastHit extends Hit { return queryPacketData; } - public void clearQueryPacketData() { - queryPacketData = null; - } - CacheKey getCacheKey() { return cacheKey; } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java index 3369eb64094..1a7e693caa7 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java @@ -396,6 +396,16 @@ public abstract class VespaBackEndSearcher extends PingableSearcher { s.append(" location=") .append(query.getRanking().getLocation().toString()); } + + if (query.getGroupingSessionCache()) { + s.append(" groupingSessionCache=true"); + } + if (query.getRanking().getQueryCache()) { + s.append(" ranking.queryCache=true"); + } + if (query.getGroupingSessionCache() || query.getRanking().getQueryCache()) { + s.append(" sessionId=" + query.getSessionId(true)); + } List<Grouping> grouping = GroupingExecutor.getGroupingList(query); s.append(" grouping=").append(grouping.size()).append(" : "); diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/NormalizingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/NormalizingSearcher.java index 3b9a03bb5da..02c8ecda60c 100644 --- a/container-search/src/main/java/com/yahoo/prelude/querytransform/NormalizingSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/querytransform/NormalizingSearcher.java @@ -52,10 +52,12 @@ public class NormalizingSearcher extends Searcher { } protected void normalize(Query query, IndexFacts.Session indexFacts) { - String oldQuery = (query.getTraceLevel() >= 2) ? query.getModel().getQueryTree().getRoot().toString() : ""; + String oldQuery = (query.getTraceLevel() >= 2) ? query.getModel().getQueryTree().getRoot().toString() : null; + normalizeBody(query, indexFacts); - if (query.getTraceLevel() >= 2) - if (!(oldQuery.equals(query.getModel().getQueryTree().getRoot().toString()))) query.trace(getFunctionName(), true, 2); + + if (query.getTraceLevel() >= 2 && ! query.getModel().getQueryTree().getRoot().toString().equals(oldQuery)) + query.trace(getFunctionName(), true, 2); } private Query normalizeBody(Query query, IndexFacts.Session indexFacts) { @@ -63,19 +65,18 @@ public class NormalizingSearcher extends Searcher { Language language = query.getModel().getParsingLanguage(); if (root instanceof BlockItem) { List<Item> rootItems = new ArrayList<>(1); - rootItems.add(root); ListIterator<Item> i = rootItems.listIterator(); - i.next(); normalizeBlocks(language, indexFacts, (BlockItem) root, i); - query.getModel().getQueryTree().setRoot(rootItems.get(0)); + if ( ! rootItems.isEmpty()) // give up normalizing if the root was removed + query.getModel().getQueryTree().setRoot(rootItems.get(0)); } else if (root instanceof CompositeItem) { query.getModel().getQueryTree().setRoot(normalizeComposite(language, indexFacts, (CompositeItem) root)); } return query; } - + private Item normalizeComposite(Language language, IndexFacts.Session indexFacts, CompositeItem item) { if (item instanceof PhraseItem) { return normalizePhrase(language, indexFacts, (PhraseItem) item); diff --git a/container-search/src/main/java/com/yahoo/search/Query.java b/container-search/src/main/java/com/yahoo/search/Query.java index 509a5f3d1de..53b39046f1d 100644 --- a/container-search/src/main/java/com/yahoo/search/Query.java +++ b/container-search/src/main/java/com/yahoo/search/Query.java @@ -153,7 +153,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { private QueryContext context = null; /** Used for downstream session caches */ - private final AtomicReference<UniqueRequestId> sessionId = new AtomicReference<>(); + private UniqueRequestId requestId = null; //--------------- Owned sub-objects containing query properties ---------------- @@ -936,6 +936,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { clone.setOffset(getOffset()); clone.setNoCache(getNoCache()); clone.setGroupingSessionCache(getGroupingSessionCache()); + clone.requestId = null; // Each clone should have their own requestId. } /** Returns the presentation to be used for this query, never null */ @@ -962,18 +963,13 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { * @return the session id of this query, or null if not set and create is false */ public SessionId getSessionId(boolean create) { - UniqueRequestId uniqId = sessionId.get(); - if (uniqId == null && ! create) return null; + if (requestId == null && ! create) return null; - if (uniqId == null && create) { - uniqId = UniqueRequestId.next(); - sessionId.compareAndSet(null, uniqId); - uniqId = sessionId.get(); + if (requestId == null && create) { + requestId = UniqueRequestId.next(); } - String rankProfile = getRanking().getProfile(); - - return new SessionId(uniqId, rankProfile); + return new SessionId(requestId, getRanking().getProfile()); } public boolean hasEncodableProperties() { diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java index 003ba9a5261..b03c0cb752f 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java @@ -21,7 +21,6 @@ import com.yahoo.slime.BinaryFormat; import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; import com.yahoo.data.access.Inspector; -import com.yahoo.text.Utf8String; import com.yahoo.vespa.config.search.DispatchConfig; import java.util.Iterator; diff --git a/container-search/src/main/java/com/yahoo/search/query/SessionId.java b/container-search/src/main/java/com/yahoo/search/query/SessionId.java index c6e34e7e430..b065bd9a0a9 100644 --- a/container-search/src/main/java/com/yahoo/search/query/SessionId.java +++ b/container-search/src/main/java/com/yahoo/search/query/SessionId.java @@ -22,4 +22,19 @@ public class SessionId { } public Utf8String asUtf8String() { return id; } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + SessionId sessionId = (SessionId) o; + + return id.equals(sessionId.id); + } + + @Override + public int hashCode() { + return id.hashCode(); + } } diff --git a/container-search/src/main/resources/configdefinitions/documentdb-info.def b/container-search/src/main/resources/configdefinitions/documentdb-info.def index af010f1c161..76096b4a6f7 100644 --- a/container-search/src/main/resources/configdefinitions/documentdb-info.def +++ b/container-search/src/main/resources/configdefinitions/documentdb-info.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=2 namespace=prelude.fastsearch ## The name of the search definition that this document database info applies to diff --git a/container-search/src/main/resources/configdefinitions/emulation.def b/container-search/src/main/resources/configdefinitions/emulation.def index 26dedb77b69..70d2d4954a4 100644 --- a/container-search/src/main/resources/configdefinitions/emulation.def +++ b/container-search/src/main/resources/configdefinitions/emulation.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=2 namespace=prelude ## Keep emulating old (4.2, 5.0, early 5.1) string type fields for structured data diff --git a/container-search/src/main/resources/configdefinitions/federation.def b/container-search/src/main/resources/configdefinitions/federation.def index da9846d5246..36eb5d4b4c8 100644 --- a/container-search/src/main/resources/configdefinitions/federation.def +++ b/container-search/src/main/resources/configdefinitions/federation.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=7 namespace=search.federation target[].id string diff --git a/container-search/src/main/resources/configdefinitions/fs4.def b/container-search/src/main/resources/configdefinitions/fs4.def index a4af65bef02..d0c68540071 100644 --- a/container-search/src/main/resources/configdefinitions/fs4.def +++ b/container-search/src/main/resources/configdefinitions/fs4.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=1 namespace=container.search ##Number of listener threads diff --git a/container-search/src/main/resources/configdefinitions/index-info.def b/container-search/src/main/resources/configdefinitions/index-info.def index c329b0de1cc..f3b905d4d0a 100644 --- a/container-search/src/main/resources/configdefinitions/index-info.def +++ b/container-search/src/main/resources/configdefinitions/index-info.def @@ -1,6 +1,5 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. ## Commands per search definition to be applied to query terms per index before searching -version=2 namespace=search.config ## The name of the search definition this index info applies to diff --git a/container-search/src/main/resources/configdefinitions/lowercasing.def b/container-search/src/main/resources/configdefinitions/lowercasing.def index 521b7624660..b656c451e11 100644 --- a/container-search/src/main/resources/configdefinitions/lowercasing.def +++ b/container-search/src/main/resources/configdefinitions/lowercasing.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=1 namespace=search.querytransform transform_weighted_sets bool default=true diff --git a/container-search/src/main/resources/configdefinitions/measure-qps.def b/container-search/src/main/resources/configdefinitions/measure-qps.def index b146002cab5..c8b38b9db6e 100644 --- a/container-search/src/main/resources/configdefinitions/measure-qps.def +++ b/container-search/src/main/resources/configdefinitions/measure-qps.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=2 namespace=search.statistics ## Configure measurements of peak QPS rates. diff --git a/container-search/src/main/resources/configdefinitions/page-templates.def b/container-search/src/main/resources/configdefinitions/page-templates.def index e69bc9c223c..31ec7644d18 100644 --- a/container-search/src/main/resources/configdefinitions/page-templates.def +++ b/container-search/src/main/resources/configdefinitions/page-templates.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=1 namespace=search.pagetemplates # The xml content of a page template diff --git a/container-search/src/main/resources/configdefinitions/provider.def b/container-search/src/main/resources/configdefinitions/provider.def index 6b23d47427d..79b09913b49 100644 --- a/container-search/src/main/resources/configdefinitions/provider.def +++ b/container-search/src/main/resources/configdefinitions/provider.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=16 namespace=search.federation node[].host string diff --git a/container-search/src/main/resources/configdefinitions/qr-binary-cache-region.def b/container-search/src/main/resources/configdefinitions/qr-binary-cache-region.def index eb7183bff40..ba2b6ed7802 100644 --- a/container-search/src/main/resources/configdefinitions/qr-binary-cache-region.def +++ b/container-search/src/main/resources/configdefinitions/qr-binary-cache-region.def @@ -1,7 +1,6 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. # TODO: Not in use - remove on Vespa 7 -version=4 namespace=search.cache # size of the region (in MB). Cache size (in qr-binary-cache.cfg) diff --git a/container-search/src/main/resources/configdefinitions/qr-binary-cache.def b/container-search/src/main/resources/configdefinitions/qr-binary-cache.def index 0df80d7e520..917832e86fe 100644 --- a/container-search/src/main/resources/configdefinitions/qr-binary-cache.def +++ b/container-search/src/main/resources/configdefinitions/qr-binary-cache.def @@ -1,7 +1,6 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. # TODO: Not in use - remove on Vespa 7 -version=3 namespace=search.cache # Size of a block in cache. A block is the smallest unit that would diff --git a/container-search/src/main/resources/configdefinitions/qr-monitor.def b/container-search/src/main/resources/configdefinitions/qr-monitor.def index 931e62049d2..2c4ff3c6167 100644 --- a/container-search/src/main/resources/configdefinitions/qr-monitor.def +++ b/container-search/src/main/resources/configdefinitions/qr-monitor.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=4 namespace=prelude.cluster ## The number of milliseconds to attempt to complete a request before diff --git a/container-search/src/main/resources/configdefinitions/qr-quotetable.def b/container-search/src/main/resources/configdefinitions/qr-quotetable.def index e387d9dc732..40979ad2a35 100644 --- a/container-search/src/main/resources/configdefinitions/qr-quotetable.def +++ b/container-search/src/main/resources/configdefinitions/qr-quotetable.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=1 namespace=prelude.searcher ## The ordinal number of the character to quote in UNICODE. diff --git a/container-search/src/main/resources/configdefinitions/query-profiles.def b/container-search/src/main/resources/configdefinitions/query-profiles.def index 75b3f122b0a..20fcbda0d72 100644 --- a/container-search/src/main/resources/configdefinitions/query-profiles.def +++ b/container-search/src/main/resources/configdefinitions/query-profiles.def @@ -2,7 +2,6 @@ # Query profiles and their types - a query profile is a nested composite of query parameters with an id # A set of query parameters can be fetched from a query profile rather than being # submitted explicitly. -version=4 namespace=search.query.profile.config # The id of this query profile. The id has the form name(:version)? diff --git a/container-search/src/main/resources/configdefinitions/resolvers.def b/container-search/src/main/resources/configdefinitions/resolvers.def index 87b7a8b84df..6003fdf81f1 100644 --- a/container-search/src/main/resources/configdefinitions/resolvers.def +++ b/container-search/src/main/resources/configdefinitions/resolvers.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=2 namespace=search.pagetemplates ## A list of resolver components used by com.yahoo.search.PageTemplateSearcher diff --git a/container-search/src/main/resources/configdefinitions/rewrites.def b/container-search/src/main/resources/configdefinitions/rewrites.def index 0179057029f..ecca422342a 100644 --- a/container-search/src/main/resources/configdefinitions/rewrites.def +++ b/container-search/src/main/resources/configdefinitions/rewrites.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=2 namespace=search.query.rewrite fsaDict[].name string diff --git a/container-search/src/main/resources/configdefinitions/search-with-renderer-handler.def b/container-search/src/main/resources/configdefinitions/search-with-renderer-handler.def index 1c489f39ed9..a34e08a1c82 100644 --- a/container-search/src/main/resources/configdefinitions/search-with-renderer-handler.def +++ b/container-search/src/main/resources/configdefinitions/search-with-renderer-handler.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=1 namespace=search.handler rendererId string diff --git a/container-search/src/main/resources/configdefinitions/searchchain-forward.def b/container-search/src/main/resources/configdefinitions/searchchain-forward.def index dd66acb2e57..0e86490e120 100644 --- a/container-search/src/main/resources/configdefinitions/searchchain-forward.def +++ b/container-search/src/main/resources/configdefinitions/searchchain-forward.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=2 namespace=search.federation ## A searcher forwarding the incoming query to a single search chain and diff --git a/container-search/src/main/resources/configdefinitions/semantic-rules.def b/container-search/src/main/resources/configdefinitions/semantic-rules.def index 4212e20fcc1..5ac0cca7ff6 100644 --- a/container-search/src/main/resources/configdefinitions/semantic-rules.def +++ b/container-search/src/main/resources/configdefinitions/semantic-rules.def @@ -1,6 +1,5 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. # Semantic production rules -version=2 namespace=prelude.semantics # Whether we should use these rule bases in pre-Vespa 2.2 compatibility mode diff --git a/container-search/src/main/resources/configdefinitions/strict-contracts.def b/container-search/src/main/resources/configdefinitions/strict-contracts.def index f7af59792ae..f9dd788c054 100644 --- a/container-search/src/main/resources/configdefinitions/strict-contracts.def +++ b/container-search/src/main/resources/configdefinitions/strict-contracts.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=2 namespace=search.federation ## A config to control whether to activate strict adherence to public contracts diff --git a/container-search/src/main/resources/configdefinitions/timing-searcher.def b/container-search/src/main/resources/configdefinitions/timing-searcher.def index 254d7c5b5bb..7c2b698bdb0 100644 --- a/container-search/src/main/resources/configdefinitions/timing-searcher.def +++ b/container-search/src/main/resources/configdefinitions/timing-searcher.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=1 namespace=search.statistics timer[].name string diff --git a/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java b/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java index ce9e3357c77..e59c03b33c3 100644 --- a/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java @@ -23,6 +23,7 @@ import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; import com.yahoo.search.query.QueryTree; +import com.yahoo.search.query.SessionId; import com.yahoo.search.query.profile.QueryProfile; import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.search.result.Hit; @@ -44,6 +45,7 @@ import java.util.stream.Collectors; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @@ -637,6 +639,54 @@ public class QueryTestCase { } @Test + public void testThatSessionIdIsUniquePerQuery() { + Query q = new Query(); + assertNull(q.getSessionId(false)); + assertNull(q.getSessionId(false)); + SessionId s1 = q.getSessionId(true); + assertNotNull(s1); + SessionId s2 = q.getSessionId(true); + assertNotSame(s1, s2); + assertEquals(s1, s2); + assertEquals(s1.toString(), s2.toString()); + + Query q2 = new Query(); + assertNotEquals(q.getSessionId(false), q2.getSessionId(true)); + assertNotEquals(q.getSessionId(false).toString(), q2.getSessionId(true).toString()); + } + @Test + public void testThatCloneGetANewSessionId() { + Query q = new Query(); + q.getSessionId(true); + Query clonedQ = q.clone(); + assertNull(clonedQ.getSessionId(false)); + assertNotEquals(q.getSessionId(false), clonedQ.getSessionId(true)); + } + + @Test + public void testThatSessionIdIsUniquePerRankProfilePerQuery() { + Query q = new Query(); + SessionId s1 = q.getSessionId(true); + q.getRanking().setProfile("my-profile"); + SessionId s2 = q.getSessionId(false); + assertNotEquals(s1, s2); + } + + @Test + public void testThatSessionIdIsNotSharedIfCreatedAfterClone() { + Query q = new Query(); + Query q2 = q.clone(); + assertNull(q.getSessionId(false)); + assertNull(q2.getSessionId(false)); + + assertNotNull(q.getSessionId(true)); + assertNull(q2.getSessionId(false)); + + assertNotNull(q2.getSessionId(true)); + assertNotEquals(q.getSessionId(false), q2.getSessionId(false)); + } + + @Test public void testPositiveTerms() { Query q = new Query(httpEncode("/?query=-a \"b c\" d e")); Item i = q.getModel().getQueryTree().getRoot(); diff --git a/container-search/src/test/vespa-configdef/int.def b/container-search/src/test/vespa-configdef/int.def index 895a5e0de68..b526a956b5c 100644 --- a/container-search/src/test/vespa-configdef/int.def +++ b/container-search/src/test/vespa-configdef/int.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=1 namespace=config.search diff --git a/container-search/src/test/vespa-configdef/string.def b/container-search/src/test/vespa-configdef/string.def index c3ee208f633..4b39de61e0a 100644 --- a/container-search/src/test/vespa-configdef/string.def +++ b/container-search/src/test/vespa-configdef/string.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=1 namespace=config.search diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 51bf530ed4a..043a4fd4f3e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -17,12 +17,15 @@ import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus; import com.yahoo.vespa.hosted.controller.api.application.v4.model.GitRevision; import com.yahoo.vespa.hosted.controller.api.application.v4.model.ScrewdriverBuildJob; +import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.ConfigChangeActions; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname; import com.yahoo.vespa.hosted.controller.api.identifiers.RevisionId; import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerClient; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.Log; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NoInstanceException; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareResponse; import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService; import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordId; @@ -216,33 +219,44 @@ public class ApplicationController { /** Deploys an application. If the application does not exist it is created. */ // TODO: Get rid of the options arg - public ActivateResult deployApplication(ApplicationId applicationId, com.yahoo.config.provision.Zone zone, + public ActivateResult deployApplication(ApplicationId applicationId, Zone zone, ApplicationPackage applicationPackage, DeployOptions options) { try (Lock lock = lock(applicationId)) { // Determine what we are doing Application application = get(applicationId).orElse(new Application(applicationId)); - DeploymentJobs.JobType jobType = DeploymentJobs.JobType.from(controller.zoneRegistry().system(), zone); - Version version = decideVersion(application, zone, options); - ApplicationRevision revision = toApplicationPackageRevision(applicationPackage, options.screwdriverBuildJob); + + // Decide version to deploy, if applicable. + Version version; + if (options.deployCurrentVersion) + version = application.currentVersion(controller, zone); + else if (application.deploymentJobs().isSelfTriggering()) // legacy mode: let the client decide + version = options.vespaVersion.map(Version::new).orElse(controller.systemVersion()); + else if ( ! application.deploying().isPresent() && ! zone.environment().isManuallyDeployed()) + return unexpectedDeployment(applicationId, zone, applicationPackage); + else + version = application.currentDeployVersion(controller, zone); // Ensure that the deploying change is tested // FIXME: For now only for non-self-triggering applications - VESPA-8418 - if (!application.deploymentJobs().isSelfTriggering() && !zone.environment().isManuallyDeployed() && !application.deploymentJobs().isDeployableTo(zone.environment(), application.deploying())) { + if ( ! application.deploymentJobs().isSelfTriggering() && + ! zone.environment().isManuallyDeployed() && + ! application.deploymentJobs().isDeployableTo(zone.environment(), application.deploying())) throw new IllegalArgumentException("Rejecting deployment of " + application + " to " + zone + - " as pending " + application.deploying().get() + - " is untested"); - } + " as pending " + application.deploying().get() + + " is untested"); - // Don't update/store applicationpackage information when deploying previous application package (initial staging step) - if(! options.deployCurrentVersion) { - // Add missing information to application + DeploymentJobs.JobType jobType = DeploymentJobs.JobType.from(controller.zoneRegistry().system(), zone); + ApplicationRevision revision = toApplicationPackageRevision(applicationPackage, options.screwdriverBuildJob); + + if( ! options.deployCurrentVersion) { + // Add missing information to application (unless we're deploying the previous version (initial staging step) application = application.with(applicationPackage.deploymentSpec()); application = application.with(applicationPackage.validationOverrides()); if (options.screwdriverBuildJob.isPresent() && options.screwdriverBuildJob.get().screwdriverId != null) application = application.withProjectId(options.screwdriverBuildJob.get().screwdriverId.value()); if (application.deploying().isPresent() && application.deploying().get() instanceof Change.ApplicationChange) application = application.withDeploying(Optional.of(Change.ApplicationChange.of(revision))); - if (!triggeredWith(revision, application, jobType) && !zone.environment().isManuallyDeployed() && jobType != null) { + if ( ! triggeredWith(revision, application, jobType) && !zone.environment().isManuallyDeployed() && jobType != null) { // Triggering information is used to store which changes were made or attempted // - For self-triggered applications we don't have any trigger information, so we add it here. // - For all applications, we don't have complete control over which revision is actually built, @@ -274,21 +288,17 @@ public class ApplicationController { return new ActivateResult(new RevisionId(applicationPackage.hash()), preparedApplication.messages(), preparedApplication.prepareResponse()); } } - - private Version decideVersion(Application application, Zone zone, DeployOptions options) { - if (options.deployCurrentVersion) - return application.currentVersion(controller, zone); - - if (application.deploymentJobs().isSelfTriggering()) // legacy mode: let the client decide - return options.vespaVersion.map(Version::new).orElse(controller.systemVersion()); - if ( ! application.deploying().isPresent() && ! zone.environment().isManuallyDeployed()) - throw new IllegalArgumentException("Rejecting deployment of " + application + " to " + zone + - " as a deployment is not currently expected"); - - return application.currentDeployVersion(controller, zone); + private ActivateResult unexpectedDeployment(ApplicationId applicationId, Zone zone, ApplicationPackage applicationPackage) { + Log logEntry = new Log(); + logEntry.level = "WARNING"; + logEntry.time = clock.instant().toEpochMilli(); + logEntry.message = "Ignoring deployment of " + get(applicationId) + " to " + zone + " as a deployment is not currently expected"; + PrepareResponse prepareResponse = new PrepareResponse(); + prepareResponse.configChangeActions = new ConfigChangeActions(Collections.emptyList(), Collections.emptyList()); + return new ActivateResult(new RevisionId(applicationPackage.hash()), Collections.singletonList(logEntry), prepareResponse); } - + private Application deleteRemovedDeployments(Application application) { List<Deployment> deploymentsToRemove = application.deployments().values().stream() .filter(deployment -> deployment.zone().environment() == Environment.prod) @@ -316,20 +326,14 @@ public class ApplicationController { private Application deleteUnreferencedDeploymentJobs(Application application) { for (DeploymentJobs.JobType job : application.deploymentJobs().jobStatus().keySet()) { - if (!job.isProduction()) { - continue; - } Optional<Zone> zone = job.zone(controller.system()); - if (!zone.isPresent()) { + if ( ! job.isProduction() || (zone.isPresent() && application.deploymentSpec().includes(zone.get().environment(), zone.map(Zone::region)))) continue; - } - if (!application.deploymentSpec().includes(zone.get().environment(), zone.map(Zone::region))) { - application = application.withoutDeploymentJob(job); - } + application = application.withoutDeploymentJob(job); } return application; } - + private boolean triggeredWith(ApplicationRevision revision, Application application, DeploymentJobs.JobType jobType) { if (jobType == null) return false; JobStatus status = application.deploymentJobs().jobStatus().get(jobType); @@ -486,8 +490,7 @@ public class ApplicationController { public Application deactivate(Application application, Deployment deployment, boolean requireThatDeploymentHasExpired) { try (Lock lock = lock(application.id())) { - // TODO: ignore no application errors for config server client, - // only return such errors from sherpa client. + // TODO: ignore no application errors for config server client, only return such errors from sherpa client. if (requireThatDeploymentHasExpired && ! DeploymentExpirer.hasExpired(controller.zoneRegistry(), deployment, clock.instant())) return application; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java index 3fcd285e0fc..fa7a48c85c2 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java @@ -80,21 +80,11 @@ public class ApplicationList { return listOf(list.stream().filter(application -> ! failingOn(version, application))); } - /** Returns the subset of applications which have one or more deployment jobs failing for the current change */ - public ApplicationList hasDeploymentFailures() { - return listOf(list.stream().filter(application -> application.deploying().isPresent() && application.deploymentJobs().failingOn(application.deploying().get()))); - } - /** Returns the subset of applications which have at least one deployment */ public ApplicationList hasDeployment() { return listOf(list.stream().filter(a -> !a.deployments().isEmpty())); } - /** Returns the subset of applications that are currently deploying a change */ - public ApplicationList isDeploying() { - return listOf(list.stream().filter(application -> application.deploying().isPresent())); - } - /** Returns the subset of applications which started failing after the given instant */ public ApplicationList startedFailingAfter(Instant instant) { return listOf(list.stream().filter(application -> application.deploymentJobs().failingSince().isAfter(instant))); @@ -140,18 +130,6 @@ public class ApplicationList { return listOf(list.stream().filter(a -> !hasRunningJob(a, change))); } - /** Returns the subset of applications which currently do not have any job in progress */ - public ApplicationList notRunningJob() { - return listOf(list.stream().filter(a -> !a.deploymentJobs().inProgress())); - } - - /** Returns the subset of applications which has a job that started running before the given instant */ - public ApplicationList jobRunningSince(Instant instant) { - return listOf(list.stream().filter(a -> a.deploymentJobs().runningSince() - .map(at -> at.isBefore(instant)) - .orElse(false))); - } - /** Returns the subset of applications which deploys to given environment and region */ public ApplicationList deploysTo(Environment environment, RegionName region) { return listOf(list.stream().filter(a -> a.deploymentSpec().includes(environment, Optional.of(region)))); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java index d9256f94086..61231404a94 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java @@ -14,7 +14,6 @@ import com.yahoo.vespa.hosted.controller.Controller; import java.time.Instant; import java.util.Collection; import java.util.Collections; -import java.util.Comparator; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; @@ -118,11 +117,6 @@ public class DeploymentJobs { return status.values().stream().anyMatch(JobStatus::inProgress); } - /** Returns whether any job is failing for the given change */ - public boolean failingOn(Change change) { - return status.values().stream().anyMatch(jobStatus -> !jobStatus.isSuccess() && jobStatus.lastCompletedFor(change)); - } - /** Returns whether change can be deployed to the given environment */ public boolean isDeployableTo(Environment environment, Optional<Change> change) { if (environment == null || !change.isPresent()) { @@ -147,15 +141,6 @@ public class DeploymentJobs { return failingSince; } - /** Returns the time at which the oldest running job started */ - public Optional<Instant> runningSince() { - return jobStatus().values().stream() - .filter(JobStatus::inProgress) - .sorted(Comparator.comparing(jobStatus -> jobStatus.lastTriggered().get().at())) - .map(jobStatus -> jobStatus.lastTriggered().get().at()) - .findFirst(); - } - /** * Returns the id of the Screwdriver project running these deployment jobs * - or empty when this is not known or does not exist. @@ -225,8 +210,8 @@ public class DeploymentJobs { } /** Returns the region of this job type, or null if it does not have a region */ - public RegionName region(SystemName system) { - return zone(system).map(Zone::region).orElse(null); + public Optional<RegionName> region(SystemName system) { + return zone(system).map(Zone::region); } public static JobType fromId(String id) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index 2bc219dde62..ce1fee4dd14 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -89,13 +89,13 @@ public class DeploymentTrigger { /** * Called periodically to cause triggering of jobs in the background */ - public void triggerFailing(ApplicationId applicationId) { + public void triggerFailing(ApplicationId applicationId, String cause) { try (Lock lock = applications().lock(applicationId)) { Application application = applications().require(applicationId); if (shouldRetryFromBeginning(application)) { // failed for a long time: Discard existing change and restart from the component job application = application.withDeploying(Optional.empty()); - application = trigger(JobType.component, application, "Retrying failing deployment from beginning", lock); + application = trigger(JobType.component, application, "Retrying failing deployment from beginning: " + cause, lock); applications().store(application, lock); } else { // retry the failed job (with backoff) @@ -103,7 +103,7 @@ public class DeploymentTrigger { JobStatus jobStatus = application.deploymentJobs().jobStatus().get(jobType); if (isFailing(jobStatus)) { if (shouldRetryNow(jobStatus)) { - application = trigger(jobType, application, "Retrying failing job", lock); + application = trigger(jobType, application, "Retrying failing job: " + cause, lock); applications().store(application, lock); } break; @@ -119,6 +119,9 @@ public class DeploymentTrigger { if ( ! application.deploying().isPresent() ) continue; if (application.deploymentJobs().hasFailures()) continue; if (application.deploymentJobs().inProgress()) continue; + if (application.deploymentSpec().steps().stream().noneMatch(step -> step instanceof DeploymentSpec.Delay)) { + continue; // Application does not have any delayed deployments + } Optional<JobStatus> lastSuccessfulJob = application.deploymentJobs().jobStatus().values() .stream() @@ -131,7 +134,7 @@ public class DeploymentTrigger { try (Lock lock = applications().lock(application.id())) { application = applications().require(application.id()); application = trigger(nextAfter(lastSuccessfulJob.get().type(), application), application, - "Delayed by deployment spec", lock); + "Resuming delayed deployment", lock); applications().store(application, lock); } } @@ -230,15 +233,9 @@ public class DeploymentTrigger { return application.deploymentSpec() .zones() .stream() - .filter(z -> { - if (jobType.isProduction()) { - return z.matches(jobType.environment(), - Optional.ofNullable(jobType.region(controller.system()))); - } else { - // Ignore region for test environments as it's not specified in deployment spec - return z.environment() == jobType.environment(); - } - }) + .filter(zone -> zone.deploysTo( + jobType.environment(), + jobType.isProduction() ? jobType.region(controller.system()) : Optional.empty())) .findFirst(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployer.java index 9e8f902a8db..c3424b8d9af 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployer.java @@ -3,12 +3,13 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.application.ApplicationList; +import com.yahoo.vespa.hosted.controller.application.JobStatus; import java.time.Duration; import java.time.Instant; -import java.util.ArrayList; +import java.util.Comparator; import java.util.List; +import java.util.Optional; /** * Attempts redeployment of failed jobs and deployments. @@ -16,6 +17,8 @@ import java.util.List; * @author bratseth */ public class FailureRedeployer extends Maintainer { + + private final static Duration jobTimeout = Duration.ofHours(12); public FailureRedeployer(Controller controller, Duration interval, JobControl jobControl) { super(controller, interval, jobControl); @@ -23,20 +26,61 @@ public class FailureRedeployer extends Maintainer { @Override public void maintain() { - ApplicationList applications = ApplicationList.from(controller().applications().asList()).isDeploying(); - List<Application> toTrigger = new ArrayList<>(); + List<Application> applications = controller().applications().asList(); + retryFailingJobs(applications); + retryStuckJobs(applications); + } + + private void retryFailingJobs(List<Application> applications) { + for (Application application : applications) { + if (!application.deploying().isPresent()) { + continue; + } + if (application.deploymentJobs().inProgress()) { + continue; + } + Optional<JobStatus> failingJob = jobFailingFor(application); + failingJob.ifPresent(job -> triggerFailing(application, "Job " + job.type().id() + + " has been failing since " + job.firstFailing().get())); + } + } - // Applications with deployment failures for current change and no running jobs - toTrigger.addAll(applications.hasDeploymentFailures() - .notRunningJob() - .asList()); + private void retryStuckJobs(List<Application> applications) { + Instant maxAge = controller().clock().instant().minus(jobTimeout); + for (Application application : applications) { + if (!application.deploying().isPresent()) { + continue; + } + Optional<JobStatus> job = oldestRunningJob(application); + if (!job.isPresent()) { + continue; + } + // Ignore job if it doesn't belong to a zone in this system + if (!job.get().type().zone(controller().system()).isPresent()) { + continue; + } + if (job.get().lastTriggered().get().at().isBefore(maxAge)) { + triggerFailing(application, "Job " + job.get().type().id() + + " has been running for more than " + jobTimeout); + } + } + } + + private Optional<JobStatus> jobFailingFor(Application application) { + return application.deploymentJobs().jobStatus().values().stream() + .filter(status -> !status.isSuccess() && status.lastCompletedFor(application.deploying().get())) + .findFirst(); + } - // Applications with jobs that have been in progress for more than 12 hours - Instant twelveHoursAgo = controller().clock().instant().minus(Duration.ofHours(12)); - toTrigger.addAll(applications.jobRunningSince(twelveHoursAgo).asList()); + private Optional<JobStatus> oldestRunningJob(Application application) { + return application.deploymentJobs().jobStatus().values().stream() + .filter(JobStatus::inProgress) + .sorted(Comparator.comparing(status -> status.lastTriggered().get().at())) + .findFirst(); + } - toTrigger.forEach(application -> controller().applications().deploymentTrigger() - .triggerFailing(application.id())); + private void triggerFailing(Application application, String cause) { + controller().applications().deploymentTrigger().triggerFailing(application.id(), cause); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java index b3d75106d2f..22d0a56c367 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java @@ -47,9 +47,11 @@ public class Upgrader extends Maintainer { switch (target.confidence()) { case broken: - log.info(String.format("Version %s is broken, cancelling all upgrades", target.versionNumber())); - cancelUpgradesOf(applications().upgradingTo(target.versionNumber()) - .without(UpgradePolicy.canary)); // keep trying canaries + ApplicationList toCancel = applications().upgradingTo(target.versionNumber()) + .without(UpgradePolicy.canary); + if (toCancel.isEmpty()) break; + log.info("Version " + target.versionNumber() + " is broken, cancelling all upgrades"); + cancelUpgradesOf(toCancel); break; case low: upgrade(applications().with(UpgradePolicy.canary), target.versionNumber()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java index 014c63a6779..04999f9e1c3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java @@ -241,14 +241,7 @@ public class ApplicationSerializer { private List<JobStatus> jobStatusListFromSlime(Inspector array) { List<JobStatus> jobStatusList = new ArrayList<>(); - array.traverse((ArrayTraverser) (int i, Inspector item) -> { - // TODO: This zone has been removed. Remove after Aug 2017 - String jobId = item.field(jobTypeField).asString(); - if ("production-ap-aue-1".equals(jobId)) { - return; - } - jobStatusList.add(jobStatusFromSlime(item)); - }); + array.traverse((ArrayTraverser) (int i, Inspector item) -> jobStatusList.add(jobStatusFromSlime(item))); return jobStatusList; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index e807762371a..3dddfaf58a1 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -11,6 +11,9 @@ import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.api.Tenant; import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus; @@ -31,6 +34,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.athens.NToken; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.ApplicationRevision; import com.yahoo.vespa.hosted.controller.application.Change; +import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobReport; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType; @@ -38,6 +42,7 @@ import com.yahoo.vespa.hosted.controller.application.JobStatus; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.BuildSystem; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; +import com.yahoo.vespa.hosted.controller.persistence.ApplicationSerializer; import com.yahoo.vespa.hosted.controller.versions.DeploymentStatistics; import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; @@ -46,6 +51,8 @@ import com.yahoo.vespa.hosted.controller.api.integration.athens.mock.NTokenMock; import org.junit.Test; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; import java.time.Duration; import java.time.Instant; import java.util.ArrayList; @@ -53,6 +60,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.function.Supplier; import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.component; import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.productionCorpUsEast1; @@ -212,13 +220,9 @@ public class ControllerTest { assertEquals(systemVersion, tester.configServerClientMock().lastPrepareVersion.get()); // Unexpected deployment - try { - tester.deploy(productionUsWest1, app1, applicationPackage); - fail("Expected exception as no change was to be deployed"); - } - catch (IllegalArgumentException expected) { - // success - } + tester.deploy(productionUsWest1, app1, applicationPackage); + // applications are immutable, so any change to one, including deployment changes, would give rise to a new instance. + assertEquals("Unexpected deployment is ignored", app1, applications.require(app1.id())); // Application change after a new system version, and a region added Version newSystemVersion = incrementSystemVersion(tester.controller()); @@ -598,4 +602,69 @@ public class ControllerTest { new DeployOptions(Optional.of(new ScrewdriverBuildJob(app1ScrewdriverId, app1RevisionId)), version, false, deployCurrentVersion)); } + + @Test + public void testCleanupOfStaleDeploymentData() throws IOException { + DeploymentTester tester = new DeploymentTester(); + tester.controllerTester().getZoneRegistryMock().setSystem(SystemName.cd); + + Supplier<Map<JobType, JobStatus>> statuses = () -> + tester.application(ApplicationId.from("vespa", "canary", "default")).deploymentJobs().jobStatus(); + + // Current system version, matches version in test data + Version version = Version.fromString("6.141.117"); + Version oldVersion = Version.fromString("6.98.12"); + tester.configServerClientMock().setDefaultConfigServerVersion(version); + tester.updateVersionStatus(version); + assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber()); + + // Load test data data + ApplicationSerializer serializer = new ApplicationSerializer(); + byte[] json = Files.readAllBytes(Paths.get("src/test/java/com/yahoo/vespa/hosted/controller/maintenance/testdata/canary-with-stale-data.json")); + Slime slime = SlimeUtils.jsonToSlime(json); + Application application = serializer.fromSlime(slime); + try (Lock lock = tester.controller().applications().lock(application.id())) { + tester.controller().applications().store(application, lock); + } + + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .upgradePolicy("canary") + .region("cd-us-central-1") + .build(); + + long cdJobsCount = statuses.get().keySet().stream() + .filter(type -> type.zone(SystemName.cd).isPresent()) + .count(); + + long mainJobsCount = statuses.get().keySet().stream() + .filter(type -> type.zone(SystemName.main).isPresent() && ! type.zone(SystemName.cd).isPresent()) + .count(); + + assertEquals("Irrelevant (main) data is present.", 8, mainJobsCount); + + // New version is released + version = Version.fromString("6.142.1"); + tester.configServerClientMock().setDefaultConfigServerVersion(version); + tester.updateVersionStatus(version); + assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber()); + tester.upgrader().maintain(); + + // Test environments pass + tester.deploy(DeploymentJobs.JobType.systemTest, application, applicationPackage); + tester.buildSystem().takeJobsToRun(); + tester.clock().advance(Duration.ofMinutes(10)); + tester.notifyJobCompletion(DeploymentJobs.JobType.systemTest, application, true); + + long newCdJobsCount = statuses.get().keySet().stream() + .filter(type -> type.zone(SystemName.cd).isPresent()) + .count(); + + long newMainJobsCount = statuses.get().keySet().stream() + .filter(type -> type.zone(SystemName.main).isPresent() && ! type.zone(SystemName.cd).isPresent()) + .count(); + + assertEquals("Irrelevant (main) job data is removed.", 0, newMainJobsCount); + assertEquals("Relevant (cd) data is not removed.", cdJobsCount, newCdJobsCount); + } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ZoneRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ZoneRegistryMock.java index 62b935842f7..bf21467bc8d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ZoneRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ZoneRegistryMock.java @@ -27,9 +27,11 @@ public class ZoneRegistryMock implements ZoneRegistry { deploymentTimeToLive.put(zone, duration); } + private SystemName system = SystemName.main; + @Override public SystemName system() { - return SystemName.main; + return system; } @Override @@ -71,4 +73,8 @@ public class ZoneRegistryMock implements ZoneRegistry { public URI getDashboardUri() { return URI.create("http://dashboard.test"); } + + public void setSystem(SystemName system) { + this.system = system; + } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java index 32d1714ea52..0e816be864d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java @@ -160,7 +160,7 @@ public class DeploymentTester { } public void deploy(JobType job, Application application, ApplicationPackage applicationPackage, boolean deployCurrentVersion) { - job.zone(SystemName.main).ifPresent(zone -> tester.deploy(application, zone, applicationPackage, deployCurrentVersion)); + job.zone(controller().system()).ifPresent(zone -> tester.deploy(application, zone, applicationPackage, deployCurrentVersion)); } public void deployAndNotify(JobType job, Application application, ApplicationPackage applicationPackage, boolean success) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java index ce06910240b..0f48afc0ca4 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java @@ -34,14 +34,14 @@ public class DeploymentTriggerTest { tester.buildSystem().takeJobsToRun(); assertEquals("Job removed", 0, tester.buildSystem().jobs().size()); tester.clock().advance(Duration.ofHours(2)); - tester.deploymentTrigger().triggerFailing(app1.id()); + tester.deploymentTrigger().triggerFailing(app1.id(), "unit test"); assertEquals("Retried job", 1, tester.buildSystem().jobs().size()); assertEquals(JobType.stagingTest.id(), tester.buildSystem().jobs().get(0).jobName()); tester.buildSystem().takeJobsToRun(); assertEquals("Job removed", 0, tester.buildSystem().jobs().size()); tester.clock().advance(Duration.ofHours(7)); - tester.deploymentTrigger().triggerFailing(app1.id()); + tester.deploymentTrigger().triggerFailing(app1.id(), "unit test"); assertEquals("Retried from the beginning", 1, tester.buildSystem().jobs().size()); assertEquals(JobType.component.id(), tester.buildSystem().jobs().get(0).jobName()); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java index cde511a9076..052ca87f791 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java @@ -3,13 +3,20 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.SystemName; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; +import com.yahoo.vespa.hosted.controller.persistence.ApplicationSerializer; import org.junit.Test; +import java.nio.file.Files; +import java.nio.file.Paths; import java.time.Duration; import static org.junit.Assert.assertEquals; @@ -166,4 +173,60 @@ public class FailureRedeployerTest { assertTrue("No jobs retried", tester.buildSystem().jobs().isEmpty()); } + @Test + public void retryIgnoresStaleJobData() throws Exception { + DeploymentTester tester = new DeploymentTester(); + tester.controllerTester().getZoneRegistryMock().setSystem(SystemName.cd); + + // Current system version, matches version in test data + Version version = Version.fromString("6.141.117"); + tester.configServerClientMock().setDefaultConfigServerVersion(version); + tester.updateVersionStatus(version); + assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber()); + + // Load test data data + ApplicationSerializer serializer = new ApplicationSerializer(); + byte[] json = Files.readAllBytes(Paths.get("src/test/java/com/yahoo/vespa/hosted/controller/maintenance/testdata/canary-with-stale-data.json")); + Slime slime = SlimeUtils.jsonToSlime(json); + Application application = serializer.fromSlime(slime); + try (Lock lock = tester.controller().applications().lock(application.id())) { + tester.controller().applications().store(application, lock); + } + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .upgradePolicy("canary") + .region("cd-us-central-1") + .build(); + + // New version is released + version = Version.fromString("6.142.1"); + tester.configServerClientMock().setDefaultConfigServerVersion(version); + tester.updateVersionStatus(version); + assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber()); + tester.upgrader().maintain(); + + // Test environments pass + tester.deploy(DeploymentJobs.JobType.systemTest, application, applicationPackage); + tester.buildSystem().takeJobsToRun(); + tester.clock().advance(Duration.ofMinutes(10)); + tester.notifyJobCompletion(DeploymentJobs.JobType.systemTest, application, true); + + tester.deploy(DeploymentJobs.JobType.stagingTest, application, applicationPackage); + tester.buildSystem().takeJobsToRun(); + tester.clock().advance(Duration.ofMinutes(10)); + tester.notifyJobCompletion(DeploymentJobs.JobType.stagingTest, application, true); + + // Production job starts, but does not complete + assertEquals(1, tester.buildSystem().jobs().size()); + assertEquals("Production job triggered", DeploymentJobs.JobType.productionCdUsCentral1.id(), tester.buildSystem().jobs().get(0).jobName()); + tester.buildSystem().takeJobsToRun(); + + // Failure re-deployer runs + tester.failureRedeployer().maintain(); + assertTrue("No jobs retried", tester.buildSystem().jobs().isEmpty()); + + // Deployment completes + tester.notifyJobCompletion(DeploymentJobs.JobType.productionCdUsCentral1, application, true); + assertFalse("Change deployed", tester.application(application.id()).deploying().isPresent()); + } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/testdata/canary-with-stale-data.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/testdata/canary-with-stale-data.json new file mode 100644 index 00000000000..323889c7c45 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/testdata/canary-with-stale-data.json @@ -0,0 +1,295 @@ +{ + "id": "vespa:canary:default", + "deploymentSpecField": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<deployment version=\"1.0\">\n <upgrade policy='canary'/>\n <test />\n <staging />\n <prod>\n <region active=\"true\">cd-us-central-1</region>\n </prod>\n</deployment>\n", + "validationOverrides": "<validation-overrides>\n <allow until=\"2017-04-27\">force-automatic-tenant-upgrade-test</allow>\n</validation-overrides>\n", + "deployments": [ + { + "zone": { + "environment": "prod", + "region": "cd-us-central-1" + }, + "version": "6.141.117", + "deployTime": 1503901783487, + "applicationPackageRevision": { + "applicationPackageHash": "72c3961314b96b1155a310f4785ae57ec74b1273", + "sourceRevision": { + "repositoryField": "git@git.test:vespa/canary-application.git", + "branchField": "origin/canary-cd", + "commitField": "566b7b30ee7886b845bb70958a0e1bdab2868633" + } + } + } + ], + "deploymentJobs": { + "projectId": 191186, + "jobStatus": [ + { + "jobType": "production-eu-west-1", + "lastTriggered": { + "version": "6.98.12", + "at": 1493034019032 + }, + "lastCompleted": { + "version": "6.98.12", + "at": 1493033995026 + }, + "lastSuccess": { + "version": "6.98.12", + "at": 1493033995026 + } + }, + { + "jobType": "production-cd-us-central-1", + "jobError": "unknown", + "lastTriggered": { + "version": "6.141.117", + "revision": { + "applicationPackageHash": "72c3961314b96b1155a310f4785ae57ec74b1273", + "sourceRevision": { + "repositoryField": "git@git.test:vespa/canary-application.git", + "branchField": "origin/canary-cd", + "commitField": "566b7b30ee7886b845bb70958a0e1bdab2868633" + } + }, + "at": 1503903384816 + }, + "lastCompleted": { + "version": "6.141.117", + "revision": { + "applicationPackageHash": "72c3961314b96b1155a310f4785ae57ec74b1273", + "sourceRevision": { + "repositoryField": "git@git.test:vespa/canary-application.git", + "branchField": "origin/canary-cd", + "commitField": "566b7b30ee7886b845bb70958a0e1bdab2868633" + } + }, + "at": 1503903659364 + }, + "firstFailing": { + "version": "6.141.117", + "revision": { + "applicationPackageHash": "72c3961314b96b1155a310f4785ae57ec74b1273", + "sourceRevision": { + "repositoryField": "git@git.test:vespa/canary-application.git", + "branchField": "origin/canary-cd", + "commitField": "566b7b30ee7886b845bb70958a0e1bdab2868633" + } + }, + "at": 1503903659364 + }, + "lastSuccess": { + "version": "6.141.117", + "revision": { + "applicationPackageHash": "72c3961314b96b1155a310f4785ae57ec74b1273", + "sourceRevision": { + "repositoryField": "git@git.test:vespa/canary-application.git", + "branchField": "origin/canary-cd", + "commitField": "566b7b30ee7886b845bb70958a0e1bdab2868633" + } + }, + "at": 1503903556127 + } + }, + { + "jobType": "component", + "lastTriggered": { + "version": "6.141.109", + "at": 1503867517105 + }, + "lastCompleted": { + "version": "6.141.109", + "at": 1503867704464 + }, + "lastSuccess": { + "version": "6.141.109", + "at": 1503867704464 + } + }, + { + "jobType": "production-corp-us-east-1", + "lastTriggered": { + "version": "6.98.12", + "at": 1493034428590 + }, + "lastCompleted": { + "version": "6.98.12", + "at": 1493034114538 + }, + "lastSuccess": { + "version": "6.98.12", + "at": 1493034114538 + } + }, + { + "jobType": "production-ap-southeast-1", + "lastTriggered": { + "version": "6.98.12", + "at": 1493034265146 + }, + "lastCompleted": { + "version": "6.98.12", + "at": 1493034097617 + }, + "lastSuccess": { + "version": "6.98.12", + "at": 1493034097617 + } + }, + { + "jobType": "production-us-central-1", + "lastTriggered": { + "version": "6.98.12", + "at": 1493033800484 + }, + "lastCompleted": { + "version": "6.98.12", + "at": 1493034273753 + }, + "lastSuccess": { + "version": "6.98.12", + "at": 1493034273753 + } + }, + { + "jobType": "staging-test", + "lastTriggered": { + "version": "6.141.117", + "revision": { + "applicationPackageHash": "72c3961314b96b1155a310f4785ae57ec74b1273", + "sourceRevision": { + "repositoryField": "git@git.test:vespa/canary-application.git", + "branchField": "origin/canary-cd", + "commitField": "566b7b30ee7886b845bb70958a0e1bdab2868633" + } + }, + "at": 1503900683154 + }, + "lastCompleted": { + "version": "6.141.117", + "revision": { + "applicationPackageHash": "72c3961314b96b1155a310f4785ae57ec74b1273", + "sourceRevision": { + "repositoryField": "git@git.test:vespa/canary-application.git", + "branchField": "origin/canary-cd", + "commitField": "566b7b30ee7886b845bb70958a0e1bdab2868633" + } + }, + "at": 1503901635745 + }, + "lastSuccess": { + "version": "6.141.117", + "revision": { + "applicationPackageHash": "72c3961314b96b1155a310f4785ae57ec74b1273", + "sourceRevision": { + "repositoryField": "git@git.test:vespa/canary-application.git", + "branchField": "origin/canary-cd", + "commitField": "566b7b30ee7886b845bb70958a0e1bdab2868633" + } + }, + "at": 1503901635745 + } + }, + { + "jobType": "system-test", + "lastTriggered": { + "version": "6.141.117", + "revision": { + "applicationPackageHash": "72c3961314b96b1155a310f4785ae57ec74b1273", + "sourceRevision": { + "repositoryField": "git@git.test:vespa/canary-application.git", + "branchField": "origin/canary-cd", + "commitField": "566b7b30ee7886b845bb70958a0e1bdab2868633" + } + }, + "at": 1503899621243 + }, + "lastCompleted": { + "version": "6.141.117", + "revision": { + "applicationPackageHash": "72c3961314b96b1155a310f4785ae57ec74b1273", + "sourceRevision": { + "repositoryField": "git@git.test:vespa/canary-application.git", + "branchField": "origin/canary-cd", + "commitField": "566b7b30ee7886b845bb70958a0e1bdab2868633" + } + }, + "at": 1503900025214 + }, + "lastSuccess": { + "version": "6.141.117", + "revision": { + "applicationPackageHash": "72c3961314b96b1155a310f4785ae57ec74b1273", + "sourceRevision": { + "repositoryField": "git@git.test:vespa/canary-application.git", + "branchField": "origin/canary-cd", + "commitField": "566b7b30ee7886b845bb70958a0e1bdab2868633" + } + }, + "at": 1503900025214 + } + }, + { + "jobType": "production-us-west-1", + "lastTriggered": { + "version": "6.98.12", + "at": 1493034273768 + }, + "lastCompleted": { + "version": "6.98.12", + "at": 1493034019015 + }, + "lastSuccess": { + "version": "6.98.12", + "at": 1493034019015 + } + }, + { + "jobType": "production-ap-northeast-1", + "lastTriggered": { + "version": "6.98.12", + "at": 1493033995045 + }, + "lastCompleted": { + "version": "6.98.12", + "at": 1493034257206 + }, + "lastSuccess": { + "version": "6.98.12", + "at": 1493034257206 + } + }, + { + "jobType": "production-ap-northeast-2", + "lastTriggered": { + "version": "6.98.12", + "at": 1493034257222 + }, + "lastCompleted": { + "version": "6.98.12", + "at": 1493034265048 + }, + "lastSuccess": { + "version": "6.98.12", + "at": 1493034265048 + } + }, + { + "jobType": "production-us-east-3", + "lastTriggered": { + "version": "6.98.12", + "at": 1493034114555 + }, + "lastCompleted": { + "version": "6.98.12", + "at": 1493033800469 + }, + "lastSuccess": { + "version": "6.98.12", + "at": 1493033800469 + } + } + ], + "selfTriggering": false + }, + "outstandingChangeField": false +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java index 645e38d0f2d..20e3aae9114 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java @@ -133,41 +133,6 @@ public class ApplicationSerializerTest { assertEquals(JobError.unknown, applicationWithFailingJob.deploymentJobs().jobStatus().get(DeploymentJobs.JobType.systemTest).jobError().get()); } - // TODO: Remove after Aug 2017 - @Test - public void serializeWithRemovedZone() throws Exception { - String json = "{\n" + - " \"id\": \"t1:a1:i1\",\n" + - " \"deploymentSpecField\": \"<deployment version='1.0'/>\",\n" + - " \"deploymentJobs\": {\n" + - " \"projectId\": 123,\n" + - " \"jobStatus\": [\n" + - " {\n" + - " \"jobType\": \"system-test\",\n" + - " \"version\": \"5.6.7\",\n" + - " \"completionTime\": 7,\n" + - " \"lastTriggered\": 8\n" + - " },\n" + - " {\n" + - " \"jobType\": \"production-ap-aue-1\",\n" + - " \"version\": \"5.6.7\",\n" + - " \"completionTime\": 7,\n" + - " \"lastTriggered\": 8\n" + - " },\n" + - " {\n" + - " \"jobType\": \"staging-test\",\n" + - " \"version\": \"5.6.7\",\n" + - " \"completionTime\": 7,\n" + - " \"lastTriggered\": 8\n" + - " }\n" + - " ],\n" + - " \"selfTriggering\": false\n" + - " }\n" + - "}\n"; - Application app = applicationSerializer.fromSlime(SlimeUtils.jsonToSlime(json.getBytes(StandardCharsets.UTF_8))); - assertEquals(2, app.deploymentJobs().jobStatus().size()); - } - private Slime applicationSlime(boolean error) { return SlimeUtils.jsonToSlime(applicationJson(error).getBytes(StandardCharsets.UTF_8)); } diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImpl.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImpl.java index f633c0a3971..e676a86d9fd 100644 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImpl.java +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImpl.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.hosted.dockerapi; import com.github.dockerjava.api.DockerClient; import com.github.dockerjava.api.command.CreateContainerCmd; -import com.github.dockerjava.api.exception.DockerException; import com.github.dockerjava.api.model.Bind; import com.github.dockerjava.api.model.Capability; import com.github.dockerjava.api.model.Ulimit; @@ -133,8 +132,8 @@ class CreateContainerCommandImpl implements Docker.CreateContainerCommand { public void create() { try { createCreateContainerCmd().exec(); - } catch (DockerException e) { - throw new RuntimeException("Failed to create container " + containerName.asString(), e); + } catch (RuntimeException e) { + throw new DockerException("Failed to create container " + containerName.asString(), e); } } diff --git a/docproc/src/main/resources/configdefinitions/docproc.def b/docproc/src/main/resources/configdefinitions/docproc.def index 76acfdfca8a..3b4c4fc5b16 100644 --- a/docproc/src/main/resources/configdefinitions/docproc.def +++ b/docproc/src/main/resources/configdefinitions/docproc.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=12 namespace=config.docproc # Queue size (in milliseconds) for this node diff --git a/docproc/src/main/resources/configdefinitions/schemamapping.def b/docproc/src/main/resources/configdefinitions/schemamapping.def index 87afe204770..d70235fbfc0 100644 --- a/docproc/src/main/resources/configdefinitions/schemamapping.def +++ b/docproc/src/main/resources/configdefinitions/schemamapping.def @@ -1,7 +1,6 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. # Use when a docproc works on a generic set of field names and the actual names # in input doc may be different -version=1 namespace=config.docproc # The chain this mapping applies to diff --git a/docproc/src/main/resources/configdefinitions/splitter-joiner-document-processor.def b/docproc/src/main/resources/configdefinitions/splitter-joiner-document-processor.def index c96a6c63d3c..dea6141e6cc 100644 --- a/docproc/src/main/resources/configdefinitions/splitter-joiner-document-processor.def +++ b/docproc/src/main/resources/configdefinitions/splitter-joiner-document-processor.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=2 namespace=config.docproc # The name of the enclosing (outer) document type diff --git a/docproc/src/test/vespa-configdef/string.def b/docproc/src/test/vespa-configdef/string.def index 9e59f7caec4..3e76855f1e8 100644 --- a/docproc/src/test/vespa-configdef/string.def +++ b/docproc/src/test/vespa-configdef/string.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=1 namespace=config.docproc stringVal string default="_default_" diff --git a/document/CMakeLists.txt b/document/CMakeLists.txt index e9694390b4b..ca2ee029c87 100644 --- a/document/CMakeLists.txt +++ b/document/CMakeLists.txt @@ -8,10 +8,6 @@ vespa_define_module( config_cloudconfig vespaeval - EXTERNAL_DEPENDS - lz4 - zstd - LIBS src/vespa/document src/vespa/document/annotation diff --git a/document/src/tests/documenttestcase.cpp b/document/src/tests/documenttestcase.cpp index 44cff2c80de..c3f88ac3696 100644 --- a/document/src/tests/documenttestcase.cpp +++ b/document/src/tests/documenttestcase.cpp @@ -16,6 +16,7 @@ #include <fcntl.h> using vespalib::nbostream; +using vespalib::compression::CompressionConfig; using namespace document::config_builder; diff --git a/document/src/tests/repo/documenttyperepo_test.cpp b/document/src/tests/repo/documenttyperepo_test.cpp index 508e6f237cf..1e8fd9ec470 100644 --- a/document/src/tests/repo/documenttyperepo_test.cpp +++ b/document/src/tests/repo/documenttyperepo_test.cpp @@ -10,7 +10,6 @@ #include <vespa/document/datatype/weightedsetdatatype.h> #include <vespa/document/repo/configbuilder.h> #include <vespa/document/repo/documenttyperepo.h> -#include <stdlib.h> #include <vespa/vespalib/objects/identifiable.h> #include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/testkit/testapp.h> @@ -25,6 +24,7 @@ using std::vector; using vespalib::Identifiable; using vespalib::IllegalArgumentException; using vespalib::string; +using vespalib::compression::CompressionConfig; using namespace document::config_builder; using namespace document; diff --git a/document/src/tests/serialization/.gitignore b/document/src/tests/serialization/.gitignore index 9f5bc440533..7e94d2757ae 100644 --- a/document/src/tests/serialization/.gitignore +++ b/document/src/tests/serialization/.gitignore @@ -2,5 +2,4 @@ .depend Makefile document_annotationserializer_test_app -document_compression_test_app document_vespadocumentserializer_test_app diff --git a/document/src/tests/serialization/CMakeLists.txt b/document/src/tests/serialization/CMakeLists.txt index 5b23bffad26..e1ce43b12d4 100644 --- a/document/src/tests/serialization/CMakeLists.txt +++ b/document/src/tests/serialization/CMakeLists.txt @@ -17,12 +17,3 @@ vespa_add_executable(document_annotationserializer_test_app TEST document_documentconfig ) vespa_add_test(NAME document_annotationserializer_test_app COMMAND document_annotationserializer_test_app) -vespa_add_executable(document_compression_test_app TEST - SOURCES - compression_test.cpp - DEPENDS - document - AFTER - document_documentconfig -) -vespa_add_test(NAME document_compression_test_app COMMAND document_compression_test_app) diff --git a/document/src/tests/serialization/vespadocumentserializer_test.cpp b/document/src/tests/serialization/vespadocumentserializer_test.cpp index d09012a4e4b..9da20e5a84c 100644 --- a/document/src/tests/serialization/vespadocumentserializer_test.cpp +++ b/document/src/tests/serialization/vespadocumentserializer_test.cpp @@ -55,6 +55,7 @@ using vespalib::tensor::Tensor; using vespalib::tensor::TensorBuilder; using vespalib::tensor::TensorCells; using vespalib::tensor::TensorDimensions; +using vespalib::compression::CompressionConfig; using namespace document; using std::string; using std::vector; diff --git a/document/src/vespa/document/config/documentmanager.def b/document/src/vespa/document/config/documentmanager.def index 9fa8c82800b..1961f67d83c 100644 --- a/document/src/vespa/document/config/documentmanager.def +++ b/document/src/vespa/document/config/documentmanager.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=15 namespace=document.config diff --git a/document/src/vespa/document/config/documenttypes.def b/document/src/vespa/document/config/documenttypes.def index 7501c5b0d5c..c0c4f50ab86 100644 --- a/document/src/vespa/document/config/documenttypes.def +++ b/document/src/vespa/document/config/documenttypes.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=15 namespace=document diff --git a/document/src/vespa/document/datatype/structdatatype.h b/document/src/vespa/document/datatype/structdatatype.h index f6c77f18f47..0f1c58316c9 100644 --- a/document/src/vespa/document/datatype/structdatatype.h +++ b/document/src/vespa/document/datatype/structdatatype.h @@ -11,15 +11,16 @@ #include <vespa/document/datatype/structureddatatype.h> #include <vespa/vespalib/stllike/hash_map.h> -#include <vespa/document/util/compressionconfig.h> +#include <vespa/vespalib/util/compressionconfig.h> #include <memory> namespace document { class StructDataType final : public StructuredDataType { public: - typedef std::unique_ptr<StructDataType> UP; - typedef std::shared_ptr<StructDataType> SP; + using UP = std::unique_ptr<StructDataType>; + using SP = std::shared_ptr<StructDataType>; + using CompressionConfig = vespalib::compression::CompressionConfig; StructDataType(); StructDataType(const vespalib::stringref &name); diff --git a/document/src/vespa/document/fieldvalue/serializablearray.cpp b/document/src/vespa/document/fieldvalue/serializablearray.cpp index 76216d75f1b..5dfd8eff891 100644 --- a/document/src/vespa/document/fieldvalue/serializablearray.cpp +++ b/document/src/vespa/document/fieldvalue/serializablearray.cpp @@ -2,7 +2,7 @@ #include "serializablearray.h" #include <vespa/document/util/serializableexceptions.h> #include <vespa/document/util/bytebuffer.h> -#include <vespa/document/util/compressor.h> +#include <vespa/vespalib/util/compressor.h> #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/vespalib/data/databuffer.h> #include <algorithm> @@ -181,7 +181,7 @@ SerializableArray::clear(int id) void SerializableArray::deCompress() // throw (DeserializeException) { - using document::compression::decompress; + using vespalib::compression::decompress; // will only do this once LOG_ASSERT(_compSerData); @@ -239,7 +239,7 @@ void SerializableArray::assign(EntryMap & entries, } } -CompressionInfo +vespalib::compression::CompressionInfo SerializableArray::getCompressionInfo() const { return CompressionInfo(_uncompressedLength, _compSerData->getRemaining()); } diff --git a/document/src/vespa/document/fieldvalue/serializablearray.h b/document/src/vespa/document/fieldvalue/serializablearray.h index 2d12c1191a6..2f7d65938aa 100644 --- a/document/src/vespa/document/fieldvalue/serializablearray.h +++ b/document/src/vespa/document/fieldvalue/serializablearray.h @@ -16,7 +16,7 @@ #pragma once -#include <vespa/document/util/compressionconfig.h> +#include <vespa/vespalib/util/compressionconfig.h> #include <vespa/vespalib/objects/cloneable.h> #include <vespa/vespalib/util/buffer.h> #include <vespa/vespalib/util/memory.h> @@ -107,6 +107,8 @@ public: using CP = vespalib::CloneablePtr<SerializableArray>; using UP = std::unique_ptr<SerializableArray>; using ByteBufferUP = std::unique_ptr<ByteBuffer>; + using CompressionConfig = vespalib::compression::CompressionConfig; + using CompressionInfo = vespalib::compression::CompressionInfo; SerializableArray(); virtual ~SerializableArray(); diff --git a/document/src/vespa/document/fieldvalue/structfieldvalue.cpp b/document/src/vespa/document/fieldvalue/structfieldvalue.cpp index c0ae342fd34..0a05ae60600 100644 --- a/document/src/vespa/document/fieldvalue/structfieldvalue.cpp +++ b/document/src/vespa/document/fieldvalue/structfieldvalue.cpp @@ -21,6 +21,7 @@ using std::vector; using vespalib::nbostream; using vespalib::nbostream_longlivedbuf; using vespalib::make_string; +using vespalib::compression::CompressionConfig; using namespace vespalib::xml; namespace document { diff --git a/document/src/vespa/document/fieldvalue/structfieldvalue.h b/document/src/vespa/document/fieldvalue/structfieldvalue.h index 6ab6f71ce56..bb6956cf012 100644 --- a/document/src/vespa/document/fieldvalue/structfieldvalue.h +++ b/document/src/vespa/document/fieldvalue/structfieldvalue.h @@ -52,7 +52,9 @@ private: mutable bool _hasChanged; public: - typedef std::unique_ptr<StructFieldValue> UP; + using UP = std::unique_ptr<StructFieldValue>; + using CompressionConfig = vespalib::compression::CompressionConfig; + StructFieldValue(const DataType &type); ~StructFieldValue(); void swap(StructFieldValue & rhs); diff --git a/document/src/vespa/document/repo/documenttyperepo.cpp b/document/src/vespa/document/repo/documenttyperepo.cpp index 870e88e5036..6bfae246c10 100644 --- a/document/src/vespa/document/repo/documenttyperepo.cpp +++ b/document/src/vespa/document/repo/documenttyperepo.cpp @@ -30,6 +30,7 @@ using vespalib::hash_map; using vespalib::make_string; using vespalib::string; using vespalib::stringref; +using vespalib::compression::CompressionConfig; namespace document { diff --git a/document/src/vespa/document/serialization/vespadocumentdeserializer.cpp b/document/src/vespa/document/serialization/vespadocumentdeserializer.cpp index bee1c258e4a..2b45e8a298c 100644 --- a/document/src/vespa/document/serialization/vespadocumentdeserializer.cpp +++ b/document/src/vespa/document/serialization/vespadocumentdeserializer.cpp @@ -38,6 +38,7 @@ using vespalib::asciistream; using vespalib::nbostream; using vespalib::Memory; using vespalib::stringref; +using vespalib::compression::CompressionConfig; namespace document { diff --git a/document/src/vespa/document/serialization/vespadocumentserializer.cpp b/document/src/vespa/document/serialization/vespadocumentserializer.cpp index 2534690b014..6c9e95a9dd6 100644 --- a/document/src/vespa/document/serialization/vespadocumentserializer.cpp +++ b/document/src/vespa/document/serialization/vespadocumentserializer.cpp @@ -27,7 +27,7 @@ #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/data/databuffer.h> #include <vespa/eval/tensor/serialization/typed_binary_format.h> -#include <vespa/document/util/compressor.h> +#include <vespa/vespalib/util/compressor.h> using std::make_pair; using std::pair; @@ -36,6 +36,7 @@ using vespalib::nbostream; using vespalib::stringref; using vespalib::string; using vespalib::slime::BinaryFormat; +using vespalib::compression::CompressionConfig; namespace document { @@ -275,7 +276,7 @@ vespalib::ConstBufferRef compressStream(const CompressionConfig &config, nbostream &stream, vespalib::DataBuffer & compressed_data) { - using compression::compress; + using vespalib::compression::compress; vespalib::ConstBufferRef buf(stream.c_str(), stream.size()); if (config.useCompression() && bigEnough(stream.size(), config)) { CompressionConfig::Type compressedType = compress(config, vespalib::ConstBufferRef(stream.c_str(), stream.size()), compressed_data, false); diff --git a/document/src/vespa/document/util/CMakeLists.txt b/document/src/vespa/document/util/CMakeLists.txt index 2179b1307d3..8cb148abe25 100644 --- a/document/src/vespa/document/util/CMakeLists.txt +++ b/document/src/vespa/document/util/CMakeLists.txt @@ -2,9 +2,6 @@ vespa_add_library(document_util OBJECT SOURCES bytebuffer.cpp - compressor.cpp - lz4compressor.cpp - zstdcompressor.cpp printable.cpp serializable.cpp stringutil.cpp diff --git a/documentapi/src/main/resources/configdefinitions/documentrouteselectorpolicy.def b/documentapi/src/main/resources/configdefinitions/documentrouteselectorpolicy.def index be5d14e96ca..f6f4461d220 100644 --- a/documentapi/src/main/resources/configdefinitions/documentrouteselectorpolicy.def +++ b/documentapi/src/main/resources/configdefinitions/documentrouteselectorpolicy.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=2 namespace=documentapi.messagebus.protocol # The name of the route. diff --git a/documentapi/src/tests/policies/testframe.cpp b/documentapi/src/tests/policies/testframe.cpp index d231672da84..877e3164a8c 100644 --- a/documentapi/src/tests/policies/testframe.cpp +++ b/documentapi/src/tests/policies/testframe.cpp @@ -7,6 +7,7 @@ #include <vespa/messagebus/testlib/simplemessage.h> #include <vespa/messagebus/testlib/simpleprotocol.h> #include <vespa/messagebus/testlib/simplereply.h> +#include <vespa/messagebus/network/rpcnetworkparams.h> #include <vespa/log/log.h> LOG_SETUP(".testframe"); diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ActiveContainerDeactivationWatchdog.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ActiveContainerDeactivationWatchdog.java index 2eb4cc0baa9..e3f6576d293 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ActiveContainerDeactivationWatchdog.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ActiveContainerDeactivationWatchdog.java @@ -32,7 +32,7 @@ import static java.util.stream.Collectors.toList; class ActiveContainerDeactivationWatchdog implements ActiveContainerMetrics, AutoCloseable { static final Duration WATCHDOG_FREQUENCY = Duration.ofMinutes(20); static final Duration ACTIVE_CONTAINER_GRACE_PERIOD = Duration.ofHours(4); - static final Duration GC_TRIGGER_FREQUENCY = ACTIVE_CONTAINER_GRACE_PERIOD.minusMinutes(5); + static final Duration GC_TRIGGER_FREQUENCY = Duration.ofHours(1); // Must be a fraction of ACTIVE_CONTAINER_GRACE_PERIOD static final Duration ENFORCE_DESTRUCTION_GCED_CONTAINERS_FREQUENCY = Duration.ofMinutes(5); private static final Logger log = Logger.getLogger(ActiveContainerDeactivationWatchdog.class.getName()); diff --git a/jdisc_http_service/pom.xml b/jdisc_http_service/pom.xml index 2dfcb0bd166..0c19f2c3d15 100644 --- a/jdisc_http_service/pom.xml +++ b/jdisc_http_service/pom.xml @@ -22,10 +22,6 @@ <classifier>no_aop</classifier> </dependency> <dependency> - <groupId>io.netty</groupId> - <artifactId>netty</artifactId> - </dependency> - <dependency> <groupId>org.apache.commons</groupId> <artifactId>commons-lang3</artifactId> </dependency> diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/Cookie.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/Cookie.java index 8f04a870dc9..a43310aff51 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/Cookie.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/Cookie.java @@ -1,18 +1,21 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.jdisc.http; -import org.jboss.netty.handler.codec.http.cookie.ClientCookieDecoder; -import org.jboss.netty.handler.codec.http.cookie.ClientCookieEncoder; -import org.jboss.netty.handler.codec.http.cookie.DefaultCookie; -import org.jboss.netty.handler.codec.http.cookie.ServerCookieDecoder; -import org.jboss.netty.handler.codec.http.cookie.ServerCookieEncoder; +import org.eclipse.jetty.server.CookieCutter; +import org.eclipse.jetty.server.Response; +import java.net.HttpCookie; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Objects; import java.util.Set; +import java.util.StringTokenizer; import java.util.concurrent.TimeUnit; +import java.util.logging.Level; +import java.util.logging.Logger; import java.util.stream.Collectors; import java.util.stream.StreamSupport; @@ -26,6 +29,8 @@ import java.util.stream.StreamSupport; */ public class Cookie { + private final static Logger log = Logger.getLogger(Cookie.class.getName()); + private final Set<Integer> ports = new HashSet<>(); private String name; private String value; @@ -33,7 +38,7 @@ public class Cookie { private String path; private String comment; private String commentUrl; - private long maxAgeMillis = TimeUnit.SECONDS.toMillis(Integer.MIN_VALUE); + private long maxAgeSeconds = Integer.MIN_VALUE; private int version; private boolean secure; private boolean httpOnly; @@ -50,7 +55,7 @@ public class Cookie { path = cookie.path; comment = cookie.comment; commentUrl = cookie.commentUrl; - maxAgeMillis = cookie.maxAgeMillis; + maxAgeSeconds = cookie.maxAgeSeconds; version = cookie.version; secure = cookie.secure; httpOnly = cookie.httpOnly; @@ -131,11 +136,11 @@ public class Cookie { } public int getMaxAge(TimeUnit unit) { - return (int)unit.convert(maxAgeMillis, TimeUnit.MILLISECONDS); + return (int)unit.convert(maxAgeSeconds, TimeUnit.SECONDS); } public Cookie setMaxAge(int maxAge, TimeUnit unit) { - this.maxAgeMillis = unit.toMillis(maxAge); + this.maxAgeSeconds = maxAge >= 0 ? unit.toSeconds(maxAge) : Integer.MIN_VALUE; return this; } @@ -189,7 +194,7 @@ public class Cookie { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; Cookie cookie = (Cookie) o; - return maxAgeMillis == cookie.maxAgeMillis && + return maxAgeSeconds == cookie.maxAgeSeconds && version == cookie.version && secure == cookie.secure && httpOnly == cookie.httpOnly && @@ -205,7 +210,7 @@ public class Cookie { @Override public int hashCode() { - return Objects.hash(ports, name, value, domain, path, comment, commentUrl, maxAgeMillis, version, secure, httpOnly, discard); + return Objects.hash(ports, name, value, domain, path, comment, commentUrl, maxAgeSeconds, version, secure, httpOnly, discard); } @Override @@ -214,25 +219,51 @@ public class Cookie { ret.append(name).append("=").append(value); return ret.toString(); } + // NOTE cookie encoding and decoding: + // The implementation uses Jetty for server-side (encoding of Set-Cookie and decoding of Cookie header), + // and java.net.HttpCookie for client-side (encoding of Cookie and decoding of Set-Cookie header). + // + // Implementation is RFC-6265 compliant. public static String toCookieHeader(Iterable<? extends Cookie> cookies) { - ClientCookieEncoder encoder = ClientCookieEncoder.STRICT; - List<org.jboss.netty.handler.codec.http.cookie.Cookie> nettyCookies = - StreamSupport.stream(cookies.spliterator(), false) - // NOTE: Only name and value is included in Cookie header as of RFC-6265 - .map(cookie -> new DefaultCookie(cookie.getName(), cookie.getValue())) - .collect(Collectors.toList()); - return encoder.encode(nettyCookies); + return StreamSupport.stream(cookies.spliterator(), false) + .map(cookie -> { + HttpCookie httpCookie = new HttpCookie(cookie.getName(), cookie.getValue()); + httpCookie.setComment(cookie.getComment()); + httpCookie.setCommentURL(cookie.getCommentURL()); + httpCookie.setDiscard(cookie.isDiscard()); + httpCookie.setDomain(cookie.getDomain()); + httpCookie.setHttpOnly(cookie.isHttpOnly()); + httpCookie.setMaxAge(cookie.getMaxAge(TimeUnit.SECONDS)); + httpCookie.setPath(cookie.getPath()); + httpCookie.setSecure(cookie.isSecure()); + httpCookie.setVersion(cookie.getVersion()); + String portList = cookie.ports().stream() + .map(Number::toString) + .collect(Collectors.joining(",")); + httpCookie.setPortlist(portList); + return httpCookie.toString(); + }) + .collect(Collectors.joining(";")); } public static List<Cookie> fromCookieHeader(String headerVal) { - if (headerVal == null) return Collections.emptyList(); - - ServerCookieDecoder decoder = ServerCookieDecoder.STRICT; - Set<org.jboss.netty.handler.codec.http.cookie.Cookie> nettyCookies = decoder.decode(headerVal); - return nettyCookies.stream() - // NOTE: Only name and value is included in Cookie header as of RFC-6265 - .map(nettyCookie -> new Cookie(nettyCookie.name(), nettyCookie.value())) + CookieCutter cookieCutter = new CookieCutter(); + cookieCutter.addCookieField(headerVal); + return Arrays.stream(cookieCutter.getCookies()) + .map(servletCookie -> { + Cookie cookie = new Cookie(); + cookie.setName(servletCookie.getName()); + cookie.setValue(servletCookie.getValue()); + cookie.setComment(servletCookie.getComment()); + cookie.setPath(servletCookie.getPath()); + cookie.setDomain(servletCookie.getDomain()); + cookie.setMaxAge(servletCookie.getMaxAge(), TimeUnit.SECONDS); + cookie.setSecure(servletCookie.getSecure()); + cookie.setVersion(servletCookie.getVersion()); + cookie.setHttpOnly(servletCookie.isHttpOnly()); + return cookie; + }) .collect(Collectors.toList()); } @@ -247,34 +278,60 @@ public class Cookie { // TODO Rename to toSetCookieHeader for Vespa 7 public static List<String> toSetCookieHeaderAll(Iterable<? extends Cookie> cookies) { - ServerCookieEncoder encoder = ServerCookieEncoder.STRICT; - List<org.jboss.netty.handler.codec.http.cookie.Cookie> nettyCookies = - StreamSupport.stream(cookies.spliterator(), false) - .map(cookie -> { - org.jboss.netty.handler.codec.http.cookie.Cookie nettyCookie - = new DefaultCookie(cookie.getName(), cookie.getValue()); - nettyCookie.setPath(cookie.getPath()); - nettyCookie.setMaxAge(cookie.getMaxAge(TimeUnit.SECONDS)); - nettyCookie.setSecure(cookie.isSecure()); - nettyCookie.setHttpOnly(cookie.isHttpOnly()); - nettyCookie.setDomain(cookie.getDomain()); - return nettyCookie; - }) - .collect(Collectors.toList()); - return encoder.encode(nettyCookies); + // Ugly, bot Jetty does not provide a dedicated cookie parser (will be included in Jetty 10) + Response response = new Response(null, null); + for (Cookie cookie : cookies) { + response.addSetRFC6265Cookie( + cookie.getName(), + cookie.getValue(), + cookie.getDomain(), + cookie.getPath(), + cookie.getMaxAge(TimeUnit.SECONDS), + cookie.isSecure(), + cookie.isHttpOnly()); + } + return new ArrayList<>(response.getHeaders("Set-Cookie")); } // TODO Change return type to Cookie for Vespa 7 public static List<Cookie> fromSetCookieHeader(String headerVal) { - if (headerVal == null) return Collections.emptyList(); + return HttpCookie.parse(headerVal).stream() + .map(httpCookie -> { + Cookie cookie = new Cookie(); + cookie.setName(httpCookie.getName()); + cookie.setValue(httpCookie.getValue()); + cookie.setComment(httpCookie.getComment()); + cookie.setCommentUrl(httpCookie.getCommentURL()); + cookie.setDiscard(httpCookie.getDiscard()); + cookie.setDomain(httpCookie.getDomain()); + cookie.setHttpOnly(httpCookie.isHttpOnly()); + cookie.setMaxAge((int)httpCookie.getMaxAge(), TimeUnit.SECONDS); + cookie.setPath(httpCookie.getPath()); + cookie.setSecure(httpCookie.getSecure()); + cookie.setVersion(httpCookie.getVersion()); + cookie.ports().addAll(parsePortList(httpCookie.getPortlist())); + return cookie; + }) + .collect(Collectors.toList()); + } - ClientCookieDecoder encoder = ClientCookieDecoder.STRICT; - org.jboss.netty.handler.codec.http.cookie.Cookie nettyCookie = encoder.decode(headerVal); - return Collections.singletonList(new Cookie(nettyCookie.name(), nettyCookie.value()) - .setHttpOnly(nettyCookie.isHttpOnly()) - .setSecure(nettyCookie.isSecure()) - .setMaxAge(nettyCookie.maxAge(), TimeUnit.SECONDS) - .setPath(nettyCookie.path()) - .setDomain(nettyCookie.domain())); + + private static List<Integer> parsePortList(String rawPortList) { + if (rawPortList == null) return Collections.emptyList(); + + List<Integer> ports = new ArrayList<>(); + StringTokenizer tokenizer = new StringTokenizer(rawPortList, ","); + while (tokenizer.hasMoreTokens()) { + String rawPort = tokenizer.nextToken().trim(); + if (!rawPort.isEmpty()) { + try { + ports.add(Integer.parseInt(rawPort)); + } catch (NumberFormatException e) { + log.log(Level.FINE, "Unable to parse port: " + rawPort, e); + } + } + } + return ports; } + } diff --git a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/CookieTestCase.java b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/CookieTestCase.java index 0c98f294c82..ca12de72ec2 100644 --- a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/CookieTestCase.java +++ b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/CookieTestCase.java @@ -106,14 +106,14 @@ public class CookieTestCase { "foo.name=foo.value", Collections.singletonList(newCookie("foo"))); assertEncodeCookie( - "foo.name=foo.value; bar.name=bar.value", + "foo.name=foo.value;bar.name=bar.value", Arrays.asList(newCookie("foo"), newCookie("bar"))); } @Test public void requireThatSetCookieCanBeEncoded() { assertEncodeSetCookie( - Collections.singletonList("foo.name=foo.value; Path=path; Domain=domain; Secure; HTTPOnly"), + Collections.singletonList("foo.name=foo.value;Path=path;Domain=domain;Secure;HttpOnly"), Collections.singletonList(newSetCookie("foo"))); } @@ -131,6 +131,7 @@ public class CookieTestCase { } @Test + @SuppressWarnings("deprecation") public void requireThatSetCookieCanBeDecoded() { final Cookie foo = new Cookie(); foo.setName("foo.name"); @@ -140,6 +141,7 @@ public class CookieTestCase { foo.setMaxAge(0, TimeUnit.SECONDS); foo.setSecure(true); foo.setHttpOnly(true); + foo.setVersion(1); assertDecodeSetCookie(foo, "foo.name=foo.value;Max-Age=0;Path=path;Domain=domain;Secure;HTTPOnly;"); final Cookie bar = new Cookie(); @@ -148,6 +150,7 @@ public class CookieTestCase { bar.setPath("path"); bar.setDomain("domain"); bar.setMaxAge(0, TimeUnit.SECONDS); + bar.setVersion(1); assertDecodeSetCookie(bar, "bar.name=bar.value;Max-Age=0;Path=path;Domain=domain;"); } diff --git a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerTest.java b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerTest.java index 221a1adc1fe..7ed13decbf6 100644 --- a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerTest.java +++ b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerTest.java @@ -356,7 +356,7 @@ public class HttpServerTest { driver.client().get("/status.html") .expectStatusCode(is(OK)) .expectHeader("Set-Cookie", - is("foo=bar; Path=/foopath; Domain=.localhost; Secure; HTTPOnly")); + is("foo=bar;Path=/foopath;Domain=.localhost;Secure;HttpOnly")); assertThat(driver.close(), is(true)); } diff --git a/messagebus/src/main/config/messagebus.def b/messagebus/src/main/config/messagebus.def index d0ddbcb8e62..b750530ec5f 100644 --- a/messagebus/src/main/config/messagebus.def +++ b/messagebus/src/main/config/messagebus.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=2 namespace=messagebus # Name of the protocol that uses this routing table. All diff --git a/metrics/src/vespa/metrics/metricsmanager.def b/metrics/src/vespa/metrics/metricsmanager.def index 11e00eda286..80f03a8e9e8 100644 --- a/metrics/src/vespa/metrics/metricsmanager.def +++ b/metrics/src/vespa/metrics/metricsmanager.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=4 namespace=metrics # If any snapshot periods is set, these override all the default ones. diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java index 2e81ef19f5e..9d2198cedcc 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java @@ -180,7 +180,9 @@ public class NodeAgentImpl implements NodeAgent { @Override public void start(int intervalMillis) { - addDebugMessage("Starting with interval " + intervalMillis + "ms"); + String message = "Starting with interval " + intervalMillis + " ms"; + logger.info(message); + addDebugMessage(message); delaysBetweenEachConvergeMillis = intervalMillis; if (loopThread != null) { throw new RuntimeException("Can not restart a node agent."); @@ -214,6 +216,8 @@ public class NodeAgentImpl implements NodeAgent { } catch (InterruptedException e) { logger.error("Interrupted; Could not stop filebeatrestarter thread"); } + + logger.info("Stopped"); } private void runLocalResumeScriptIfNeeded() { diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImplTest.java index 4650e9bf317..bfabf0a4e4e 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/noderepository/NodeRepositoryImplTest.java @@ -149,6 +149,7 @@ public class NodeRepositoryImplTest { NodeRepository nodeRepositoryApi = new NodeRepositoryImpl(requestExecutor, port, "dockerhost1.yahoo.com"); waitForJdiscContainerToServe(); + nodeRepositoryApi.markAsDirty("host5.yahoo.com"); nodeRepositoryApi.markNodeAvailableForNewAllocation("host5.yahoo.com"); try { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java index 1885b54e9c0..7f595b4a541 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java @@ -25,6 +25,7 @@ import com.yahoo.vespa.hosted.provision.node.filter.StateFilter; import com.yahoo.vespa.hosted.provision.persistence.CuratorDatabaseClient; import com.yahoo.vespa.hosted.provision.persistence.DnsNameResolver; import com.yahoo.vespa.hosted.provision.persistence.NameResolver; +import com.yahoo.vespa.hosted.provision.restapi.v2.NotFoundException; import java.time.Clock; import java.time.Duration; @@ -466,27 +467,88 @@ public class NodeRepository extends AbstractComponent { } } - /** - * Removes a node. A node must be in a legal state before it can be removed. + /* + * This method is used to enable a smooth rollout of dynamic docker flavor allocations. Once we have switch + * everything this can be simplified to only deleting the node. * - * @return true if the node was removed, false if it was not found in one of the legal states + * Should only be called by node-admin for docker containers */ - public boolean remove(String hostname) { + public List<Node> markNodeAvailableForNewAllocation(String hostname) { + Node node = getNode(hostname).orElseThrow(() -> new NotFoundException("No node with hostname \"" + hostname + '"')); + if (node.flavor().getType() != Flavor.Type.DOCKER_CONTAINER) { + throw new IllegalArgumentException( + "Cannot make " + hostname + " available for new allocation, must be a docker container node"); + } else if (node.state() != Node.State.dirty) { + throw new IllegalArgumentException( + "Cannot make " + hostname + " available for new allocation, must be in state dirty, but was in " + node.state()); + } - Node.State[] legalStates = {Node.State.provisioned, Node.State.failed, Node.State.parked}; - Node.State[] legalDynamicStates = {Node.State.provisioned, Node.State.failed, Node.State.parked, Node.State.dirty}; + if (dynamicAllocationEnabled()) { + return removeRecursively(node, true); + } else { + return setReady(Collections.singletonList(node)); + } + } - Optional<Node> nodeToRemove = getNode(hostname, dynamicAllocationEnabled() ? legalDynamicStates : legalStates); - if ( ! nodeToRemove.isPresent()) return false; + /** + * Removes all the nodes that are children of hostname before finally removing the hostname itself. + * + * @return List of all the nodes that have been removed + */ + public List<Node> removeRecursively(String hostname) { + Node node = getNode(hostname).orElseThrow(() -> new NotFoundException("No node with hostname \"" + hostname + '"')); + return removeRecursively(node, false); + } - // Only docker nodes are allowed to be deleted in state dirty. - if ( nodeToRemove.get().state().equals(Node.State.dirty)) { - if (!(nodeToRemove.get().flavor().getType().equals(Flavor.Type.DOCKER_CONTAINER))) return false; + private List<Node> removeRecursively(Node node, boolean force) { + try (Mutex lock = lockUnallocated()) { + List<Node> removed = node.type() != NodeType.host ? + new ArrayList<>() : + getChildNodes(node.hostname()).stream() + .filter(child -> force || verifyRemovalIsAllowed(child, true)) + .collect(Collectors.toList()); + + if (force || verifyRemovalIsAllowed(node, false)) removed.add(node); + db.removeNodes(removed); + + return removed; + } catch (RuntimeException e) { + throw new IllegalArgumentException("Failed to delete " + node.hostname(), e); } + } - try (Mutex lock = lock(nodeToRemove.get())) { - return db.removeNode(nodeToRemove.get().state(), hostname); + /** + * Allowed to a node delete if: + * Non-docker-container node: iff in state provisioned|failed|parked + * Docker-container-node: + * If only removing the container node: node in state ready + * If also removing the parent node: child is in state provisioned|failed|parked|ready + */ + private boolean verifyRemovalIsAllowed(Node nodeToRemove, boolean deletingAsChild) { + // TODO: Enable once controller no longer deletes child nodes manually + /*if (nodeToRemove.flavor().getType() == Flavor.Type.DOCKER_CONTAINER && !deletingAsChild) { + if (nodeToRemove.state() != Node.State.ready) { + throw new IllegalArgumentException( + String.format("Docker container node %s can only be removed when in state ready", nodeToRemove.hostname())); + } + + } else */ if (nodeToRemove.flavor().getType() == Flavor.Type.DOCKER_CONTAINER) { + List<Node.State> legalStates = Arrays.asList(Node.State.provisioned, Node.State.failed, Node.State.parked, Node.State.ready); + + if (! legalStates.contains(nodeToRemove.state())) { + throw new IllegalArgumentException(String.format("Child node %s can only be removed from following states: %s", + nodeToRemove.hostname(), legalStates.stream().map(Node.State::name).collect(Collectors.joining(", ")))); + } + } else { + List<Node.State> legalStates = Arrays.asList(Node.State.provisioned, Node.State.failed, Node.State.parked); + + if (! legalStates.contains(nodeToRemove.state())) { + throw new IllegalArgumentException(String.format("Node %s can only be removed from following states: %s", + nodeToRemove.hostname(), legalStates.stream().map(Node.State::name).collect(Collectors.joining(", ")))); + } } + + return true; } /** diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DirtyExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DirtyExpirer.java index d057cf492ba..520ceaf323b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DirtyExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DirtyExpirer.java @@ -33,7 +33,7 @@ public class DirtyExpirer extends Expirer { @Override protected void expire(List<Node> expired) { - for (Node expiredNode : expired.stream().collect(Collectors.toList())) + for (Node expiredNode : expired) nodeRepository.fail(expiredNode.hostname(), Agent.system, "Node is stuck in dirty"); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java index c7ef7f35ce0..2057fd7e36f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java @@ -44,6 +44,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { private final RetiredEarlyExpirer retiredEarlyExpirer; private final FailedExpirer failedExpirer; private final DirtyExpirer dirtyExpirer; + private final ProvisionedExpirer provisionedExpirer; private final NodeRebooter nodeRebooter; private final NodeRetirer nodeRetirer; private final MetricsReporter metricsReporter; @@ -73,6 +74,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { inactiveExpirer = new InactiveExpirer(nodeRepository, clock, durationFromEnv("inactive_expiry").orElse(defaults.inactiveExpiry), jobControl); failedExpirer = new FailedExpirer(nodeRepository, zone, clock, durationFromEnv("failed_expiry").orElse(defaults.failedExpiry), jobControl); dirtyExpirer = new DirtyExpirer(nodeRepository, clock, durationFromEnv("dirty_expiry").orElse(defaults.dirtyExpiry), jobControl); + provisionedExpirer = new ProvisionedExpirer(nodeRepository, clock, durationFromEnv("provisioned_expiry").orElse(defaults.provisionedExpiry), jobControl); nodeRebooter = new NodeRebooter(nodeRepository, clock, durationFromEnv("reboot_interval").orElse(defaults.rebootInterval), jobControl); metricsReporter = new MetricsReporter(nodeRepository, metric, durationFromEnv("metrics_interval").orElse(defaults.metricsInterval), jobControl); @@ -96,6 +98,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { dirtyExpirer.deconstruct(); nodeRebooter.deconstruct(); nodeRetirer.deconstruct(); + provisionedExpirer.deconstruct(); metricsReporter.deconstruct(); } @@ -133,6 +136,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { private final Duration retiredExpiry; private final Duration failedExpiry; private final Duration dirtyExpiry; + private final Duration provisionedExpiry; private final Duration rebootInterval; private final Duration nodeRetirerInterval; private final Duration metricsInterval; @@ -154,6 +158,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { retiredEarlyInterval = Duration.ofMinutes(29); failedExpiry = Duration.ofDays(4); // enough time to recover data even if it happens friday night dirtyExpiry = Duration.ofHours(2); // enough time to clean the node + provisionedExpiry = Duration.ofHours(4); rebootInterval = Duration.ofDays(30); nodeRetirerInterval = Duration.ofMinutes(30); metricsInterval = Duration.ofMinutes(1); @@ -171,6 +176,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { retiredEarlyInterval = Duration.ofMinutes(5); failedExpiry = Duration.ofMinutes(10); dirtyExpiry = Duration.ofMinutes(30); + provisionedExpiry = Duration.ofHours(4); rebootInterval = Duration.ofDays(30); nodeRetirerInterval = Duration.ofMinutes(30); metricsInterval = Duration.ofMinutes(1); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ProvisionedExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ProvisionedExpirer.java new file mode 100644 index 00000000000..b41eedd4694 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ProvisionedExpirer.java @@ -0,0 +1,33 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.maintenance; + +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.node.Agent; +import com.yahoo.vespa.hosted.provision.node.History; + +import java.time.Clock; +import java.time.Duration; +import java.util.List; + +/** + * This moves nodes from provisioned to parked if they have been in provisioned too long. + * + * @author freva + */ +public class ProvisionedExpirer extends Expirer { + + private final NodeRepository nodeRepository; + + public ProvisionedExpirer(NodeRepository nodeRepository, Clock clock, Duration dirtyTimeout, JobControl jobControl) { + super(Node.State.provisioned, History.Event.Type.provisioned, nodeRepository, clock, dirtyTimeout, jobControl); + this.nodeRepository = nodeRepository; + } + + @Override + protected void expire(List<Node> expired) { + for (Node expiredNode : expired) + nodeRepository.parkRecursively(expiredNode.hostname(), Agent.system, "Node is stuck in provisioned"); + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/History.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/History.java index 8f7317b28eb..3228e6ca17e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/History.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/History.java @@ -65,14 +65,15 @@ public class History { public History recordStateTransition(Node.State from, Node.State to, Agent agent, Instant at) { if (from == to) return this; switch (to) { - case ready: return this.withoutApplicationEvents().with(new Event(Event.Type.readied, agent, at)); - case active: return this.with(new Event(Event.Type.activated, agent, at)); - case inactive: return this.with(new Event(Event.Type.deactivated, agent, at)); - case reserved: return this.with(new Event(Event.Type.reserved, agent, at)); - case failed: return this.with(new Event(Event.Type.failed, agent, at)); - case dirty: return this.with(new Event(Event.Type.deallocated, agent, at)); - case parked: return this.with(new Event(Event.Type.parked, agent, at)); - default: return this; + case provisioned: return this.with(new Event(Event.Type.provisioned, agent, at)); + case ready: return this.withoutApplicationEvents().with(new Event(Event.Type.readied, agent, at)); + case active: return this.with(new Event(Event.Type.activated, agent, at)); + case inactive: return this.with(new Event(Event.Type.deactivated, agent, at)); + case reserved: return this.with(new Event(Event.Type.reserved, agent, at)); + case failed: return this.with(new Event(Event.Type.failed, agent, at)); + case dirty: return this.with(new Event(Event.Type.deallocated, agent, at)); + case parked: return this.with(new Event(Event.Type.parked, agent, at)); + default: return this; } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java index cd63599fed6..d38c9179986 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java @@ -84,6 +84,8 @@ public class CuratorDatabaseClient { for (Node node : nodes) { if (node.state() != expectedState) throw new IllegalArgumentException(node + " is not in the " + node.state() + " state"); + + node = node.with(node.history().recordStateTransition(null, expectedState, Agent.system, clock.instant())); curatorTransaction.add(CuratorOperations.create(toPath(node).getAbsolute(), nodeSerializer.toJson(node))); } transaction.commit(); @@ -104,20 +106,21 @@ public class CuratorDatabaseClient { } /** - * Removes a node. + * Removes multiple nodes in a single transaction. * - * @param state the current state of the node - * @param hostName the host name of the node to remove - * @return true if the node was removed, false if it was not found + * @param nodes list of the nodes to remove */ - public boolean removeNode(Node.State state, String hostName) { - Path path = toPath(state, hostName); + public void removeNodes(List<Node> nodes) { NestedTransaction transaction = new NestedTransaction(); - CuratorTransaction curatorTransaction = curatorDatabase.newCuratorTransactionIn(transaction); - curatorTransaction.add(CuratorOperations.delete(path.getAbsolute())); + + for (Node node : nodes) { + Path path = toPath(node.state(), node.hostname()); + CuratorTransaction curatorTransaction = curatorDatabase.newCuratorTransactionIn(transaction); + curatorTransaction.add(CuratorOperations.delete(path.getAbsolute())); + } + transaction.commit(); - log.log(LogLevel.INFO, "Removed: " + state + " node " + hostName); - return true; + nodes.forEach(node -> log.log(LogLevel.INFO, "Removed node " + node.hostname() + " in state " + node.state())); } /** diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java index 914658302b6..e2ff17ba782 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java @@ -24,7 +24,6 @@ public class CapacityPolicies { this.flavors = flavors; } - /** provides capacity defaults for various environments */ public int decideSize(Capacity requestedCapacity) { int requestedNodes = requestedCapacity.nodeCount(); if (requestedCapacity.isRequired()) return requestedNodes; @@ -39,10 +38,10 @@ public class CapacityPolicies { } public Flavor decideFlavor(Capacity requestedCapacity, ClusterSpec cluster, Optional<String> defaultFlavorOverride) { - // for now, always use requested docker flavor when requested + // for now, always use the requested flavor if a docker flavor is requested Optional<String> requestedFlavor = requestedCapacity.flavor(); if (requestedFlavor.isPresent() && - flavors.getFlavorOrThrow(requestedFlavor.get()).getType() == Flavor.Type.DOCKER_CONTAINER) + flavors.getFlavorOrThrow(requestedFlavor.get()).getType() == Flavor.Type.DOCKER_CONTAINER) return flavors.getFlavorOrThrow(requestedFlavor.get()); String defaultFlavorName = defaultFlavorOverride.isPresent() ? diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacity.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacity.java index 77d91c7bea7..78ea258107b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacity.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacity.java @@ -65,12 +65,12 @@ public class DockerHostCapacity { * Checks the node capacity and free ip addresses to see * if we could allocate a flavor on the docker host. */ - boolean hasCapacity(Node dockerHost, Flavor flavor) { - return freeCapacityOf(dockerHost, false).hasCapacityFor(flavor) && freeIPs(dockerHost) > 0; + boolean hasCapacity(Node dockerHost, ResourceCapacity requestedCapacity) { + return freeCapacityOf(dockerHost, false).hasCapacityFor(requestedCapacity) && freeIPs(dockerHost) > 0; } - boolean hasCapacityWhenRetiredAndInactiveNodesAreGone(Node dockerHost, Flavor flavor) { - return freeCapacityOf(dockerHost, true).hasCapacityFor(flavor) && freeIPs(dockerHost) > 0; + boolean hasCapacityWhenRetiredAndInactiveNodesAreGone(Node dockerHost, ResourceCapacity requestedCapacity) { + return freeCapacityOf(dockerHost, true).hasCapacityFor(requestedCapacity) && freeIPs(dockerHost) > 0; } /** @@ -105,7 +105,7 @@ public class DockerHostCapacity { public long getNofHostsAvailableFor(Flavor flavor) { return allNodes.asList().stream() .filter(n -> n.type().equals(NodeType.host)) - .filter(n -> hasCapacity(n, flavor)) + .filter(n -> hasCapacity(n, ResourceCapacity.of(flavor))) .count(); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java index 960d0b9d729..3df79174b5c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java @@ -30,7 +30,7 @@ import java.util.stream.Collectors; * * @author smorgrav */ -public class NodePrioritizer { +class NodePrioritizer { private final Map<Node, PrioritizableNode> nodes = new HashMap<>(); private final List<Node> allNodes; @@ -39,10 +39,10 @@ public class NodePrioritizer { private final ApplicationId appId; private final ClusterSpec clusterSpec; + private final boolean isDocker; private final boolean isAllocatingForReplacement; private final Set<Node> spareHosts; - private final Map<Node, Boolean> headroomHosts; - private final boolean isDocker; + private final Map<Node, ResourceCapacity> headroomHosts; NodePrioritizer(List<Node> allNodes, ApplicationId appId, ClusterSpec clusterSpec, NodeSpec nodeSpec, NodeFlavors nodeFlavors, int spares) { this.allNodes = Collections.unmodifiableList(allNodes); @@ -50,8 +50,8 @@ public class NodePrioritizer { this.clusterSpec = clusterSpec; this.appId = appId; - spareHosts = findSpareHosts(allNodes, spares); - headroomHosts = findHeadroomHosts(allNodes, spareHosts, nodeFlavors); + this.spareHosts = findSpareHosts(allNodes, spares); + this.headroomHosts = findHeadroomHosts(allNodes, spareHosts, nodeFlavors); this.capacity = new DockerHostCapacity(allNodes); @@ -68,14 +68,14 @@ public class NodePrioritizer { .filter(node -> node.allocation().get().membership().cluster().id().equals(clusterSpec.id())) .count(); - isAllocatingForReplacement = isReplacement(nofNodesInCluster, nofFailedNodes); - isDocker = isDocker(); + this.isAllocatingForReplacement = isReplacement(nofNodesInCluster, nofFailedNodes); + this.isDocker = isDocker(); } /** * From ipAddress - get hostname * - * @return hostname or null if not able to do the loopup + * @return hostname or null if not able to do the lookup */ private static String lookupHostname(String ipAddress) { try { @@ -104,14 +104,14 @@ public class NodePrioritizer { } /** - * Headroom are the nodes with the least but sufficient space for the requested headroom. + * Headroom hosts are the host with the least but sufficient capacity for the requested headroom. * - * If not enough headroom - the headroom violating hosts are the once that are closest to fulfull + * If not enough headroom - the headroom violating hosts are the once that are closest to fulfill * a headroom request. */ - private static Map<Node, Boolean> findHeadroomHosts(List<Node> nodes, Set<Node> spareNodes, NodeFlavors flavors) { + private static Map<Node, ResourceCapacity> findHeadroomHosts(List<Node> nodes, Set<Node> spareNodes, NodeFlavors flavors) { DockerHostCapacity capacity = new DockerHostCapacity(nodes); - Map<Node, Boolean> headroomNodesToViolation = new HashMap<>(); + Map<Node, ResourceCapacity> headroomHosts = new HashMap<>(); List<Node> hostsSortedOnLeastCapacity = nodes.stream() .filter(n -> !spareNodes.contains(n)) @@ -121,20 +121,25 @@ public class NodePrioritizer { .sorted((a, b) -> capacity.compareWithoutInactive(b, a)) .collect(Collectors.toList()); + // For all flavors with ideal headroom - find which hosts this headroom should be allocated to for (Flavor flavor : flavors.getFlavors().stream().filter(f -> f.getIdealHeadroom() > 0).collect(Collectors.toList())) { Set<Node> tempHeadroom = new HashSet<>(); Set<Node> notEnoughCapacity = new HashSet<>(); + + ResourceCapacity headroomCapacity = ResourceCapacity.of(flavor); + + // Select hosts that has available capacity for both headroom and for new allocations for (Node host : hostsSortedOnLeastCapacity) { - if (headroomNodesToViolation.containsKey(host)) continue; - if (capacity.hasCapacityWhenRetiredAndInactiveNodesAreGone(host, flavor)) { - headroomNodesToViolation.put(host, false); + if (headroomHosts.containsKey(host)) continue; + if (capacity.hasCapacityWhenRetiredAndInactiveNodesAreGone(host, headroomCapacity)) { + headroomHosts.put(host, headroomCapacity); tempHeadroom.add(host); } else { notEnoughCapacity.add(host); } if (tempHeadroom.size() == flavor.getIdealHeadroom()) { - continue; + break; } } @@ -145,14 +150,13 @@ public class NodePrioritizer { .limit(flavor.getIdealHeadroom() - tempHeadroom.size()) .collect(Collectors.toList()); - for (Node nodeViolatingHeadrom : violations) { - headroomNodesToViolation.put(nodeViolatingHeadrom, true); + for (Node hostViolatingHeadrom : violations) { + headroomHosts.put(hostViolatingHeadrom, headroomCapacity); } - } } - return headroomNodesToViolation; + return headroomHosts; } /** @@ -197,14 +201,14 @@ public class NodePrioritizer { } } - if (!conflictingCluster && capacity.hasCapacity(node, getFlavor())) { + if (!conflictingCluster && capacity.hasCapacity(node, ResourceCapacity.of(getFlavor(requestedNodes)))) { Set<String> ipAddresses = DockerHostCapacity.findFreeIps(node, allNodes); if (ipAddresses.isEmpty()) continue; String ipAddress = ipAddresses.stream().findFirst().get(); String hostname = lookupHostname(ipAddress); if (hostname == null) continue; Node newNode = Node.createDockerNode("fake-" + hostname, Collections.singleton(ipAddress), - Collections.emptySet(), hostname, Optional.of(node.hostname()), getFlavor(), NodeType.tenant); + Collections.emptySet(), hostname, Optional.of(node.hostname()), getFlavor(requestedNodes), NodeType.tenant); PrioritizableNode nodePri = toNodePriority(newNode, false, true); if (!nodePri.violatesSpares || isAllocatingForReplacement) { nodes.put(newNode, nodePri); @@ -249,7 +253,7 @@ public class NodePrioritizer { pri.node = node; pri.isSurplusNode = isSurplusNode; pri.isNewNode = isNewNode; - pri.preferredOnFlavor = requestedNodes.specifiesNonStockFlavor() && node.flavor().equals(getFlavor()); + pri.preferredOnFlavor = requestedNodes.specifiesNonStockFlavor() && node.flavor().equals(getFlavor(requestedNodes)); pri.parent = findParentNode(node); if (pri.parent.isPresent()) { @@ -260,14 +264,29 @@ public class NodePrioritizer { pri.violatesSpares = true; } - if (headroomHosts.containsKey(parent)) { - pri.violatesHeadroom = headroomHosts.get(parent); + if (headroomHosts.containsKey(parent) && isPreferredNodeToBeReloacted(allNodes, node, parent)) { + ResourceCapacity neededCapacity = headroomHosts.get(parent); + + // If the node is new then we need to check the headroom requirement after it has been added + if (isNewNode) { + neededCapacity = ResourceCapacity.composite(neededCapacity, new ResourceCapacity(node)); + } + pri.violatesHeadroom = !capacity.hasCapacity(parent, neededCapacity); } } return pri; } + static boolean isPreferredNodeToBeReloacted(List<Node> nodes, Node node, Node parent) { + NodeList list = new NodeList(nodes); + return list.childNodes(parent).asList().stream() + .sorted(NodePrioritizer::compareForRelocation) + .findFirst() + .filter(n -> n.equals(node)) + .isPresent(); + } + private boolean isReplacement(long nofNodesInCluster, long nodeFailedNodes) { if (nodeFailedNodes == 0) return false; @@ -280,7 +299,7 @@ public class NodePrioritizer { return (wantedCount > nofNodesInCluster - nodeFailedNodes); } - private Flavor getFlavor() { + private static Flavor getFlavor(NodeSpec requestedNodes) { if (requestedNodes instanceof NodeSpec.CountNodeSpec) { NodeSpec.CountNodeSpec countSpec = (NodeSpec.CountNodeSpec) requestedNodes; return countSpec.getFlavor(); @@ -289,7 +308,7 @@ public class NodePrioritizer { } private boolean isDocker() { - Flavor flavor = getFlavor(); + Flavor flavor = getFlavor(requestedNodes); return (flavor != null) && flavor.getType().equals(Flavor.Type.DOCKER_CONTAINER); } @@ -299,4 +318,27 @@ public class NodePrioritizer { .filter(n -> n.hostname().equals(node.parentHostname().orElse(" NOT A NODE"))) .findAny(); } + + private static int compareForRelocation(Node a, Node b) { + // Choose smallest node + int capacity = ResourceCapacity.of(a).compare(ResourceCapacity.of(b)); + if (capacity != 0) return capacity; + + // Choose unallocated over allocated (this case is when we have ready docker nodes) + if (!a.allocation().isPresent() && b.allocation().isPresent()) return -1; + if (a.allocation().isPresent() && !b.allocation().isPresent()) return 1; + + // Choose container over content nodes + if (a.allocation().isPresent() && b.allocation().isPresent()) { + if (a.allocation().get().membership().cluster().type().equals(ClusterSpec.Type.container) && + !b.allocation().get().membership().cluster().type().equals(ClusterSpec.Type.container)) + return -1; + if (!a.allocation().get().membership().cluster().type().equals(ClusterSpec.Type.container) && + b.allocation().get().membership().cluster().type().equals(ClusterSpec.Type.container)) + return 1; + } + + // To get a stable algorithm - choose lexicographical from hostname + return a.hostname().compareTo(b.hostname()); + } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNode.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNode.java index 06acd646ea7..807fbfae1c9 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNode.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNode.java @@ -23,7 +23,7 @@ class PrioritizableNode implements Comparable<PrioritizableNode> { /** True if the node is allocated to a host that should be dedicated as a spare */ boolean violatesSpares; - /** True if the node is allocated on slots that should be dedicated to headroom */ + /** True if the node is (or would be) allocated on slots that should be dedicated to headroom */ boolean violatesHeadroom; /** True if this is a node that has been retired earlier in the allocation process */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ResourceCapacity.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ResourceCapacity.java index fdec29d5b97..8373cf9e17f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ResourceCapacity.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ResourceCapacity.java @@ -28,6 +28,18 @@ public class ResourceCapacity { disk = node.flavor().getMinDiskAvailableGb(); } + static ResourceCapacity of(Flavor flavor) { + ResourceCapacity capacity = new ResourceCapacity(); + capacity.memory = flavor.getMinMainMemoryAvailableGb(); + capacity.cpu = flavor.getMinCpuCores(); + capacity.disk = flavor.getMinDiskAvailableGb(); + return capacity; + } + + static ResourceCapacity of(Node node) { + return new ResourceCapacity(node); + } + public double getMemory() { return memory; } @@ -40,6 +52,15 @@ public class ResourceCapacity { return disk; } + static ResourceCapacity composite(ResourceCapacity a, ResourceCapacity b) { + ResourceCapacity composite = new ResourceCapacity(); + composite.memory = a.memory + b.memory; + composite.cpu -= a.cpu + b.cpu; + composite.disk -= a.disk + b.disk; + + return composite; + } + void subtract(Node node) { memory -= node.flavor().getMinMainMemoryAvailableGb(); cpu -= node.flavor().getMinCpuCores(); @@ -54,14 +75,18 @@ public class ResourceCapacity { return result; } + boolean hasCapacityFor(ResourceCapacity capacity) { + return memory >= capacity.memory && + cpu >= capacity.cpu && + disk >= capacity.disk; + } + boolean hasCapacityFor(Flavor flavor) { - return memory >= flavor.getMinMainMemoryAvailableGb() && - cpu >= flavor.getMinCpuCores() && - disk >= flavor.getMinDiskAvailableGb(); + return hasCapacityFor(ResourceCapacity.of(flavor)); } int freeCapacityInFlavorEquivalence(Flavor flavor) { - if (!hasCapacityFor(flavor)) return 0; + if (!hasCapacityFor(ResourceCapacity.of(flavor))) return 0; double memoryFactor = Math.floor(memory/flavor.getMinMainMemoryAvailableGb()); double cpuFactor = Math.floor(cpu/flavor.getMinCpuCores()); @@ -85,11 +110,4 @@ public class ResourceCapacity { if (cpu < that.cpu) return -1; return 0; } - - Flavor asFlavor() { - FlavorConfigBuilder b = new FlavorConfigBuilder(); - b.addFlavor("spareflavor", cpu, memory, disk, Flavor.Type.DOCKER_CONTAINER).idealHeadroom(1); - return new Flavor(b.build().flavor(0)); - } - } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java index db75894673e..b16ce5f818e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java @@ -125,24 +125,10 @@ public class NodesApiHandler extends LoggingRequestHandler { return new MessageResponse("Moved " + lastElement(path) + " to active"); } else if (path.startsWith("/nodes/v2/state/availablefornewallocations/")) { - /** - * This is a temporary "state" or rest call that we use to enable a smooth rollout of - * dynamic docker flavor allocations. Once we have switch everything we remove this - * and change the code in the nodeadmin to delete directly (remember to allow deletion of dirty nodes then). - * - * Should only be called by node-admin for docker containers (the docker constraint is - * enforced in the remove method) - */ String hostname = lastElement(path); - if (nodeRepository.dynamicAllocationEnabled()) { - if (nodeRepository.remove(hostname)) - return new MessageResponse("Removed " + hostname); - else - throw new NotFoundException("No node in the provisioned, parked, dirty or failed state with hostname " + hostname); - } else { - nodeRepository.setReady(hostname); - return new MessageResponse("Moved " + hostname + " to ready"); - } + List<Node> available = nodeRepository.markNodeAvailableForNewAllocation(hostname); + return new MessageResponse("Marked following nodes as available for new allocation: " + + available.stream().map(Node::hostname).collect(Collectors.joining(", "))); } throw new NotFoundException("Cannot put to path '" + path + "'"); @@ -182,10 +168,8 @@ public class NodesApiHandler extends LoggingRequestHandler { String path = request.getUri().getPath(); if (path.startsWith("/nodes/v2/node/")) { String hostname = lastElement(path); - if (nodeRepository.remove(hostname)) - return new MessageResponse("Removed " + hostname); - else - throw new NotFoundException("No node in the provisioned, parked or failed state with hostname " + hostname); + List<Node> removedNodes = nodeRepository.removeRecursively(hostname); + return new MessageResponse("Removed " + removedNodes.stream().map(Node::hostname).collect(Collectors.joining(", "))); } else if (path.startsWith("/nodes/v2/maintenance/inactive/")) { return setActive(lastElement(path), true); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTest.java index 12c76638604..8eec56a1c00 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTest.java @@ -8,11 +8,13 @@ import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.TenantName; import com.yahoo.path.Path; import com.yahoo.vespa.hosted.provision.node.Agent; +import org.junit.Ignore; import org.junit.Test; import java.nio.charset.StandardCharsets; import java.util.Optional; +import static junit.framework.TestCase.fail; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -27,18 +29,18 @@ public class NodeRepositoryTest { @Test public void nodeRepositoryTest() { NodeRepositoryTester tester = new NodeRepositoryTester(); - assertEquals(0, tester.getNodes(NodeType.tenant).size()); + assertEquals(0, tester.nodeRepository().getNodes().size()); tester.addNode("id1", "host1", "default", NodeType.tenant); tester.addNode("id2", "host2", "default", NodeType.tenant); tester.addNode("id3", "host3", "default", NodeType.tenant); - assertEquals(3, tester.getNodes(NodeType.tenant).size()); + assertEquals(3, tester.nodeRepository().getNodes().size()); tester.nodeRepository().park("host2", Agent.system, "Parking to unit test"); - assertTrue(tester.nodeRepository().remove("host2")); + tester.nodeRepository().removeRecursively("host2"); - assertEquals(2, tester.getNodes(NodeType.tenant).size()); + assertEquals(2, tester.nodeRepository().getNodes().size()); } @Test @@ -68,20 +70,55 @@ public class NodeRepositoryTest { assertTrue(tester.nodeRepository().dynamicAllocationEnabled()); } - @Test - public void only_allow_to_delete_dirty_nodes_when_dynamic_allocation_feature_enabled() { + @Test @Ignore // TODO: Enable once controller no longer deletes child nodes manually + public void only_allow_docker_containers_remove_in_ready() { NodeRepositoryTester tester = new NodeRepositoryTester(); + tester.addNode("id1", "host1", "docker", NodeType.tenant); + try { - tester.addNode("id1", "host1", "default", NodeType.host); - tester.addNode("id2", "host2", "docker", NodeType.tenant); - tester.nodeRepository().setDirty("host2"); + tester.nodeRepository().removeRecursively("host1"); // host1 is in state provisioned + fail("Should not be able to delete docker container node by itself in state provisioned"); + } catch (IllegalArgumentException ignored) { + // Expected + } - assertFalse(tester.nodeRepository().remove("host2")); + tester.nodeRepository().setDirty("host1"); + tester.nodeRepository().setReady("host1"); + tester.nodeRepository().removeRecursively("host1"); + } + + @Test + public void delete_host_only_after_all_the_children_have_been_deleted() { + NodeRepositoryTester tester = new NodeRepositoryTester(); - tester.curator().set(Path.fromString("/provision/v1/dynamicDockerAllocation"), new byte[0]); - assertTrue(tester.nodeRepository().remove("host2")); - } finally { - tester.curator().delete(Path.fromString("/provision/v1/dynamicDockerAllocation")); + tester.addNode("id1", "host1", "default", NodeType.host); + tester.addNode("id2", "host2", "default", NodeType.host); + tester.addNode("node10", "node10", "host1", "docker", NodeType.tenant); + tester.addNode("node11", "node11", "host1", "docker", NodeType.tenant); + tester.addNode("node12", "node12", "host1", "docker", NodeType.tenant); + tester.addNode("node20", "node20", "host2", "docker", NodeType.tenant); + assertEquals(6, tester.nodeRepository().getNodes().size()); + + tester.nodeRepository().setDirty("node11"); + + try { + tester.nodeRepository().removeRecursively("host1"); + fail("Should not be able to delete host node, one of the children is in state dirty"); + } catch (IllegalArgumentException ignored) { + // Expected } + assertEquals(6, tester.nodeRepository().getNodes().size()); + + // Should be OK to delete host2 as both host2 and its only child, node20, are in state provisioned + tester.nodeRepository().removeRecursively("host2"); + assertEquals(4, tester.nodeRepository().getNodes().size()); + + // Now node10 and node12 are in provisioned, set node11 to ready, and it should be OK to delete host1 + tester.nodeRepository().setReady("node11"); + tester.nodeRepository().removeRecursively("node11"); // Remove one of the children first instead + assertEquals(3, tester.nodeRepository().getNodes().size()); + + tester.nodeRepository().removeRecursively("host1"); + assertEquals(0, tester.nodeRepository().getNodes().size()); } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java index 784fc1a274a..3d01bde4291 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java @@ -51,6 +51,12 @@ public class NodeRepositoryTester { return nodeRepository.addNodes(Collections.singletonList(node)).get(0); } + public Node addNode(String id, String hostname, String parentHostname, String flavor, NodeType type) { + Node node = nodeRepository.createNode(id, hostname, Optional.of(parentHostname), + nodeFlavors.getFlavorOrThrow(flavor), type); + return nodeRepository.addNodes(Collections.singletonList(node)).get(0); + } + private FlavorsConfig createConfig() { FlavorConfigBuilder b = new FlavorConfigBuilder(); b.addFlavor("default", 2., 4., 100, Flavor.Type.BARE_METAL).cost(3); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java index e27ca004862..704ded54479 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java @@ -20,10 +20,12 @@ import com.yahoo.vespa.hosted.provision.testutils.MockDeployer; import org.junit.Test; import java.time.Duration; +import java.util.Arrays; import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Optional; +import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -62,8 +64,9 @@ public class InactiveAndFailedExpirerTest { // One node is set back to ready Node ready = tester.nodeRepository().setReady(Collections.singletonList(dirty.get(0))).get(0); - assertEquals("Allocated history is removed on readying", 1, ready.history().events().size()); - assertEquals(History.Event.Type.readied, ready.history().events().iterator().next().type()); + assertEquals("Allocated history is removed on readying", + Arrays.asList(History.Event.Type.provisioned, History.Event.Type.readied), + ready.history().events().stream().map(History.Event::type).collect(Collectors.toList())); // Dirty times out for the other one tester.advanceTime(Duration.ofMinutes(14)); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailerTest.java index e304aac5463..8fd67f949d9 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailerTest.java @@ -342,7 +342,7 @@ public class NodeFailerTest { assertEquals(15, tester.nodeRepository.getNodes(NodeType.proxy, Node.State.active).size()); // The first down host is removed, which causes the second one to be moved to failed - tester.nodeRepository.remove(failedHost1); + tester.nodeRepository.removeRecursively(failedHost1); tester.failer.run(); assertEquals( 2, tester.deployer.redeployments); assertEquals(14, tester.nodeRepository.getNodes(NodeType.proxy, Node.State.active).size()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ZooKeeperAccessMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ZooKeeperAccessMaintainerTest.java index ee0b8f55a4b..bba5aa2db8d 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ZooKeeperAccessMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ZooKeeperAccessMaintainerTest.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.config.provision.NodeType; -import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepositoryTester; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.zookeeper.ZooKeeperServer; @@ -49,7 +48,7 @@ public class ZooKeeperAccessMaintainerTest { assertEquals(asSet("host1,host2,host3,host4,host5,server1,server2"), ZooKeeperServer.getAllowedClientHostnames()); tester.nodeRepository().park("host2", Agent.system, "Parking to unit test"); - assertTrue(tester.nodeRepository().remove("host2")); + tester.nodeRepository().removeRecursively("host2"); maintainer.maintain(); assertEquals(2, tester.getNodes(NodeType.tenant).size()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacityTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacityTest.java index 55e1ff8de9f..dce9f694647 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacityTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacityTest.java @@ -72,20 +72,20 @@ public class DockerHostCapacityTest { @Test public void hasCapacity() { - assertTrue(capacity.hasCapacity(host1, flavorDocker)); - assertTrue(capacity.hasCapacity(host1, flavorDocker2)); - assertTrue(capacity.hasCapacity(host2, flavorDocker)); - assertTrue(capacity.hasCapacity(host2, flavorDocker2)); - assertFalse(capacity.hasCapacity(host3, flavorDocker)); // No ip available - assertFalse(capacity.hasCapacity(host3, flavorDocker2)); // No ip available + assertTrue(capacity.hasCapacity(host1, ResourceCapacity.of(flavorDocker))); + assertTrue(capacity.hasCapacity(host1, ResourceCapacity.of(flavorDocker2))); + assertTrue(capacity.hasCapacity(host2, ResourceCapacity.of(flavorDocker))); + assertTrue(capacity.hasCapacity(host2, ResourceCapacity.of(flavorDocker2))); + assertFalse(capacity.hasCapacity(host3, ResourceCapacity.of(flavorDocker))); // No ip available + assertFalse(capacity.hasCapacity(host3, ResourceCapacity.of(flavorDocker2))); // No ip available // Add a new node to host1 to deplete the memory resource Node nodeF = Node.create("nodeF", Collections.singleton("::6"), Collections.emptySet(), "nodeF", Optional.of("host1"), flavorDocker, NodeType.tenant); nodes.add(nodeF); capacity = new DockerHostCapacity(nodes); - assertFalse(capacity.hasCapacity(host1, flavorDocker)); - assertFalse(capacity.hasCapacity(host1, flavorDocker2)); + assertFalse(capacity.hasCapacity(host1, ResourceCapacity.of(flavorDocker))); + assertFalse(capacity.hasCapacity(host1, ResourceCapacity.of(flavorDocker2))); } @Test diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisioningTest.java index f2b5624d3b8..c26865d5690 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisioningTest.java @@ -48,18 +48,17 @@ public class DynamicDockerProvisioningTest { /** * Test relocation of nodes that violate headroom. - * + * <p> * Setup 4 docker hosts and allocate one container on each (from two different applications) * No spares - only headroom (4xd-2) - * + * <p> * One application is now violating headroom and need relocation - * - * Initial allocation of app 1 and 2 --> final allocation (headroom marked as H): - * + * <p> + * Initial allocation of app 1 and 2 --> final allocation (headroom marked as H): + * <p> * | H | H | H | H | | | | | | * | H | H | H1a | H1b | --> | | | | | * | | | 2a | 2b | | 1a | 1b | 2a | 2b | - * */ @Test public void relocate_nodes_from_headroom_hosts() { @@ -97,18 +96,17 @@ public class DynamicDockerProvisioningTest { /** * Test relocation of nodes from spare hosts. - * + * <p> * Setup 4 docker hosts and allocate one container on each (from two different applications) * No headroom defined - only 2 spares. - * + * <p> * Check that it relocates containers away from the 2 spares - * - * Initial allocation of app 1 and 2 --> final allocation: - * + * <p> + * Initial allocation of app 1 and 2 --> final allocation: + * <p> * | | | | | | | | | | * | | | | | --> | 2a | 2b | | | * | 1a | 1b | 2a | 2b | | 1a | 1b | | | - * */ @Test public void relocate_nodes_from_spare_hosts() { @@ -146,8 +144,136 @@ public class DynamicDockerProvisioningTest { } /** - * Test an allocation workflow: + * Test that new docker nodes that will result in headroom violations are + * correctly marked as this. + * <p> + * When redeploying app1 - should not do anything (as moving app1 to host 0 and 1 would violate headroom). + * Then redeploy app 2 - should cause a relocation. + * <p> + * | H | H | H2a | H2b | | H | H | H | H | + * | H | H | H1a | H1b | --> | H | H | H1a | H1b | + * | | | 1a | 1b | | 2a | 2b | 1a | 1b | + */ + @Test + public void new_docker_nodes_are_marked_as_headroom_violations() { + ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.perf, RegionName.from("us-east")), flavorsConfig(true)); + enableDynamicAllocation(tester); + tester.makeReadyNodes(4, "host", "host-small", NodeType.host, 32); + deployZoneApp(tester); + List<Node> dockerHosts = tester.nodeRepository().getNodes(NodeType.host, Node.State.active); + Flavor flavorD2 = tester.nodeRepository().getAvailableFlavors().getFlavorOrThrow("d-2"); + Flavor flavorD1 = tester.nodeRepository().getAvailableFlavors().getFlavorOrThrow("d-1"); + + // Application 1 + ApplicationId application1 = makeApplicationId("t1", "1"); + ClusterSpec clusterSpec1 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), Version.fromString("6.100")); + String hostParent2 = dockerHosts.get(2).hostname(); + String hostParent3 = dockerHosts.get(3).hostname(); + addAndAssignNode(application1, "1a", hostParent2, flavorD2, 0, tester); + addAndAssignNode(application1, "1b", hostParent3, flavorD2, 1, tester); + + // Application 2 + ApplicationId application2 = makeApplicationId("t2", "2"); + ClusterSpec clusterSpec2 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), Version.fromString("6.100")); + addAndAssignNode(application2, "2a", hostParent2, flavorD1, 0, tester); + addAndAssignNode(application2, "2b", hostParent3, flavorD1, 1, tester); + + // Assert allocation placement - prior to re-deployment + assertApplicationHosts(tester.nodeRepository().getNodes(application1), hostParent2, hostParent3); + assertApplicationHosts(tester.nodeRepository().getNodes(application2), hostParent2, hostParent3); + + // Redeploy application 1 + deployapp(application1, clusterSpec1, flavorD2, tester, 2); + + // Re-assert allocation placement + assertApplicationHosts(tester.nodeRepository().getNodes(application1), hostParent2, hostParent3); + assertApplicationHosts(tester.nodeRepository().getNodes(application2), hostParent2, hostParent3); + + // Redeploy application 2 + deployapp(application2, clusterSpec2, flavorD1, tester, 2); + + // Now app2 should have re-located + assertApplicationHosts(tester.nodeRepository().getNodes(application1), hostParent2, hostParent3); + assertApplicationHosts(tester.nodeRepository().getNodes(application2), dockerHosts.get(0).hostname(), dockerHosts.get(1).hostname()); + } + + /** + * Test that we only relocate the smallest nodes from a host to free up headroom. + * <p> + * The reason we want to do this is that it is an cheap approximation for the optimal solution as we + * pick headroom to be on the hosts were we are closest to fulfill the headroom requirement. * + * Both applications could be moved here to free up headroom - but we want app2 (which is smallest) to be moved. + * <p> + * | H | H | H2a | H2b | | H | H | H | H | + * | H | H | H1a | H1b | --> | H | H | H | H | + * | | | 1a | 1b | | 2a | 2b | 1a | 1b | + * | | | | | | | | 1a | 1b | + */ + @Test + public void only_preferred_container_is_moved_from_hosts_with_headroom_violations() { + ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.perf, RegionName.from("us-east")), flavorsConfig(true)); + enableDynamicAllocation(tester); + tester.makeReadyNodes(4, "host", "host-medium", NodeType.host, 32); + deployZoneApp(tester); + List<Node> dockerHosts = tester.nodeRepository().getNodes(NodeType.host, Node.State.active); + Flavor flavorD2 = tester.nodeRepository().getAvailableFlavors().getFlavorOrThrow("d-2"); + Flavor flavorD1 = tester.nodeRepository().getAvailableFlavors().getFlavorOrThrow("d-1"); + + // Application 1 + ApplicationId application1 = makeApplicationId("t1", "1"); + ClusterSpec clusterSpec1 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), Version.fromString("6.100")); + String hostParent2 = dockerHosts.get(2).hostname(); + String hostParent3 = dockerHosts.get(3).hostname(); + addAndAssignNode(application1, "1a", hostParent2, flavorD2, 0, tester); + addAndAssignNode(application1, "1b", hostParent3, flavorD2, 1, tester); + + // Application 2 + ApplicationId application2 = makeApplicationId("t2", "2"); + ClusterSpec clusterSpec2 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), Version.fromString("6.100")); + addAndAssignNode(application2, "2a", hostParent2, flavorD1, 0, tester); + addAndAssignNode(application2, "2b", hostParent3, flavorD1, 1, tester); + + // Assert allocation placement - prior to re-deployment + assertApplicationHosts(tester.nodeRepository().getNodes(application1), hostParent2, hostParent3); + assertApplicationHosts(tester.nodeRepository().getNodes(application2), hostParent2, hostParent3); + + // Redeploy application 1 + deployapp(application1, clusterSpec1, flavorD2, tester, 2); + + // Re-assert allocation placement + assertApplicationHosts(tester.nodeRepository().getNodes(application1), hostParent2, hostParent3); + assertApplicationHosts(tester.nodeRepository().getNodes(application2), hostParent2, hostParent3); + + // Redeploy application 2 + deployapp(application2, clusterSpec2, flavorD1, tester, 2); + + // Now app2 should have re-located + assertApplicationHosts(tester.nodeRepository().getNodes(application1), hostParent2, hostParent3); + assertApplicationHosts(tester.nodeRepository().getNodes(application2), dockerHosts.get(0).hostname(), dockerHosts.get(1).hostname()); + } + + private void assertApplicationHosts(List<Node> nodes, String... parents) { + for (Node node : nodes) { + // Ignore retired and non-active nodes + if (!node.state().equals(Node.State.active) || + node.allocation().get().membership().retired()) { + continue; + } + boolean found = false; + for (String parent : parents) { + if (node.parentHostname().get().equals(parent)) { + found = true; + break; + } + } + Assert.assertTrue(found); + } + } + + /** + * Test an allocation workflow: + * <p> * 5 Hosts of capacity 3 (2 spares) * - Allocate app with 3 nodes * - Allocate app with 2 nodes @@ -195,23 +321,22 @@ public class DynamicDockerProvisioningTest { numberOfChildrenStat.put(nofChildren, numberOfChildrenStat.get(nofChildren) + 1); } - assertEquals(3l, (long)numberOfChildrenStat.get(3)); - assertEquals(1l, (long)numberOfChildrenStat.get(0)); - assertEquals(1l, (long)numberOfChildrenStat.get(1)); + assertEquals(3l, (long) numberOfChildrenStat.get(3)); + assertEquals(1l, (long) numberOfChildrenStat.get(0)); + assertEquals(1l, (long) numberOfChildrenStat.get(1)); } /** * Test redeployment of nodes that violates spare headroom - but without alternatives - * + * <p> * Setup 2 docker hosts and allocate one app with a container on each * No headroom defined - only 2 spares. - * + * <p> * Initial allocation of app 1 --> final allocation: - * + * <p> * | | | | | | * | | | --> | | | * | 1a | 1b | | 1a | 1b | - * */ @Test public void do_not_relocate_nodes_from_spare_if_no_where_to_reloacte_them() { @@ -341,15 +466,15 @@ public class DynamicDockerProvisioningTest { } private void deployapp(ApplicationId id, ClusterSpec spec, Flavor flavor, ProvisioningTester tester, int nodecount) { - List<HostSpec> hostSpec = tester.prepare(id, spec, nodecount,1, flavor.canonicalName()); + List<HostSpec> hostSpec = tester.prepare(id, spec, nodecount, 1, flavor.canonicalName()); tester.activate(id, new HashSet<>(hostSpec)); } private Node addAndAssignNode(ApplicationId id, String hostname, String parentHostname, Flavor flavor, int index, ProvisioningTester tester) { Node node1a = Node.create("open1", Collections.singleton("127.0.0.100"), new HashSet<>(), hostname, Optional.of(parentHostname), flavor, NodeType.tenant); ClusterSpec clusterSpec = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), Version.fromString("6.100")).changeGroup(Optional.of(ClusterSpec.Group.from(0))); - ClusterMembership clusterMembership1 = ClusterMembership.from(clusterSpec,index); - Node node1aAllocation = node1a.allocate(id,clusterMembership1, Instant.now()); + ClusterMembership clusterMembership1 = ClusterMembership.from(clusterSpec, index); + Node node1aAllocation = node1a.allocate(id, clusterMembership1, Instant.now()); tester.nodeRepository().addNodes(Collections.singletonList(node1aAllocation)); NestedTransaction transaction = new NestedTransaction().add(new CuratorTransaction(tester.getCurator())); @@ -372,6 +497,7 @@ public class DynamicDockerProvisioningTest { FlavorConfigBuilder b = new FlavorConfigBuilder(); b.addFlavor("host-large", 6., 6., 6, Flavor.Type.BARE_METAL); b.addFlavor("host-small", 3., 3., 3, Flavor.Type.BARE_METAL); + b.addFlavor("host-medium", 4., 4., 4, Flavor.Type.BARE_METAL); b.addFlavor("d-1", 1, 1., 1, Flavor.Type.DOCKER_CONTAINER); b.addFlavor("d-2", 2, 2., 2, Flavor.Type.DOCKER_CONTAINER); if (includeHeadroom) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizerTest.java new file mode 100644 index 00000000000..04c0af5d98a --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizerTest.java @@ -0,0 +1,86 @@ +package com.yahoo.vespa.hosted.provision.provisioning;// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +import com.yahoo.component.Version; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterMembership; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.NodeFlavors; +import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provisioning.FlavorsConfig; +import com.yahoo.vespa.hosted.provision.Node; +import org.junit.Assert; +import org.junit.Test; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; + +/** + * @author smorgrav + */ +public class NodePrioritizerTest { + + private static NodeFlavors flavors = new NodeFlavors(flavorsConfig()); + + @Test + public void relocated_nodes_are_preferred() { + List<Node> nodes = new ArrayList<>(); + Node parent = createParent("parent"); + Node b = createNode(parent, "b", "d2"); + nodes.add(b); + + // Only one node - should be obvious what to prefer + Assert.assertTrue(NodePrioritizer.isPreferredNodeToBeReloacted(nodes, b, parent)); + + // Two equal nodes - choose lexically + Node a = createNode(parent, "a", "d2"); + nodes.add(a); + Assert.assertTrue(NodePrioritizer.isPreferredNodeToBeReloacted(nodes, a, parent)); + Assert.assertFalse(NodePrioritizer.isPreferredNodeToBeReloacted(nodes, b, parent)); + + // Smallest node should be preferred + Node c = createNode(parent, "c", "d1"); + nodes.add(c); + Assert.assertTrue(NodePrioritizer.isPreferredNodeToBeReloacted(nodes, c, parent)); + + // Unallocated over allocated + ClusterSpec spec = ClusterSpec.from(ClusterSpec.Type.content, ClusterSpec.Id.from("mycluster"), ClusterSpec.Group.from(0), Version.fromString("6.142.22")); + c = c.allocate(ApplicationId.defaultId(), ClusterMembership.from(spec, 0), Instant.now()); + nodes.remove(c); + nodes.add(c); + Node d = createNode(parent, "d", "d1"); + nodes.add(d); + Assert.assertTrue(NodePrioritizer.isPreferredNodeToBeReloacted(nodes, d, parent)); + Assert.assertFalse(NodePrioritizer.isPreferredNodeToBeReloacted(nodes, c, parent)); + + // Container over content + ClusterSpec spec2 = ClusterSpec.from(ClusterSpec.Type.container, ClusterSpec.Id.from("mycluster"), ClusterSpec.Group.from(0), Version.fromString("6.142.22")); + d = d.allocate(ApplicationId.defaultId(), ClusterMembership.from(spec2, 0), Instant.now()); + nodes.remove(d); + nodes.add(d); + Assert.assertFalse(NodePrioritizer.isPreferredNodeToBeReloacted(nodes, c, parent)); + Assert.assertTrue(NodePrioritizer.isPreferredNodeToBeReloacted(nodes, d, parent)); + } + + private static Node createNode(Node parent, String hostname, String flavor) { + return Node.createDockerNode("openid", Collections.singleton("127.0.0.1"), new HashSet<>(), hostname, Optional.of(parent.hostname()), + flavors.getFlavorOrThrow(flavor), NodeType.tenant); + } + + private static Node createParent(String hostname) { + return Node.create("openid", Collections.singleton("127.0.0.1"), new HashSet<>(), hostname, Optional.empty(), + flavors.getFlavorOrThrow("host-large"), NodeType.host); + } + + private static FlavorsConfig flavorsConfig() { + FlavorConfigBuilder b = new FlavorConfigBuilder(); + b.addFlavor("host-large", 6., 6., 6, Flavor.Type.BARE_METAL); + b.addFlavor("d1", 1, 1., 1, Flavor.Type.DOCKER_CONTAINER); + b.addFlavor("d2", 2, 2., 2, Flavor.Type.DOCKER_CONTAINER); + return b.build(); + } +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java index 6bd158e8311..1c82675dbab 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java @@ -132,6 +132,7 @@ public class ProvisioningTester implements AutoCloseable { public List<HostSpec> prepare(ApplicationId application, ClusterSpec cluster, int nodeCount, int groups, String flavor) { return prepare(application, cluster, Capacity.fromNodeCount(nodeCount, Optional.ofNullable(flavor)), groups); } + public List<HostSpec> prepare(ApplicationId application, ClusterSpec cluster, Capacity capacity, int groups) { Set<String> reservedBefore = toHostNames(nodeRepository.getNodes(application, Node.State.reserved)); Set<String> inactiveBefore = toHostNames(nodeRepository.getNodes(application, Node.State.inactive)); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java index deb3378679c..38d3bf46028 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java @@ -401,7 +401,7 @@ public class RestApiTest { // Attempt to DELETE a node which is not put in a deletable state first assertResponse(new Request("http://localhost:8080/nodes/v2/node/host2.yahoo.com", new byte[0], Request.Method.DELETE), - 404, "{\"error-code\":\"NOT_FOUND\",\"message\":\"No node in the provisioned, parked or failed state with hostname host2.yahoo.com\"}"); + 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Failed to delete host2.yahoo.com: Node host2.yahoo.com can only be removed from following states: provisioned, failed, parked\"}"); // PUT current restart generation with string instead of long assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node1.json index 075ce1693cb..56523f58164 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node1.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node1.json @@ -37,6 +37,11 @@ "wantToDeprovision": false, "history": [ { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { "event": "readied", "at": 123, "agent": "system" diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node10.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node10.json index 120d6286634..cb24df66cf8 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node10.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node10.json @@ -42,6 +42,11 @@ "wantToDeprovision": false, "history": [ { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { "event": "readied", "at": 123, "agent": "system" diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node11.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node11.json index 9b3feffad42..e0e899afbef 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node11.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node11.json @@ -1,26 +1,38 @@ { - "url":"http://localhost:8080/nodes/v2/node/host11.yahoo.com", - "id":"host11.yahoo.com", - "state":"provisioned", + "url": "http://localhost:8080/nodes/v2/node/host11.yahoo.com", + "id": "host11.yahoo.com", + "state": "provisioned", "type": "tenant", - "hostname":"host11.yahoo.com", - "parentHostname":"parent.host.yahoo.com", - "openStackId":"host11.yahoo.com", - "flavor":"docker", - "canonicalFlavor":"docker", - "minDiskAvailableGb":100.0, - "minMainMemoryAvailableGb":0.5, - "description":"Flavor-name-is-docker", - "minCpuCores":0.2, - "fastDisk":true, - "environment":"DOCKER_CONTAINER", - "rebootGeneration":0, - "currentRebootGeneration":0, - "failCount":0, - "hardwareFailure":false, - "wantToRetire":false, - "wantToDeprovision" : false, - "history":[], - "ipAddresses":["::1", "127.0.0.1"], - "additionalIpAddresses":["::10","::11"] + "hostname": "host11.yahoo.com", + "parentHostname": "parent.host.yahoo.com", + "openStackId": "host11.yahoo.com", + "flavor": "docker", + "canonicalFlavor": "docker", + "minDiskAvailableGb": 100.0, + "minMainMemoryAvailableGb": 0.5, + "description": "Flavor-name-is-docker", + "minCpuCores": 0.2, + "fastDisk": true, + "environment": "DOCKER_CONTAINER", + "rebootGeneration": 0, + "currentRebootGeneration": 0, + "failCount": 0, + "hardwareFailure": false, + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "provisioned", + "at": 123, + "agent": "system" + } + ], + "ipAddresses": [ + "::1", + "127.0.0.1" + ], + "additionalIpAddresses": [ + "::10", + "::11" + ] } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node2.json index 52864fc165c..30057dda1d7 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node2.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node2.json @@ -37,6 +37,11 @@ "wantToDeprovision": false, "history": [ { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { "event": "readied", "at": 123, "agent": "system" diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node3.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node3.json index 4b7af75ee3c..eb13d077d7f 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node3.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node3.json @@ -22,6 +22,11 @@ "wantToDeprovision": false, "history": [ { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { "event": "readied", "at": 123, "agent": "system" diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json index 4082db74ff4..1fa8feb4586 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json @@ -44,6 +44,11 @@ "wantToDeprovision" : true, "history": [ { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { "event": "readied", "at": 123, "agent": "system" diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4.json index 10b5689f8ce..1e9138283f7 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4.json @@ -42,6 +42,11 @@ "wantToDeprovision": false, "history": [ { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { "event": "readied", "at": 123, "agent": "system" diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5-after-changes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5-after-changes.json index bf81509b79a..1e1ea1a2445 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5-after-changes.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5-after-changes.json @@ -22,6 +22,11 @@ "wantToDeprovision": false, "history": [ { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { "event": "readied", "at": 123, "agent": "system" diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5.json index 1fc001fa224..f8b9fb72e5e 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5.json @@ -26,6 +26,11 @@ "wantToDeprovision": false, "history": [ { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { "event": "readied", "at": 123, "agent": "system" diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node55.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node55.json index 3c483fa3412..7d07037d076 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node55.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node55.json @@ -7,19 +7,33 @@ "openStackId": "node55", "flavor": "default", "canonicalFlavor": "default", - "minDiskAvailableGb":400.0, - "minMainMemoryAvailableGb":16.0, - "description":"Flavor-name-is-default", - "minCpuCores":2.0, - "fastDisk":true, - "environment":"BARE_METAL", + "minDiskAvailableGb": 400.0, + "minMainMemoryAvailableGb": 16.0, + "description": "Flavor-name-is-default", + "minCpuCores": 2.0, + "fastDisk": true, + "environment": "BARE_METAL", "rebootGeneration": 1, "currentRebootGeneration": 0, "failCount": 0, - "hardwareFailure" : false, - "wantToRetire" : false, - "wantToDeprovision" : false, - "history":[{"event":"deallocated","at":123,"agent":"system"}], - "ipAddresses":["::1", "127.0.0.1"], - "additionalIpAddresses":[] + "hardwareFailure": false, + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { + "event": "deallocated", + "at": 123, + "agent": "system" + } + ], + "ipAddresses": [ + "::1", + "127.0.0.1" + ], + "additionalIpAddresses": [] } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-after-changes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-after-changes.json index adb7ce18c80..be9f3d78663 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-after-changes.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-after-changes.json @@ -27,16 +27,40 @@ }, "restartGeneration": 0, "currentRestartGeneration": 0, - "wantedDockerImage":"docker-registry.domain.tld:8080/dist/vespa:6.42.0", - "wantedVespaVersion":"6.42.0", + "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", + "wantedVespaVersion": "6.42.0", "rebootGeneration": 1, "currentRebootGeneration": 0, "failCount": 0, - "hardwareFailure" : false, - "wantToRetire" : false, - "wantToDeprovision" : false, - "history":[{"event":"readied","at":123,"agent":"system"},{"event":"reserved","at":123,"agent":"application"},{"event":"activated","at":123,"agent":"application"}], - "ipAddresses":["::1", "127.0.0.1"], - "additionalIpAddresses":[], - "hardwareDivergence":"{\"actualCpuCores\":2}" + "hardwareFailure": false, + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { + "event": "readied", + "at": 123, + "agent": "system" + }, + { + "event": "reserved", + "at": 123, + "agent": "application" + }, + { + "event": "activated", + "at": 123, + "agent": "application" + } + ], + "ipAddresses": [ + "::1", + "127.0.0.1" + ], + "additionalIpAddresses": [], + "hardwareDivergence": "{\"actualCpuCores\":2}" } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6.json index 750ebbd695e..bd3b15f16ba 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6.json @@ -37,6 +37,11 @@ "wantToDeprovision": false, "history": [ { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { "event": "readied", "at": 123, "agent": "system" diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node7.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node7.json index 9b52598cc67..b1c654b4e00 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node7.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node7.json @@ -7,19 +7,28 @@ "openStackId": "node7", "flavor": "default", "canonicalFlavor": "default", - "minDiskAvailableGb":400.0, - "minMainMemoryAvailableGb":16.0, - "description":"Flavor-name-is-default", - "minCpuCores":2.0, - "fastDisk":true, - "environment":"BARE_METAL", + "minDiskAvailableGb": 400.0, + "minMainMemoryAvailableGb": 16.0, + "description": "Flavor-name-is-default", + "minCpuCores": 2.0, + "fastDisk": true, + "environment": "BARE_METAL", "rebootGeneration": 0, "currentRebootGeneration": 0, "failCount": 0, - "hardwareFailure" : false, - "wantToRetire" : false, - "wantToDeprovision" : false, - "history":[], - "ipAddresses":["::1", "127.0.0.1"], - "additionalIpAddresses":[] + "hardwareFailure": false, + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "provisioned", + "at": 123, + "agent": "system" + } + ], + "ipAddresses": [ + "::1", + "127.0.0.1" + ], + "additionalIpAddresses": [] } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node8.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node8.json index 34fb9d13d81..0fa0bf6631d 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node8.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node8.json @@ -7,19 +7,27 @@ "openStackId": "host8.yahoo.com", "flavor": "default", "canonicalFlavor": "default", - "minDiskAvailableGb":400.0, - "minMainMemoryAvailableGb":16.0, - "description":"Flavor-name-is-default", - "minCpuCores":2.0, - "fastDisk":true, - "environment":"BARE_METAL", + "minDiskAvailableGb": 400.0, + "minMainMemoryAvailableGb": 16.0, + "description": "Flavor-name-is-default", + "minCpuCores": 2.0, + "fastDisk": true, + "environment": "BARE_METAL", "rebootGeneration": 0, "currentRebootGeneration": 0, "failCount": 0, - "hardwareFailure" : false, - "wantToRetire" : false, - "wantToDeprovision" : false, - "history":[], - "ipAddresses":["127.0.0.1"], - "additionalIpAddresses":[] + "hardwareFailure": false, + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "provisioned", + "at": 123, + "agent": "system" + } + ], + "ipAddresses": [ + "127.0.0.1" + ], + "additionalIpAddresses": [] } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node9.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node9.json index f0695598264..43405f0aafc 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node9.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node9.json @@ -7,19 +7,27 @@ "openStackId": "host9.yahoo.com", "flavor": "large-variant", "canonicalFlavor": "large", - "minDiskAvailableGb":2000.0, - "minMainMemoryAvailableGb":128.0, - "description":"Flavor-name-is-large-variant", - "minCpuCores":64.0, - "fastDisk":true, - "environment":"BARE_METAL", + "minDiskAvailableGb": 2000.0, + "minMainMemoryAvailableGb": 128.0, + "description": "Flavor-name-is-large-variant", + "minCpuCores": 64.0, + "fastDisk": true, + "environment": "BARE_METAL", "rebootGeneration": 0, "currentRebootGeneration": 0, "failCount": 0, - "hardwareFailure" : false, - "wantToRetire" : false, - "wantToDeprovision" : false, - "history":[], - "ipAddresses":["::1", "127.0.0.1"], - "additionalIpAddresses":[] + "hardwareFailure": false, + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "provisioned", + "at": 123, + "agent": "system" + } + ], + "ipAddresses": [ + "::1", + "127.0.0.1"], + "additionalIpAddresses": [] } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent1.json index da6742275a4..190dbc41f34 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent1.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent1.json @@ -17,15 +17,23 @@ "currentRebootGeneration": 0, "failCount": 0, "hardwareFailure": false, - "wantToRetire" : false, - "wantToDeprovision" : false, + "wantToRetire": false, + "wantToDeprovision": false, "history": [ { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { "event": "readied", "at": 123, "agent": "system" } ], - "ipAddresses":["::1", "127.0.0.1"], - "additionalIpAddresses":[] + "ipAddresses": [ + "::1", + "127.0.0.1" + ], + "additionalIpAddresses": [] } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent2.json index 65735ecb37c..c7f6344d974 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent2.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent2.json @@ -17,9 +17,18 @@ "currentRebootGeneration": 0, "failCount": 0, "hardwareFailure": false, - "wantToRetire" : false, - "wantToDeprovision" : false, - "history": [], - "ipAddresses":["::1", "127.0.0.1"], - "additionalIpAddresses":[] + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "provisioned", + "at": 123, + "agent": "system" + } + ], + "ipAddresses": [ + "::1", + "127.0.0.1" + ], + "additionalIpAddresses": [] } diff --git a/persistence/src/main/resources/configdefinitions/persistence-rpc.def b/persistence/src/main/resources/configdefinitions/persistence-rpc.def index 2400905e3b7..9cae5812760 100644 --- a/persistence/src/main/resources/configdefinitions/persistence-rpc.def +++ b/persistence/src/main/resources/configdefinitions/persistence-rpc.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=1 namespace=persistence port int default=3456 restart @@ -557,11 +557,6 @@ <version>${curator.version}</version> </dependency> <dependency> - <groupId>io.netty</groupId> - <artifactId>netty</artifactId> - <version>3.10.6.Final</version> - </dependency> - <dependency> <groupId>javax.servlet</groupId> <artifactId>javax.servlet-api</artifactId> <version>3.1.0</version> diff --git a/searchcore/src/apps/vespa-dump-feed/vespa-dump-feed.cpp b/searchcore/src/apps/vespa-dump-feed/vespa-dump-feed.cpp index d942049192a..26a745df148 100644 --- a/searchcore/src/apps/vespa-dump-feed/vespa-dump-feed.cpp +++ b/searchcore/src/apps/vespa-dump-feed/vespa-dump-feed.cpp @@ -9,6 +9,7 @@ #include <vespa/messagebus/destinationsession.h> #include <vespa/messagebus/protocolset.h> #include <vespa/messagebus/rpcmessagebus.h> +#include <vespa/messagebus/network/rpcnetworkparams.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/util/signalhandler.h> #include <vespa/vespalib/util/slaveproc.h> diff --git a/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp b/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp index c9761ee7e6e..a389e702c69 100644 --- a/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp +++ b/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp @@ -2017,6 +2017,53 @@ TEST("require that document sizes are saved") TEST_DO(assertSize(dms4, 3, 1)); } +namespace { + +void +assertLidGidFound(uint32_t lid, DocumentMetaStore &dms) +{ + GlobalId gid = createGid(lid); + EXPECT_TRUE(assertLid(lid, gid, dms)); + EXPECT_TRUE(assertGid(gid, lid, dms)); + EXPECT_TRUE(dms.validLid(lid)); +} + +void +assertLidGidNotFound(uint32_t lid, DocumentMetaStore &dms) +{ + GlobalId gid = createGid(lid); + uint32_t resultLid; + GlobalId resultGid; + EXPECT_FALSE(dms.getLid(gid, resultLid)); + EXPECT_FALSE(dms.getGid(lid, resultGid)); + EXPECT_FALSE(dms.validLid(lid)); +} + +} + +TEST("require that multiple lids can be removed with removeBatch()") +{ + DocumentMetaStore dms(createBucketDB()); + dms.constructFreeList(); + TEST_DO(addLid(dms, 1)); + TEST_DO(addLid(dms, 2)); + TEST_DO(addLid(dms, 3)); + TEST_DO(addLid(dms, 4)); + + TEST_DO(assertLidGidFound(1, dms)); + TEST_DO(assertLidGidFound(2, dms)); + TEST_DO(assertLidGidFound(3, dms)); + TEST_DO(assertLidGidFound(4, dms)); + + dms.removeBatch({1, 3}, 5); + dms.removeBatchComplete({1, 3}); + + TEST_DO(assertLidGidNotFound(1, dms)); + TEST_DO(assertLidGidFound(2, dms)); + TEST_DO(assertLidGidNotFound(3, dms)); + TEST_DO(assertLidGidFound(4, dms)); +} + } TEST_MAIN() diff --git a/searchcore/src/tests/proton/summaryengine/summaryengine.cpp b/searchcore/src/tests/proton/summaryengine/summaryengine.cpp index a17c19804c5..82c7b22c8c7 100644 --- a/searchcore/src/tests/proton/summaryengine/summaryengine.cpp +++ b/searchcore/src/tests/proton/summaryengine/summaryengine.cpp @@ -6,7 +6,7 @@ #include <vespa/searchlib/util/rawbuf.h> #include <vespa/searchlib/util/slime_output_raw_buf_adapter.h> #include <vespa/vespalib/data/databuffer.h> -#include <vespa/document/util/compressor.h> +#include <vespa/vespalib/util/compressor.h> #include <vespa/searchlib/common/transport.h> #include <vespa/fnet/frt/rpcrequest.h> #include <vespa/log/log.h> @@ -20,6 +20,8 @@ using vespalib::stringref; using vespalib::ConstBufferRef; using vespalib::DataBuffer; using vespalib::Memory; +using vespalib::compression::CompressionConfig; + namespace proton { @@ -377,7 +379,7 @@ TEST("requireThatSlimeInterfaceWorksFine") { } void -verifyReply(size_t count, document::CompressionConfig::Type encoding, size_t orgSize, size_t compressedSize, +verifyReply(size_t count, CompressionConfig::Type encoding, size_t orgSize, size_t compressedSize, FRT_RPCRequest *request) { FRT_Values &ret = *request->GetReturn(); EXPECT_EQUAL(encoding, ret[0]._intval8); @@ -386,7 +388,8 @@ verifyReply(size_t count, document::CompressionConfig::Type encoding, size_t org DataBuffer uncompressed; ConstBufferRef blob(ret[2]._data._buf, ret[2]._data._len); - compression::decompress(CompressionConfig::toType(ret[0]._intval8), ret[1]._intval32, blob, uncompressed, false); + vespalib::compression::decompress(CompressionConfig::toType(ret[0]._intval8), ret[1]._intval32, + blob, uncompressed, false); EXPECT_EQUAL(orgSize, uncompressed.getDataLen()); vespalib::Slime summaries; @@ -396,8 +399,8 @@ verifyReply(size_t count, document::CompressionConfig::Type encoding, size_t org void verifyRPC(size_t count, - document::CompressionConfig::Type requestCompression, size_t requestSize, size_t requestBlobSize, - document::CompressionConfig::Type replyCompression, size_t replySize, size_t replyBlobSize) { + CompressionConfig::Type requestCompression, size_t requestSize, size_t requestBlobSize, + CompressionConfig::Type replyCompression, size_t replySize, size_t replyBlobSize) { Server server; vespalib::Slime slimeRequest = createSlimeRequestLarger(count); vespalib::SimpleBuffer buf; @@ -406,8 +409,9 @@ verifyRPC(size_t count, CompressionConfig config(requestCompression, 9, 100); DataBuffer compressed(const_cast<char *>(buf.get().data), buf.get().size); - CompressionConfig::Type type = compression::compress(config, ConstBufferRef(buf.get().data, buf.get().size), - compressed, true); + CompressionConfig::Type type = vespalib::compression::compress(config, + ConstBufferRef(buf.get().data, buf.get().size), + compressed, true); EXPECT_EQUAL(type, requestCompression); FRT_RPCRequest *request = new FRT_RPCRequest(); @@ -424,9 +428,9 @@ verifyRPC(size_t count, } TEST("requireThatRPCInterfaceWorks") { - verifyRPC(1, document::CompressionConfig::NONE, 55, 55, document::CompressionConfig::NONE, 38, 38); - verifyRPC(100, document::CompressionConfig::NONE, 2631, 2631, document::CompressionConfig::LZ4, 1426, 46); - verifyRPC(100, document::CompressionConfig::LZ4, 2631, 69, document::CompressionConfig::LZ4, 1426, 46); + verifyRPC(1, CompressionConfig::NONE, 55, 55, CompressionConfig::NONE, 38, 38); + verifyRPC(100, CompressionConfig::NONE, 2631, 2631, CompressionConfig::LZ4, 1426, 46); + verifyRPC(100, CompressionConfig::LZ4, 2631, 69, CompressionConfig::LZ4, 1426, 46); } } diff --git a/searchcore/src/vespa/searchcore/fdispatch/program/fdispatch.cpp b/searchcore/src/vespa/searchcore/fdispatch/program/fdispatch.cpp index fc0a57ff519..b6a1133d01d 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/program/fdispatch.cpp +++ b/searchcore/src/vespa/searchcore/fdispatch/program/fdispatch.cpp @@ -19,7 +19,7 @@ LOG_SETUP(".fdispatch"); using search::fs4transport::FS4PersistentPacketStreamer; using vespa::config::search::core::FdispatchrcConfig; using vespa::config::search::core::internal::InternalFdispatchrcType; -using document::CompressionConfig; +using vespalib::compression::CompressionConfig; char FastS_VersionTag[] = V_TAG; diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp index e65209bf526..79d82108ee8 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp @@ -13,7 +13,6 @@ #include <vespa/searchsummary/docsummary/docsumconfig.h> #include <vespa/vespalib/util/exceptions.h> #include <sstream> -#include <future> #include <vespa/log/log.h> LOG_SETUP(".proton.docsummary.summarymanager"); @@ -26,6 +25,8 @@ using namespace vespa::config::search::summary; using namespace vespa::config::search; using vespalib::make_string; using vespalib::IllegalArgumentException; +using vespalib::compression::CompressionConfig; + using search::DocumentStore; using search::IDocumentStore; using search::LogDocumentStore; @@ -160,13 +161,13 @@ SummaryManager::createSummarySetup(const SummaryConfig & summaryCfg, namespace { template<typename T> -document::CompressionConfig +CompressionConfig deriveCompression(const T & config) { - document::CompressionConfig compression; + CompressionConfig compression; if (config.type == T::LZ4) { - compression.type = document::CompressionConfig::LZ4; + compression.type = CompressionConfig::LZ4; } else if (config.type == T::ZSTD) { - compression.type = document::CompressionConfig::ZSTD; + compression.type = CompressionConfig::ZSTD; } compression.compressionLevel = config.level; return compression; diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp index c47d8592b69..2fc395d4438 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp @@ -591,7 +591,7 @@ DocumentMetaStore::updateMetaData(DocId lid, } bool -DocumentMetaStore::remove(DocId lid) +DocumentMetaStore::remove(DocId lid, BucketDBOwner::Guard &bucketGuard) { if (!validLid(lid)) { return false; @@ -606,15 +606,23 @@ DocumentMetaStore::remove(DocId lid) lid, gid.toString().c_str())); } _lidAlloc.unregisterLid(lid); - incGeneration(); RawDocumentMetaData &oldMetaData = _metaDataStore[lid]; - _bucketDB->takeGuard()->remove(oldMetaData.getGid(), - oldMetaData.getBucketId().stripUnused(), - oldMetaData.getTimestamp(), oldMetaData.getDocSize(), - _subDbType); + bucketGuard->remove(oldMetaData.getGid(), + oldMetaData.getBucketId().stripUnused(), + oldMetaData.getTimestamp(), oldMetaData.getDocSize(), + _subDbType); return true; } +bool +DocumentMetaStore::remove(DocId lid) +{ + BucketDBOwner::Guard bucketGuard = _bucketDB->takeGuard(); + bool result = remove(lid, bucketGuard); + incGeneration(); + return result; +} + void DocumentMetaStore::removeComplete(DocId lid) { @@ -649,14 +657,16 @@ DocumentMetaStore::move(DocId fromLid, DocId toLid) void DocumentMetaStore::removeBatch(const std::vector<DocId> &lidsToRemove, const uint32_t docIdLimit) { + BucketDBOwner::Guard bucketGuard = _bucketDB->takeGuard(); for (const auto &lid : lidsToRemove) { assert(lid > 0 && lid < docIdLimit); (void) docIdLimit; - bool removed = remove(lid); + bool removed = remove(lid, bucketGuard); assert(removed); (void) removed; } + incGeneration(); } void diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.h index 613952f5c4f..180fb438808 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.h +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.h @@ -136,6 +136,8 @@ private: VESPA_DLL_LOCAL DocId readNextDoc(documentmetastore::Reader & reader, TreeType::Builder & treeBuilder); + bool remove(DocId lid, BucketDBOwner::Guard &bucketGuard); + public: typedef TreeType::Iterator Iterator; typedef TreeType::ConstIterator ConstIterator; diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp index ed350453da2..d95b0fd44d1 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp @@ -48,7 +48,7 @@ using search::index::SchemaBuilder; using search::transactionlog::DomainStats; using vespa::config::search::core::ProtonConfig; using vespa::config::search::core::internal::InternalProtonType; -using document::CompressionConfig; +using vespalib::compression::CompressionConfig; namespace proton { diff --git a/searchcore/src/vespa/searchcore/proton/server/rpc_hooks.cpp b/searchcore/src/vespa/searchcore/proton/server/rpc_hooks.cpp index 2e8c985fc0a..0595cb5d983 100644 --- a/searchcore/src/vespa/searchcore/proton/server/rpc_hooks.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/rpc_hooks.cpp @@ -9,7 +9,7 @@ LOG_SETUP(".proton.server.rtchooks"); using namespace vespalib; -using document::CompressionConfig; +using vespalib::compression::CompressionConfig; namespace { diff --git a/searchcore/src/vespa/searchcore/proton/summaryengine/docsum_by_slime.cpp b/searchcore/src/vespa/searchcore/proton/summaryengine/docsum_by_slime.cpp index 9a37e0ae495..79928f96d7a 100644 --- a/searchcore/src/vespa/searchcore/proton/summaryengine/docsum_by_slime.cpp +++ b/searchcore/src/vespa/searchcore/proton/summaryengine/docsum_by_slime.cpp @@ -1,6 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "docsum_by_slime.h" -#include <vespa/document/util/compressor.h> +#include <vespa/vespalib/util/compressor.h> #include <vespa/searchlib/util/slime_output_raw_buf_adapter.h> #include <vespa/searchlib/common/packets.h> #include <vespa/fnet/frt/rpcrequest.h> @@ -23,7 +23,7 @@ using vespalib::slime::ArrayTraverser; using vespalib::SimpleBuffer; using vespalib::DataBuffer; using vespalib::ConstBufferRef; -using document::CompressionConfig; +using vespalib::compression::CompressionConfig; namespace { @@ -99,8 +99,8 @@ DocsumByRPC::DocsumByRPC(DocsumBySlime & slimeDocsumServer) : void DocsumByRPC::getDocsums(FRT_RPCRequest & req) { - using document::compression::decompress; - using document::compression::compress; + using vespalib::compression::decompress; + using vespalib::compression::compress; FRT_Values &arg = *req.GetParams(); uint8_t encoding = arg[0]._intval8; uint32_t uncompressedSize = arg[1]._intval32; diff --git a/searchlib/src/apps/docstore/create-idx-from-dat.cpp b/searchlib/src/apps/docstore/create-idx-from-dat.cpp index ac3b34a3b41..6124b36de8d 100644 --- a/searchlib/src/apps/docstore/create-idx-from-dat.cpp +++ b/searchlib/src/apps/docstore/create-idx-from-dat.cpp @@ -39,7 +39,7 @@ bool tryDecode(size_t chunks, size_t offset, const char * p, size_t sz, size_t n } bool validUncompressed(const char * n, size_t offset) { - return (n[1] == document::CompressionConfig::NONE) && + return (n[1] == vespalib::compression::CompressionConfig::NONE) && (n[2] == 0) && (n[3] == 0) && (n[4] == 0) && diff --git a/searchlib/src/tests/common/packets/packets_test.cpp b/searchlib/src/tests/common/packets/packets_test.cpp index 7504e1c1570..cf0e858a014 100644 --- a/searchlib/src/tests/common/packets/packets_test.cpp +++ b/searchlib/src/tests/common/packets/packets_test.cpp @@ -8,6 +8,8 @@ #include <vespa/fnet/controlpacket.h> using namespace search::fs4transport; +using vespalib::compression::CompressionConfig; + // ---------------------------------------------------------------------------- // @@ -524,7 +526,7 @@ TEST("test pre serializing packets with compression") { EXPECT_EQUAL(src->GetLength(), decoded->GetLength()); FS4PersistentPacketStreamer::Instance.SetCompressionLimit(100); FS4Packet_PreSerialized serialized(*src); - EXPECT_EQUAL(218u | (document::CompressionConfig::LZ4 << 24), serialized.GetPCODE()); + EXPECT_EQUAL(218u | (CompressionConfig::LZ4 << 24), serialized.GetPCODE()); EXPECT_GREATER_EQUAL(321u, serialized.GetLength()); FNET_Packet::UP decoded2(testEncodeDecode(serialized)); EXPECT_EQUAL(500u, decoded2->GetLength()); diff --git a/searchlib/src/tests/docstore/chunk/chunk_test.cpp b/searchlib/src/tests/docstore/chunk/chunk_test.cpp index 4687f45acde..84ac877c54d 100644 --- a/searchlib/src/tests/docstore/chunk/chunk_test.cpp +++ b/searchlib/src/tests/docstore/chunk/chunk_test.cpp @@ -12,7 +12,7 @@ LOG_SETUP("chunk_test"); using namespace search; -using document::CompressionConfig; +using vespalib::compression::CompressionConfig; TEST("require that Chunk obey limits") { diff --git a/searchlib/src/tests/docstore/document_store/document_store_test.cpp b/searchlib/src/tests/docstore/document_store/document_store_test.cpp index e3e4a1432d1..e8c2173a87f 100644 --- a/searchlib/src/tests/docstore/document_store/document_store_test.cpp +++ b/searchlib/src/tests/docstore/document_store/document_store_test.cpp @@ -5,6 +5,7 @@ #include <vespa/document/repo/documenttyperepo.h> using namespace search; +using CompressionConfig = vespalib::compression::CompressionConfig; document::DocumentTypeRepo repo; @@ -43,7 +44,7 @@ struct NullDataStore : IDataStore { }; TEST_FFF("require that uncache docstore lookups are counted", - DocumentStore::Config(document::CompressionConfig::NONE, 0, 0), + DocumentStore::Config(CompressionConfig::NONE, 0, 0), NullDataStore(), DocumentStore(f1, f2)) { EXPECT_EQUAL(0u, f3.getCacheStats().misses); @@ -52,7 +53,7 @@ TEST_FFF("require that uncache docstore lookups are counted", } TEST_FFF("require that cached docstore lookups are counted", - DocumentStore::Config(document::CompressionConfig::NONE, 100000, 100), + DocumentStore::Config(CompressionConfig::NONE, 100000, 100), NullDataStore(), DocumentStore(f1, f2)) { EXPECT_EQUAL(0u, f3.getCacheStats().misses); diff --git a/searchlib/src/tests/docstore/document_store/visitcache_test.cpp b/searchlib/src/tests/docstore/document_store/visitcache_test.cpp index 70e0c7f01fb..d5d95097c66 100644 --- a/searchlib/src/tests/docstore/document_store/visitcache_test.cpp +++ b/searchlib/src/tests/docstore/document_store/visitcache_test.cpp @@ -56,11 +56,12 @@ void verifyAB(const BlobSet & a) { using B=vespalib::ConstBufferRef; TEST("require that BlobSet can be built") { + using CompressionConfig = vespalib::compression::CompressionConfig; BlobSet a; a.append(7, B("aaaaaa",6)); a.append(9, B("bbbbb",5)); verifyAB(a); - document::CompressionConfig cfg(document::CompressionConfig::LZ4); + CompressionConfig cfg(CompressionConfig::LZ4); CompressedBlobSet ca(cfg, a); BlobSet b = ca.getBlobSet(); verifyAB(b); diff --git a/searchlib/src/tests/docstore/document_store_visitor/document_store_visitor_test.cpp b/searchlib/src/tests/docstore/document_store_visitor/document_store_visitor_test.cpp index 944d0a543f5..1c7053500c7 100644 --- a/searchlib/src/tests/docstore/document_store_visitor/document_store_visitor_test.cpp +++ b/searchlib/src/tests/docstore/document_store_visitor/document_store_visitor_test.cpp @@ -24,7 +24,7 @@ using document::Document; using document::DocumentId; using document::DocumentType; using document::DocumentTypeRepo; -using document::CompressionConfig; +using vespalib::compression::CompressionConfig; using vespalib::asciistream; using index::DummyFileHeaderContext; diff --git a/searchlib/src/tests/docstore/file_chunk/file_chunk_test.cpp b/searchlib/src/tests/docstore/file_chunk/file_chunk_test.cpp index 6d75c6365f9..598913a3222 100644 --- a/searchlib/src/tests/docstore/file_chunk/file_chunk_test.cpp +++ b/searchlib/src/tests/docstore/file_chunk/file_chunk_test.cpp @@ -113,6 +113,7 @@ struct ReadFixture : public FixtureBase { struct WriteFixture : public FixtureBase { WriteableFileChunk chunk; + using CompressionConfig = vespalib::compression::CompressionConfig; WriteFixture(const vespalib::string &baseName, uint32_t docIdLimit, @@ -124,7 +125,7 @@ struct WriteFixture : public FixtureBase { baseName, serialNum, docIdLimit, - WriteableFileChunk::Config(document::CompressionConfig(), 0x1000), + WriteableFileChunk::Config(CompressionConfig(), 0x1000), tuneFile, fileHeaderCtx, &bucketizer, diff --git a/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp b/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp index 82a158fd53e..ed6afb06681 100644 --- a/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp +++ b/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp @@ -15,7 +15,6 @@ #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/threadstackexecutor.h> #include <iomanip> -#include <iostream> using document::BucketId; using namespace search::docstore; @@ -36,7 +35,7 @@ public: using namespace search; using namespace search::docstore; using search::index::DummyFileHeaderContext; -using document::CompressionConfig; +using vespalib::compression::CompressionConfig; namespace { diff --git a/searchlib/src/tests/docstore/store_by_bucket/store_by_bucket_test.cpp b/searchlib/src/tests/docstore/store_by_bucket/store_by_bucket_test.cpp index 3faf9395ec5..e9514c1d385 100644 --- a/searchlib/src/tests/docstore/store_by_bucket/store_by_bucket_test.cpp +++ b/searchlib/src/tests/docstore/store_by_bucket/store_by_bucket_test.cpp @@ -4,17 +4,17 @@ #include <vespa/document/bucket/bucketid.h> #include <vespa/document/base/documentid.h> -#include <vespa/log/log.h> #include <vespa/searchlib/docstore/storebybucket.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/stllike/hash_set.h> #include <vespa/vespalib/util/threadstackexecutor.h> +#include <vespa/log/log.h> LOG_SETUP("store_by_bucket_test"); using namespace search::docstore; using document::BucketId; -using document::CompressionConfig; +using vespalib::compression::CompressionConfig; vespalib::string createPayload(BucketId b) { diff --git a/searchlib/src/vespa/searchlib/attribute/attribute_header.cpp b/searchlib/src/vespa/searchlib/attribute/attribute_header.cpp index 0848d9996c3..4a69e0a827d 100644 --- a/searchlib/src/vespa/searchlib/attribute/attribute_header.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attribute_header.cpp @@ -2,9 +2,9 @@ #include "attribute_header.h" #include <vespa/vespalib/data/fileheader.h> +#include <vespa/vespalib/data/databuffer.h> -namespace search { -namespace attribute { +namespace search::attribute { namespace { @@ -168,5 +168,4 @@ AttributeHeader::hasWeightedSetType() const return _collectionType.isWeightedSet(); } -} // namespace search::attribute -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/attribute/imported_search_context.cpp b/searchlib/src/vespa/searchlib/attribute/imported_search_context.cpp index b0ac12ce8f9..0834df280fd 100644 --- a/searchlib/src/vespa/searchlib/attribute/imported_search_context.cpp +++ b/searchlib/src/vespa/searchlib/attribute/imported_search_context.cpp @@ -33,6 +33,7 @@ ImportedSearchContext::ImportedSearchContext( _target_attribute(*_imported_attribute.getTargetAttribute()), _target_search_context(_target_attribute.getSearch(std::move(term), params)), _referencedLids(_reference_attribute.getReferencedLids()), + _referencedLidLimit(_target_attribute.getCommittedDocIdLimit()), _merger(_reference_attribute.getCommittedDocIdLimit()), _fetchPostingsDone(false) { @@ -190,13 +191,5 @@ const vespalib::string& ImportedSearchContext::attributeName() const { return _imported_attribute.getName(); } -bool ImportedSearchContext::cmp(DocId docId, int32_t& weight) const { - return _target_search_context->cmp(_referencedLids[docId], weight); -} - -bool ImportedSearchContext::cmp(DocId docId) const { - return _target_search_context->cmp(_referencedLids[docId]); -} - } // attribute } // search diff --git a/searchlib/src/vespa/searchlib/attribute/imported_search_context.h b/searchlib/src/vespa/searchlib/attribute/imported_search_context.h index 9be4578fac0..c142754e302 100644 --- a/searchlib/src/vespa/searchlib/attribute/imported_search_context.h +++ b/searchlib/src/vespa/searchlib/attribute/imported_search_context.h @@ -35,9 +35,15 @@ class ImportedSearchContext : public ISearchContext { const AttributeVector& _target_attribute; std::unique_ptr<AttributeVector::SearchContext> _target_search_context; ReferencedLids _referencedLids; + uint32_t _referencedLidLimit; PostingListMerger<int32_t> _merger; bool _fetchPostingsDone; + uint32_t getReferencedLid(uint32_t lid) const { + uint32_t referencedLid = _referencedLids[lid]; + return ((referencedLid >= _referencedLidLimit) ? 0u : referencedLid); + } + void makeMergedPostings(); public: ImportedSearchContext(std::unique_ptr<QueryTermSimple> term, @@ -62,8 +68,13 @@ public: using DocId = IAttributeVector::DocId; - bool cmp(DocId docId, int32_t& weight) const; - bool cmp(DocId docId) const; + bool cmp(DocId docId, int32_t& weight) const { + return _target_search_context->cmp(getReferencedLid(docId), weight); + } + + bool cmp(DocId docId) const { + return _target_search_context->cmp(getReferencedLid(docId)); + } const ReferenceAttribute& attribute() const noexcept { return _reference_attribute; } @@ -74,6 +85,3 @@ public: } // attribute } // search - - - diff --git a/searchlib/src/vespa/searchlib/attribute/readerbase.h b/searchlib/src/vespa/searchlib/attribute/readerbase.h index 88081906f71..358038f9ba2 100644 --- a/searchlib/src/vespa/searchlib/attribute/readerbase.h +++ b/searchlib/src/vespa/searchlib/attribute/readerbase.h @@ -3,6 +3,7 @@ #pragma once #include <vespa/searchlib/util/fileutil.h> +#include <cassert> namespace search { diff --git a/searchlib/src/vespa/searchlib/bitcompression/compression.cpp b/searchlib/src/vespa/searchlib/bitcompression/compression.cpp index e50ebec105b..aa39b798b7a 100644 --- a/searchlib/src/vespa/searchlib/bitcompression/compression.cpp +++ b/searchlib/src/vespa/searchlib/bitcompression/compression.cpp @@ -5,10 +5,9 @@ #include <vespa/searchlib/fef/termfieldmatchdata.h> #include <vespa/searchlib/fef/termfieldmatchdataarray.h> #include <vespa/vespalib/data/fileheader.h> +#include <vespa/vespalib/data/databuffer.h> -namespace search { - -namespace bitcompression { +namespace search::bitcompression { using vespalib::nbostream; @@ -157,17 +156,12 @@ DecodeContext64Base::checkPointRead(nbostream &in) (void) in; } -} // namespace bitcompression - - namespace { vespalib::string noFeatures = "NoFeatures"; } -namespace bitcompression { - template <bool bigEndian> void FeatureDecodeContext<bigEndian>:: @@ -486,6 +480,4 @@ template class FeatureEncodeContext<true>; template class FeatureEncodeContext<false>; -} // namespace bitcompression - -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/common/packets.cpp b/searchlib/src/vespa/searchlib/common/packets.cpp index 667334bf64e..a0b64e2ef94 100644 --- a/searchlib/src/vespa/searchlib/common/packets.cpp +++ b/searchlib/src/vespa/searchlib/common/packets.cpp @@ -4,7 +4,7 @@ #include "packets.h" #include "sortdata.h" #include <vespa/searchlib/util/rawbuf.h> -#include <vespa/document/util/compressor.h> +#include <vespa/vespalib/util/compressor.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/data/databuffer.h> @@ -12,7 +12,6 @@ #include <vespa/log/log.h> LOG_SETUP(".searchlib.common.fs4packets"); -using document::CompressionConfig; using vespalib::ConstBufferRef; using vespalib::make_string; using vespalib::stringref; @@ -151,7 +150,7 @@ FS4PersistentPacketStreamer::Decode(FNET_DataBuffer *src, uint32_t plen, uint32_ uint32_t uncompressed_size = src->ReadInt32(); ConstBufferRef org(src->GetData(), plen - sizeof(uint32_t)); vespalib::DataBuffer uncompressed(uncompressed_size); - document::compression::decompress(compressionType, uncompressed_size, org, uncompressed, false); + vespalib::compression::decompress(compressionType, uncompressed_size, org, uncompressed, false); FNET_DataBuffer buf(uncompressed.getData(), uncompressed.getDataLen()); decodePacket(packet, buf, uncompressed_size, pcode); src->DataToDead(plen - sizeof(uint32_t)); @@ -192,7 +191,7 @@ FS4PersistentPacketStreamer::Encode(FNET_Packet *packet, uint32_t chid, FNET_Dat CompressionConfig config(_compressionType, _compressionLevel, 90); ConstBufferRef org(dst->GetData() + packet_start + header_len, body_len); vespalib::DataBuffer compressed(org.size()); - CompressionConfig::Type r = document::compression::compress(config, org, compressed, false); + CompressionConfig::Type r = vespalib::compression::compress(config, org, compressed, false); if (r != CompressionConfig::NONE) { dst->DataToFree(body_len + header_len); // sizeof(data + header + uncompressed_size) - sizeof(uint32_t) @@ -455,7 +454,7 @@ FS4Packet_PreSerialized::FS4Packet_PreSerialized(FNET_Packet & packet) 90); ConstBufferRef org(tmp.GetData(), tmp.GetDataLen()); vespalib::DataBuffer compressed(org.size()); - _compressionType = document::compression::compress(config, org, compressed, false); + _compressionType = vespalib::compression::compress(config, org, compressed, false); if (_compressionType != CompressionConfig::NONE) { _data.WriteInt32Fast(body_len); _data.WriteBytes(compressed.getData(), compressed.getDataLen()); @@ -1285,18 +1284,18 @@ FS4Packet_QUERYX::Encode(FNET_DataBuffer *dst) void FS4Packet::throwPropertieDecodeError(size_t i) { - throw vespalib::IllegalArgumentException(vespalib::make_string("Failed decoding properties[%ld]", i)); + throw vespalib::IllegalArgumentException(make_string("Failed decoding properties[%ld]", i)); } void FS4Packet::throwUnsupportedFeatures(uint32_t features, uint32_t set) { - throw vespalib::UnderflowException(vespalib::make_string("Unsupported features(%x), supported set(%x)", features, set)); + throw vespalib::UnderflowException(make_string("Unsupported features(%x), supported set(%x)", features, set)); } void FS4Packet::throwNotEnoughData(FNET_DataBuffer & buf, uint32_t left, uint32_t needed, const char * text) { (void) buf; - throw vespalib::UnderflowException(vespalib::make_string("Failed decoding packet of type %d. Only %d bytes left, needed %d from '%s'", GetPCODE(), left, needed, text)); + throw vespalib::UnderflowException(make_string("Failed decoding packet of type %d. Only %d bytes left, needed %d from '%s'", GetPCODE(), left, needed, text)); } #define VERIFY_LEN(needed, text) \ @@ -1436,7 +1435,7 @@ FS4Packet_QUERYX::toString(uint32_t indent) const } s += make_string("%*s sortspec : %s\n", indent, "", _sortSpec.c_str()); s += make_string("%*s groupspec : (%d bytes)\n", indent, "", (int)_groupSpec.size()); - s += make_string("%*s sessionId : (%d bytes)\n", indent, "", (int)_sessionId.size()); + s += make_string("%*s sessionId : (%d bytes) %s\n", indent, "", (int)_sessionId.size(), _sessionId.c_str()); s += make_string("%*s location : %s\n", indent, "", _location.c_str()); s += make_string("%*s timeout : %d\n", indent, "", _timeout); s += make_string("%*s stackitems : %d\n", indent, "", _numStackItems); diff --git a/searchlib/src/vespa/searchlib/common/packets.h b/searchlib/src/vespa/searchlib/common/packets.h index d9bc4d50462..52130c57374 100644 --- a/searchlib/src/vespa/searchlib/common/packets.h +++ b/searchlib/src/vespa/searchlib/common/packets.h @@ -9,7 +9,7 @@ #include <vespa/fnet/packet.h> #include <vespa/fnet/databuffer.h> #include <vespa/document/base/globalid.h> -#include <vespa/document/util/compressionconfig.h> +#include <vespa/vespalib/util/compressionconfig.h> #include <vespa/vespalib/util/memory.h> #include <vespa/fastos/timestamp.h> #include <vector> @@ -110,10 +110,11 @@ public: class FS4PersistentPacketStreamer : public FNET_IPacketStreamer { FS4PersistentPacketStreamer(const FS4PersistentPacketStreamer &); FS4PersistentPacketStreamer& operator=(const FS4PersistentPacketStreamer &); + using CompressionConfig = vespalib::compression::CompressionConfig; unsigned int _compressionLimit; unsigned int _compressionLevel; - document::CompressionConfig::Type _compressionType; + CompressionConfig::Type _compressionType; protected: bool _conservative; // Set to true if out of sync should mark the // stream as broken. @@ -139,8 +140,8 @@ public: void SetConservativeMode(bool cons) { _conservative = cons; } void SetCompressionLimit(unsigned int limit) { _compressionLimit = limit; } void SetCompressionLevel(unsigned int level) { _compressionLevel = level; } - void SetCompressionType(document::CompressionConfig::Type compressionType) { _compressionType = compressionType; } - document::CompressionConfig::Type getCompressionType() const { return _compressionType; } + void SetCompressionType(CompressionConfig::Type compressionType) { _compressionType = compressionType; } + CompressionConfig::Type getCompressionType() const { return _compressionType; } uint32_t getCompressionLimit() const { return _compressionLimit; } uint32_t getCompressionLevel() const { return _compressionLevel; } }; @@ -243,9 +244,10 @@ public: bool Decode(FNET_DataBuffer *src, uint32_t len) override; vespalib::string toString(uint32_t indent) const override; private: - uint32_t _pcode; - document::CompressionConfig::Type _compressionType; - FNET_DataBuffer _data; + using CompressionConfig = vespalib::compression::CompressionConfig; + uint32_t _pcode; + CompressionConfig::Type _compressionType; + FNET_DataBuffer _data; }; class FS4Packet_Shared : public FS4Packet diff --git a/searchlib/src/vespa/searchlib/diskindex/checkpointfile.cpp b/searchlib/src/vespa/searchlib/diskindex/checkpointfile.cpp index a55ec9494dc..0324f00f63c 100644 --- a/searchlib/src/vespa/searchlib/diskindex/checkpointfile.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/checkpointfile.cpp @@ -1,18 +1,16 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - #include "checkpointfile.h" #include <vespa/vespalib/data/fileheader.h> #include <vespa/searchlib/common/fileheadercontext.h> +#include <cassert> #include <vespa/log/log.h> LOG_SETUP(".diskindex.checkpointfile"); using vespalib::getLastErrorString; -namespace search { - -namespace diskindex { +namespace search::diskindex { using common::FileHeaderContext; @@ -182,6 +180,4 @@ CheckPointFile::readHeader() } -} // namespace diskindex - -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/docstore/chunk.cpp b/searchlib/src/vespa/searchlib/docstore/chunk.cpp index 179fda8689b..ba467501fba 100644 --- a/searchlib/src/vespa/searchlib/docstore/chunk.cpp +++ b/searchlib/src/vespa/searchlib/docstore/chunk.cpp @@ -43,12 +43,12 @@ Chunk::hasRoom(size_t len) const } size_t -Chunk::getMaxPackSize(const document::CompressionConfig & compression) const { +Chunk::getMaxPackSize(const CompressionConfig & compression) const { return _format->getMaxPackSize(compression); } void -Chunk::pack(uint64_t lastSerial, vespalib::DataBuffer & compressed, const document::CompressionConfig & compression) +Chunk::pack(uint64_t lastSerial, vespalib::DataBuffer & compressed, const CompressionConfig & compression) { _lastSerial = lastSerial; _format->pack(_lastSerial, compressed, compression); diff --git a/searchlib/src/vespa/searchlib/docstore/chunk.h b/searchlib/src/vespa/searchlib/docstore/chunk.h index bdf0d9793b2..cd551de1a9e 100644 --- a/searchlib/src/vespa/searchlib/docstore/chunk.h +++ b/searchlib/src/vespa/searchlib/docstore/chunk.h @@ -4,7 +4,7 @@ #include <vespa/searchlib/util/memoryusage.h> #include <vespa/vespalib/util/buffer.h> -#include <vespa/document/util/compressionconfig.h> +#include <vespa/vespalib/util/compressionconfig.h> #include <memory> #include <vector> @@ -60,7 +60,8 @@ private: class Chunk { public: - typedef std::unique_ptr<Chunk> UP; + using UP = std::unique_ptr<Chunk>; + using CompressionConfig = vespalib::compression::CompressionConfig; class Config { public: Config(size_t maxBytes) : _maxBytes(maxBytes) { } @@ -93,8 +94,8 @@ public: size_t size() const; const LidList & getLids() const { return _lids; } LidList getUniqueLids() const; - size_t getMaxPackSize(const document::CompressionConfig & compression) const; - void pack(uint64_t lastSerial, vespalib::DataBuffer & buffer, const document::CompressionConfig & compression); + size_t getMaxPackSize(const CompressionConfig & compression) const; + void pack(uint64_t lastSerial, vespalib::DataBuffer & buffer, const CompressionConfig & compression); uint64_t getLastSerial() const { return _lastSerial; } uint32_t getId() const { return _id; } bool validSerial() const { return getLastSerial() != static_cast<uint64_t>(-1l); } diff --git a/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp b/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp index 204c7b07acb..37381cfa3f6 100644 --- a/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp +++ b/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp @@ -1,17 +1,17 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "chunkformats.h" -#include <vespa/document/util/compressor.h> +#include <vespa/vespalib/util/compressor.h> #include <vespa/vespalib/util/stringfmt.h> namespace search { using vespalib::make_string; using vespalib::Exception; -using document::compression::compress; -using document::compression::decompress; -using document::compression::computeMaxCompressedsize; -using document::CompressionConfig; +using vespalib::compression::compress; +using vespalib::compression::decompress; +using vespalib::compression::computeMaxCompressedsize; +using vespalib::compression::CompressionConfig; ChunkException::ChunkException(const vespalib::stringref & msg, const vespalib::stringref & location) : Exception(make_string("Illegal chunk: %s", msg.c_str()), location) diff --git a/searchlib/src/vespa/searchlib/docstore/chunkformat.h b/searchlib/src/vespa/searchlib/docstore/chunkformat.h index 5f1e3d852a9..9f9580f1f1d 100644 --- a/searchlib/src/vespa/searchlib/docstore/chunkformat.h +++ b/searchlib/src/vespa/searchlib/docstore/chunkformat.h @@ -2,7 +2,7 @@ #pragma once -#include <vespa/document/util/compressionconfig.h> +#include <vespa/vespalib/util/compressionconfig.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/data/databuffer.h> #include <vespa/vespalib/util/exception.h> @@ -20,7 +20,8 @@ class ChunkFormat { public: virtual ~ChunkFormat(); - typedef std::unique_ptr<ChunkFormat> UP; + using UP = std::unique_ptr<ChunkFormat>; + using CompressionConfig = vespalib::compression::CompressionConfig; vespalib::nbostream & getBuffer() { return _dataBuf; } const vespalib::nbostream & getBuffer() const { return _dataBuf; } @@ -30,7 +31,7 @@ public: * @param compressed The buffer where the serialized data shall be placed. * @param compression What kind of compression shall be employed. */ - void pack(uint64_t lastSerial, vespalib::DataBuffer & compressed, const document::CompressionConfig & compression); + void pack(uint64_t lastSerial, vespalib::DataBuffer & compressed, const CompressionConfig & compression); /** * Will deserialize and create a representation of the uncompressed data. * param buffer Pointer to the serialized data @@ -44,7 +45,7 @@ public: * @param compression Compression config to be used. * @return maximum number of bytes a packet can take in serialized form. */ - size_t getMaxPackSize(const document::CompressionConfig & compression) const; + size_t getMaxPackSize(const CompressionConfig & compression) const; protected: /** * Constructor used when deserializing diff --git a/searchlib/src/vespa/searchlib/docstore/compacter.h b/searchlib/src/vespa/searchlib/docstore/compacter.h index 264cce8d3aa..362896487aa 100644 --- a/searchlib/src/vespa/searchlib/docstore/compacter.h +++ b/searchlib/src/vespa/searchlib/docstore/compacter.h @@ -33,7 +33,7 @@ private: **/ class BucketCompacter : public IWriteData, public StoreByBucket::IWrite { - using CompressionConfig = document::CompressionConfig; + using CompressionConfig = vespalib::compression::CompressionConfig; using ThreadExecutor = vespalib::ThreadExecutor; public: using FileId = FileChunk::FileId; diff --git a/searchlib/src/vespa/searchlib/docstore/documentstore.cpp b/searchlib/src/vespa/searchlib/docstore/documentstore.cpp index a13fffdaa24..bf59614a297 100644 --- a/searchlib/src/vespa/searchlib/docstore/documentstore.cpp +++ b/searchlib/src/vespa/searchlib/docstore/documentstore.cpp @@ -6,12 +6,12 @@ #include "ibucketizer.h" #include <vespa/vespalib/stllike/cache.hpp> #include <vespa/vespalib/data/databuffer.h> -#include <vespa/document/util/compressor.h> +#include <vespa/vespalib/util/compressor.h> using document::DocumentTypeRepo; -using document::CompressionConfig; -using document::compression::compress; -using document::compression::decompress; +using vespalib::compression::CompressionConfig; +using vespalib::compression::compress; +using vespalib::compression::decompress; namespace search { diff --git a/searchlib/src/vespa/searchlib/docstore/documentstore.h b/searchlib/src/vespa/searchlib/docstore/documentstore.h index 45dfcb35e37..4ba5c27cd07 100644 --- a/searchlib/src/vespa/searchlib/docstore/documentstore.h +++ b/searchlib/src/vespa/searchlib/docstore/documentstore.h @@ -26,25 +26,26 @@ class DocumentStore : public IDocumentStore public: class Config { public: + using CompressionConfig = vespalib::compression::CompressionConfig; Config() : - _compression(document::CompressionConfig::LZ4, 9, 70), + _compression(CompressionConfig::LZ4, 9, 70), _maxCacheBytes(1000000000), _initialCacheEntries(0), _allowVisitCaching(false) { } - Config(const document::CompressionConfig & compression, size_t maxCacheBytes, size_t initialCacheEntries) : - _compression((maxCacheBytes != 0) ? compression : document::CompressionConfig::NONE), + Config(const CompressionConfig & compression, size_t maxCacheBytes, size_t initialCacheEntries) : + _compression((maxCacheBytes != 0) ? compression : CompressionConfig::NONE), _maxCacheBytes(maxCacheBytes), _initialCacheEntries(initialCacheEntries), _allowVisitCaching(false) { } - const document::CompressionConfig & getCompression() const { return _compression; } + const CompressionConfig & getCompression() const { return _compression; } size_t getMaxCacheBytes() const { return _maxCacheBytes; } size_t getInitialCacheEntries() const { return _initialCacheEntries; } bool allowVisitCaching() const { return _allowVisitCaching; } Config & allowVisitCaching(bool allow) { _allowVisitCaching = allow; return *this; } private: - document::CompressionConfig _compression; + CompressionConfig _compression; size_t _maxCacheBytes; size_t _initialCacheEntries; bool _allowVisitCaching; diff --git a/searchlib/src/vespa/searchlib/docstore/filechunk.cpp b/searchlib/src/vespa/searchlib/docstore/filechunk.cpp index f271e6f320b..2a748a302c6 100644 --- a/searchlib/src/vespa/searchlib/docstore/filechunk.cpp +++ b/searchlib/src/vespa/searchlib/docstore/filechunk.cpp @@ -5,6 +5,7 @@ #include "summaryexceptions.h" #include "randreaders.h" #include <vespa/vespalib/data/fileheader.h> +#include <vespa/vespalib/data/databuffer.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/util/array.hpp> #include <vespa/vespalib/stllike/hash_map.hpp> diff --git a/searchlib/src/vespa/searchlib/docstore/logdatastore.h b/searchlib/src/vespa/searchlib/docstore/logdatastore.h index 0ca658ba803..080e6f80503 100644 --- a/searchlib/src/vespa/searchlib/docstore/logdatastore.h +++ b/searchlib/src/vespa/searchlib/docstore/logdatastore.h @@ -5,7 +5,7 @@ #include "idatastore.h" #include "lid_info.h" #include "writeablefilechunk.h" -#include <vespa/document/util/compressionconfig.h> +#include <vespa/vespalib/util/compressionconfig.h> #include <vespa/searchlib/common/rcuvector.h> #include <vespa/searchlib/common/tunefileinfo.h> #include <vespa/searchlib/transactionlog/syncproxy.h> @@ -33,7 +33,7 @@ private: public: using NameIdSet = std::set<NameId>; using LockGuard = vespalib::LockGuard; - using CompressionConfig = document::CompressionConfig; + using CompressionConfig = vespalib::compression::CompressionConfig; class Config { public: Config() diff --git a/searchlib/src/vespa/searchlib/docstore/storebybucket.h b/searchlib/src/vespa/searchlib/docstore/storebybucket.h index f93826027f5..ac1f6fbe007 100644 --- a/searchlib/src/vespa/searchlib/docstore/storebybucket.h +++ b/searchlib/src/vespa/searchlib/docstore/storebybucket.h @@ -23,9 +23,9 @@ class StoreByBucket using MemoryDataStore = vespalib::MemoryDataStore; using ThreadExecutor = vespalib::ThreadExecutor; using ConstBufferRef = vespalib::ConstBufferRef; - using CompressionConfig = document::CompressionConfig; + using CompressionConfig = vespalib::compression::CompressionConfig; public: - StoreByBucket(vespalib::MemoryDataStore & backingMemory, const document::CompressionConfig & compression); + StoreByBucket(vespalib::MemoryDataStore & backingMemory, const CompressionConfig & compression); StoreByBucket(MemoryDataStore & backingMemory, ThreadExecutor & executor, const CompressionConfig & compression); StoreByBucket(StoreByBucket &&) = default; ~StoreByBucket(); diff --git a/searchlib/src/vespa/searchlib/docstore/visitcache.cpp b/searchlib/src/vespa/searchlib/docstore/visitcache.cpp index 22a16947b67..b3fd236d73d 100644 --- a/searchlib/src/vespa/searchlib/docstore/visitcache.cpp +++ b/searchlib/src/vespa/searchlib/docstore/visitcache.cpp @@ -5,7 +5,7 @@ #include <vespa/vespalib/stllike/cache.hpp> #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/vespalib/data/databuffer.h> -#include <vespa/document/util/compressor.h> +#include <vespa/vespalib/util/compressor.h> #include <algorithm> namespace search::docstore { @@ -74,7 +74,7 @@ BlobSet::get(uint32_t lid) const } CompressedBlobSet::CompressedBlobSet() : - _compression(document::CompressionConfig::Type::LZ4), + _compression(CompressionConfig::Type::LZ4), _positions(), _buffer() { @@ -83,7 +83,7 @@ CompressedBlobSet::CompressedBlobSet() : CompressedBlobSet::~CompressedBlobSet() { } -CompressedBlobSet::CompressedBlobSet(const document::CompressionConfig &compression, const BlobSet & uncompressed) : +CompressedBlobSet::CompressedBlobSet(const CompressionConfig &compression, const BlobSet & uncompressed) : _compression(compression.type), _positions(uncompressed.getPositions()), _buffer() @@ -91,7 +91,7 @@ CompressedBlobSet::CompressedBlobSet(const document::CompressionConfig &compress if ( ! _positions.empty() ) { DataBuffer compressed; ConstBufferRef org = uncompressed.getBuffer(); - _compression = document::compression::compress(compression, org, compressed, false); + _compression = vespalib::compression::compress(compression, org, compressed, false); _buffer.resize(compressed.getDataLen()); memcpy(_buffer, compressed.getData(), compressed.getDataLen()); } @@ -100,7 +100,7 @@ CompressedBlobSet::CompressedBlobSet(const document::CompressionConfig &compress BlobSet CompressedBlobSet::getBlobSet() const { - using document::compression::decompress; + using vespalib::compression::decompress; // These are frequent lage allocations that are to expensive to mmap. DataBuffer uncompressed(0, 1, Alloc::alloc(0, 16 * MemoryAllocator::HUGEPAGE_SIZE)); if ( ! _positions.empty() ) { @@ -145,7 +145,7 @@ VisitCache::BackingStore::read(const KeySet &key, CompressedBlobSet &blobs) cons return ! blobs.empty(); } -VisitCache::VisitCache(IDataStore &store, size_t cacheSize, const document::CompressionConfig &compression) : +VisitCache::VisitCache(IDataStore &store, size_t cacheSize, const CompressionConfig &compression) : _store(store, compression), _cache(std::make_unique<Cache>(_store, cacheSize)) { diff --git a/searchlib/src/vespa/searchlib/docstore/visitcache.h b/searchlib/src/vespa/searchlib/docstore/visitcache.h index 44ee5542a72..a89620b7bde 100644 --- a/searchlib/src/vespa/searchlib/docstore/visitcache.h +++ b/searchlib/src/vespa/searchlib/docstore/visitcache.h @@ -72,8 +72,9 @@ private: **/ class CompressedBlobSet { public: + using CompressionConfig = vespalib::compression::CompressionConfig; CompressedBlobSet(); - CompressedBlobSet(const document::CompressionConfig &compression, const BlobSet & uncompressed); + CompressedBlobSet(const CompressionConfig &compression, const BlobSet & uncompressed); CompressedBlobSet(CompressedBlobSet && rhs) = default; CompressedBlobSet & operator=(CompressedBlobSet && rhs) = default; CompressedBlobSet(const CompressedBlobSet & rhs) = default; @@ -83,7 +84,7 @@ public: bool empty() const { return _positions.empty(); } BlobSet getBlobSet() const; private: - document::CompressionConfig::Type _compression; + CompressionConfig::Type _compression; BlobSet::Positions _positions; vespalib::MallocPtr _buffer; }; @@ -95,7 +96,8 @@ private: **/ class VisitCache { public: - VisitCache(IDataStore &store, size_t cacheSize, const document::CompressionConfig &compression); + using CompressionConfig = vespalib::compression::CompressionConfig; + VisitCache(IDataStore &store, size_t cacheSize, const CompressionConfig &compression); CompressedBlobSet read(const IDocumentStore::LidVector & keys) const; void remove(uint32_t key); @@ -111,17 +113,17 @@ private: */ class BackingStore { public: - BackingStore(IDataStore &store, const document::CompressionConfig &compression) : + BackingStore(IDataStore &store, const CompressionConfig &compression) : _backingStore(store), _compression(compression) { } bool read(const KeySet &key, CompressedBlobSet &blobs) const; void write(const KeySet &, const CompressedBlobSet &) { } void erase(const KeySet &) { } - const document::CompressionConfig &getCompression() const { return _compression; } + const CompressionConfig &getCompression() const { return _compression; } private: IDataStore &_backingStore; - const document::CompressionConfig _compression; + const CompressionConfig _compression; }; using CacheParams = vespalib::CacheParam< diff --git a/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp b/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp index c6b93a59ae8..5cf5c646148 100644 --- a/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp +++ b/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp @@ -6,6 +6,7 @@ #include <vespa/vespalib/util/closuretask.h> #include <vespa/vespalib/util/array.hpp> #include <vespa/vespalib/data/fileheader.h> +#include <vespa/vespalib/data/databuffer.h> #include <vespa/searchlib/common/fileheadercontext.h> #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/vespalib/objects/nbostream.h> diff --git a/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.h b/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.h index 86415e0cb40..2b21f12a314 100644 --- a/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.h +++ b/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.h @@ -22,20 +22,21 @@ public: class Config { public: + using CompressionConfig = vespalib::compression::CompressionConfig; Config() - : _compression(document::CompressionConfig::LZ4, 9, 60), + : _compression(CompressionConfig::LZ4, 9, 60), _maxChunkBytes(0x10000) { } - Config(const document::CompressionConfig &compression, size_t maxChunkBytes) + Config(const CompressionConfig &compression, size_t maxChunkBytes) : _compression(compression), _maxChunkBytes(maxChunkBytes) { } - const document::CompressionConfig & getCompression() const { return _compression; } + const CompressionConfig & getCompression() const { return _compression; } size_t getMaxChunkBytes() const { return _maxChunkBytes; } private: - document::CompressionConfig _compression; + CompressionConfig _compression; size_t _maxChunkBytes; }; diff --git a/searchlib/src/vespa/searchlib/grouping/hyperloglog.h b/searchlib/src/vespa/searchlib/grouping/hyperloglog.h index 753b81d92f2..931b832c76d 100644 --- a/searchlib/src/vespa/searchlib/grouping/hyperloglog.h +++ b/searchlib/src/vespa/searchlib/grouping/hyperloglog.h @@ -3,8 +3,8 @@ #pragma once #include "sketch.h" -#include <vespa/document/util/compressionconfig.h> -#include <vespa/document/util/compressor.h> +#include <vespa/vespalib/util/compressionconfig.h> +#include <vespa/vespalib/util/compressor.h> #include <vespa/vespalib/data/databuffer.h> #include <vespa/vespalib/objects/deserializer.h> #include <vespa/vespalib/objects/serializer.h> diff --git a/searchlib/src/vespa/searchlib/grouping/sketch.h b/searchlib/src/vespa/searchlib/grouping/sketch.h index bb7db02d81c..317c1bfef9d 100644 --- a/searchlib/src/vespa/searchlib/grouping/sketch.h +++ b/searchlib/src/vespa/searchlib/grouping/sketch.h @@ -2,8 +2,8 @@ #pragma once -#include <vespa/document/util/compressionconfig.h> -#include <vespa/document/util/compressor.h> +#include <vespa/vespalib/util/compressionconfig.h> +#include <vespa/vespalib/util/compressor.h> #include <lz4.h> #include <vespa/searchlib/common/identifiable.h> #include <vespa/vespalib/data/databuffer.h> @@ -205,13 +205,13 @@ deserialize(vespalib::Deserializer &is) { template <int BucketBits, typename HashT> uint32_t NormalSketch<BucketBits, HashT>:: compress_buckets_into(char *buffer, uint32_t size) const { - document::CompressionConfig config(document::CompressionConfig::LZ4, 9, 9); + vespalib::compression::CompressionConfig config(vespalib::compression::CompressionConfig::LZ4, 9, 9); vespalib::ConstBufferRef org(&bucket[0], BUCKET_COUNT); vespalib::DataBuffer compress_buffer(buffer, size); - document::CompressionConfig::Type r = - document::compression::compress(config, org, compress_buffer, false); + vespalib::compression::CompressionConfig::Type r = + vespalib::compression::compress(config, org, compress_buffer, false); assert(compress_buffer.getDead() == buffer); - if (r == document::CompressionConfig::LZ4) { + if (r == vespalib::compression::CompressionConfig::LZ4) { assert(compress_buffer.getDataLen() < BUCKET_COUNT); return compress_buffer.getDataLen(); } else { @@ -228,7 +228,8 @@ decompress_buckets_from(char *buffer, uint32_t size) { } else { vespalib::ConstBufferRef compressed(buffer, size); vespalib::DataBuffer uncompressed(reinterpret_cast<char *>(&bucket[0]), BUCKET_COUNT); - document::compression::decompress(document::CompressionConfig::LZ4, BUCKET_COUNT, compressed, uncompressed, false); + vespalib::compression::decompress(vespalib::compression::CompressionConfig::LZ4, BUCKET_COUNT, + compressed, uncompressed, false); } } template <int BucketBits, typename HashT> diff --git a/searchlib/src/vespa/searchlib/index/dummyfileheadercontext.cpp b/searchlib/src/vespa/searchlib/index/dummyfileheadercontext.cpp index 6c4155b035e..e5e582b673b 100644 --- a/searchlib/src/vespa/searchlib/index/dummyfileheadercontext.cpp +++ b/searchlib/src/vespa/searchlib/index/dummyfileheadercontext.cpp @@ -4,6 +4,7 @@ #include <vespa/vespalib/data/fileheader.h> #include <vespa/searchlib/util/fileheadertk.h> #include <vespa/vespalib/util/host_name.h> +#include <cassert> #include <unistd.h> namespace search::index { diff --git a/searchlib/src/vespa/searchlib/util/fileutil.cpp b/searchlib/src/vespa/searchlib/util/fileutil.cpp index c1d34f66b0b..674608bdda1 100644 --- a/searchlib/src/vespa/searchlib/util/fileutil.cpp +++ b/searchlib/src/vespa/searchlib/util/fileutil.cpp @@ -4,6 +4,7 @@ #include "filesizecalculator.h" #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/guard.h> +#include <cassert> #include <fcntl.h> #include <sys/mman.h> #include <sys/stat.h> @@ -18,7 +19,6 @@ using vespalib::FileDescriptor; using vespalib::getLastErrorString; namespace search { - namespace fileutil { LoadedMmap::LoadedMmap(const vespalib::string &fileName) @@ -177,4 +177,5 @@ FileWriterBase::write(const void *buf, size_t sz) { } return numWritten; } + } diff --git a/simplemetrics/src/main/resources/configdefinitions/manager.def b/simplemetrics/src/main/resources/configdefinitions/manager.def index 0ca1a93bda3..6446e0df8b6 100644 --- a/simplemetrics/src/main/resources/configdefinitions/manager.def +++ b/simplemetrics/src/main/resources/configdefinitions/manager.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=1 namespace=metrics reportPeriodSeconds int default=60 diff --git a/slobrok/src/apps/slobrok/CMakeLists.txt b/slobrok/src/apps/slobrok/CMakeLists.txt index 09d6a3ce9eb..5e1bcbf56ed 100644 --- a/slobrok/src/apps/slobrok/CMakeLists.txt +++ b/slobrok/src/apps/slobrok/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(slobrok_app SOURCES slobrok.cpp OUTPUT_NAME vespa-slobrok - INSTALL bin + INSTALL sbin DEPENDS slobrok_slobrokserver ) diff --git a/staging_vespalib/CMakeLists.txt b/staging_vespalib/CMakeLists.txt index 5a169416775..d59529df828 100644 --- a/staging_vespalib/CMakeLists.txt +++ b/staging_vespalib/CMakeLists.txt @@ -13,7 +13,6 @@ vespa_define_module( src/tests/bits src/tests/clock src/tests/crc - src/tests/databuffer src/tests/directio src/tests/encoding/base64 src/tests/fileheader diff --git a/staging_vespalib/src/tests/databuffer/.gitignore b/staging_vespalib/src/tests/databuffer/.gitignore deleted file mode 100644 index e7b0e69c372..00000000000 --- a/staging_vespalib/src/tests/databuffer/.gitignore +++ /dev/null @@ -1 +0,0 @@ -staging_vespalib_databuffer_test_app diff --git a/staging_vespalib/src/tests/databuffer/CMakeLists.txt b/staging_vespalib/src/tests/databuffer/CMakeLists.txt deleted file mode 100644 index 0c96a8cdf3e..00000000000 --- a/staging_vespalib/src/tests/databuffer/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(staging_vespalib_databuffer_test_app TEST - SOURCES - databuffer_test.cpp - DEPENDS - staging_vespalib -) -vespa_add_test(NAME staging_vespalib_databuffer_test_app COMMAND staging_vespalib_databuffer_test_app) diff --git a/staging_vespalib/src/tests/fileheader/fileheader_test.cpp b/staging_vespalib/src/tests/fileheader/fileheader_test.cpp index b895b92db4c..f6f9c19b48c 100644 --- a/staging_vespalib/src/tests/fileheader/fileheader_test.cpp +++ b/staging_vespalib/src/tests/fileheader/fileheader_test.cpp @@ -2,6 +2,7 @@ #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/data/fileheader.h> +#include <vespa/vespalib/data/databuffer.h> #include <vespa/fastos/file.h> #include <iostream> diff --git a/staging_vespalib/src/vespa/vespalib/data/CMakeLists.txt b/staging_vespalib/src/vespa/vespalib/data/CMakeLists.txt index 3fbece6211d..8ca70d2a63d 100644 --- a/staging_vespalib/src/vespa/vespalib/data/CMakeLists.txt +++ b/staging_vespalib/src/vespa/vespalib/data/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_library(staging_vespalib_vespalib_data OBJECT SOURCES - databuffer.cpp fileheader.cpp DEPENDS ) diff --git a/staging_vespalib/src/vespa/vespalib/data/fileheader.cpp b/staging_vespalib/src/vespa/vespalib/data/fileheader.cpp index 3cd8190d894..a9e131b0c63 100644 --- a/staging_vespalib/src/vespa/vespalib/data/fileheader.cpp +++ b/staging_vespalib/src/vespa/vespalib/data/fileheader.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "fileheader.h" #include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/vespalib/data/databuffer.h> #include <vespa/fastos/file.h> #include <vespa/log/log.h> diff --git a/staging_vespalib/src/vespa/vespalib/data/fileheader.h b/staging_vespalib/src/vespa/vespalib/data/fileheader.h index 157703ef0a8..e4449a0c36a 100644 --- a/staging_vespalib/src/vespa/vespalib/data/fileheader.h +++ b/staging_vespalib/src/vespa/vespalib/data/fileheader.h @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include "databuffer.h" #include <vespa/vespalib/util/exception.h> #include <map> @@ -9,6 +8,7 @@ class FastOS_FileInterface; namespace vespalib { +class DataBuffer; class asciistream; /** diff --git a/statistics/src/main/resources/configdefinitions/statistics.def b/statistics/src/main/resources/configdefinitions/statistics.def index 5a9bd30e50b..92da7704714 100644 --- a/statistics/src/main/resources/configdefinitions/statistics.def +++ b/statistics/src/main/resources/configdefinitions/statistics.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=6 namespace=container ## Interval between internal sample points measured in seconds diff --git a/storage/src/tests/distributor/statecheckerstest.cpp b/storage/src/tests/distributor/statecheckerstest.cpp index fbaa8a7662d..29c922248e7 100644 --- a/storage/src/tests/distributor/statecheckerstest.cpp +++ b/storage/src/tests/distributor/statecheckerstest.cpp @@ -83,6 +83,7 @@ struct StateCheckersTest : public CppUnit::TestFixture, void inhibitBucketDeactivationIfDisabledInConfig(); void retiredNodesOutOfSyncAreMerged(); void testGarbageCollection(); + void gc_ops_are_prioritized_with_low_priority_category(); void gcInhibitedWhenIdealNodeInMaintenance(); void testNoRemoveWhenIdealNodeInMaintenance(); void testStepwiseJoinForSmallBucketsWithoutSiblings(); @@ -186,7 +187,8 @@ struct StateCheckersTest : public CppUnit::TestFixture, uint32_t nowTimestamp, uint32_t checkInterval, uint32_t lastChangeTime = 0, - bool includePriority = false); + bool includePriority = false, + bool includeSchedulingPri = false); std::string testSplit(uint32_t splitCount, uint32_t splitSize, @@ -321,6 +323,7 @@ struct StateCheckersTest : public CppUnit::TestFixture, CPPUNIT_TEST(inhibitBucketActivationIfDisabledInConfig); CPPUNIT_TEST(inhibitBucketDeactivationIfDisabledInConfig); CPPUNIT_TEST(testGarbageCollection); + CPPUNIT_TEST(gc_ops_are_prioritized_with_low_priority_category); CPPUNIT_TEST(gcInhibitedWhenIdealNodeInMaintenance); CPPUNIT_TEST(testNoRemoveWhenIdealNodeInMaintenance); CPPUNIT_TEST(testStepwiseJoinForSmallBucketsWithoutSiblings); @@ -1458,7 +1461,7 @@ StateCheckersTest::inhibitBucketDeactivationIfDisabledInConfig() std::string StateCheckersTest::testGarbageCollection( uint32_t prevTimestamp, uint32_t nowTimestamp, uint32_t checkInterval, uint32_t lastChangeTime, - bool includePriority) + bool includePriority, bool includeSchedulingPri) { BucketDatabase::Entry e(document::BucketId(17, 0)); e.getBucketInfo().addNode(BucketCopy(prevTimestamp, 0, @@ -1475,7 +1478,7 @@ std::string StateCheckersTest::testGarbageCollection( e.getBucketId()); getClock().setAbsoluteTimeInSeconds(nowTimestamp); return testStateChecker(checker, c, false, PendingMessage(), - includePriority); + includePriority, includeSchedulingPri); } void @@ -1527,6 +1530,13 @@ StateCheckersTest::testGarbageCollection() testGarbageCollection(3850, 4000, 300, 1)); } +void StateCheckersTest::gc_ops_are_prioritized_with_low_priority_category() { + CPPUNIT_ASSERT_EQUAL( + std::string("[Needs garbage collection: Last check at 3, current time 4000, " + "configured interval 300] (scheduling pri LOW)"), + testGarbageCollection(3, 4000, 300, 1, false, true)); +} + /** * When a node is in maintenance, we want to do our best to avoid any unneeded * changes to the bucket replicas' states, as this will require re-syncing of diff --git a/storage/src/vespa/storage/config/stor-communicationmanager.def b/storage/src/vespa/storage/config/stor-communicationmanager.def index 23c713d41f9..2c00dbe2f37 100644 --- a/storage/src/vespa/storage/config/stor-communicationmanager.def +++ b/storage/src/vespa/storage/config/stor-communicationmanager.def @@ -17,3 +17,11 @@ mbus_content_node_max_pending_count int default=0 mbus_distributor_node_max_pending_size int default=0 mbus_content_node_max_pending_size int default=0 +# Minimum size of packets to compress (0 means no compression) +mbus.compress.limit int default=1024 + +## Compression level for packets +mbus.compress.level int default=3 + +## Compression type for packets. +mbus.compress.type enum {NONE, LZ4, ZSTD} default=LZ4 diff --git a/storage/src/vespa/storage/distributor/statecheckers.cpp b/storage/src/vespa/storage/distributor/statecheckers.cpp index f81b6d2bcb6..701a147a56f 100644 --- a/storage/src/vespa/storage/distributor/statecheckers.cpp +++ b/storage/src/vespa/storage/distributor/statecheckers.cpp @@ -1140,7 +1140,7 @@ GarbageCollectionStateChecker::check(Context& c) op->setPriority(c.distributorConfig.getMaintenancePriorities() .garbageCollection); op->setDetailedReason(reason.c_str()); - return Result::createStoredResult(std::move(op), MaintenancePriority::MEDIUM); + return Result::createStoredResult(std::move(op), MaintenancePriority::LOW); } else { return Result::noMaintenanceNeeded(); } diff --git a/storage/src/vespa/storage/storageserver/communicationmanager.cpp b/storage/src/vespa/storage/storageserver/communicationmanager.cpp index aac52550fa0..ae8f3290fef 100644 --- a/storage/src/vespa/storage/storageserver/communicationmanager.cpp +++ b/storage/src/vespa/storage/storageserver/communicationmanager.cpp @@ -8,6 +8,7 @@ #include <vespa/documentapi/messagebus/messages/wrongdistributionreply.h> #include <vespa/storageapi/message/state.h> #include <vespa/messagebus/rpcmessagebus.h> +#include <vespa/messagebus/network/rpcnetworkparams.h> #include <vespa/messagebus/emptyreply.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/util/stringfmt.h> @@ -362,8 +363,7 @@ void CommunicationManager::onClose() } void -CommunicationManager::configureMessageBusLimits( - const CommunicationManagerConfig& cfg) +CommunicationManager::configureMessageBusLimits(const CommunicationManagerConfig& cfg) { const bool isDist(_component.getNodeType() == lib::NodeType::DISTRIBUTOR); auto& mbus(_mbus->getMessageBus()); diff --git a/storageserver/src/tests/storageservertest.cpp b/storageserver/src/tests/storageservertest.cpp index f3595afe1e7..21b190d9e76 100644 --- a/storageserver/src/tests/storageservertest.cpp +++ b/storageserver/src/tests/storageservertest.cpp @@ -6,6 +6,7 @@ #include <vespa/document/base/testdocman.h> #include <vespa/documentapi/documentapi.h> #include <vespa/messagebus/rpcmessagebus.h> +#include <vespa/messagebus/network/rpcnetworkparams.h> #include <vespa/memfilepersistence/spi/memfilepersistenceprovider.h> #include <vespa/messagebus/staticthrottlepolicy.h> #include <vespa/messagebus/testlib/slobrok.h> @@ -20,8 +21,8 @@ #include <vespa/storageserver/app/distributorprocess.h> #include <vespa/storageserver/app/memfileservicelayerprocess.h> #include <vespa/vespalib/util/exceptions.h> +#include <vespa/fnet/frt/supervisor.h> #include <sys/time.h> -#include <fstream> #include <vespa/log/log.h> LOG_SETUP(".storageservertest"); diff --git a/travis/travis-build-cpp.sh b/travis/travis-build-cpp.sh index 42dbf0e6467..825da67bf54 100755 --- a/travis/travis-build-cpp.sh +++ b/travis/travis-build-cpp.sh @@ -7,11 +7,13 @@ BUILD_DIR=~/build mkdir "${BUILD_DIR}" -export CCACHE_SIZE="1G" +export CCACHE_MAXSIZE="1250M" export CCACHE_COMPRESS=1 NUM_THREADS=4 +ccache --print-config cd ${BUILD_DIR} bash ${SOURCE_DIR}/bootstrap-cpp.sh ${SOURCE_DIR} ${BUILD_DIR} make -j ${NUM_THREADS} ctest3 --output-on-failure -j ${NUM_THREADS} +ccache --show-stats diff --git a/vagrant/.gitignore b/vagrant/.gitignore new file mode 100644 index 00000000000..a977916f658 --- /dev/null +++ b/vagrant/.gitignore @@ -0,0 +1 @@ +.vagrant/ diff --git a/vagrant/README.md b/vagrant/README.md index 7c69f8f22b0..02d11900558 100644 --- a/vagrant/README.md +++ b/vagrant/README.md @@ -25,4 +25,4 @@ This is needed in order to compile and run tests fast on the local file system i ## Build C++ modules -Please follow the instructions described [here](../README.md). +Please follow the instructions described [here](../README.md#build-c-modules). diff --git a/vagrant/Vagrantfile b/vagrant/Vagrantfile index 7238e7de5b4..74b2f5bdbdb 100644 --- a/vagrant/Vagrantfile +++ b/vagrant/Vagrantfile @@ -35,7 +35,7 @@ Vagrant.configure("2") do |config| yum-builddep -y /vagrant/dist/vespa.spec echo -e "* soft nproc 409600\n* hard nproc 409600" > /etc/security/limits.d/99-nproc.conf echo -e "* soft nofile 262144\n* hard nofile 262144" > /etc/security/limits.d/99-nofile.conf - wget -q -O - https://download.jetbrains.com/cpp/CLion-2017.2.1.tar.gz | tar -C /opt -zx - ln -sf /opt/clion-2017.2.1/bin/clion.sh /usr/bin/clion + wget -q -O - https://download.jetbrains.com/cpp/CLion-2017.2.2.tar.gz | tar -C /opt -zx + ln -sf /opt/clion-2017.2.2/bin/clion.sh /usr/bin/clion SHELL end diff --git a/vdslib/src/vespa/vdslib/container/mutabledocumentlist.cpp b/vdslib/src/vespa/vdslib/container/mutabledocumentlist.cpp index ce55c3a87f8..940d142db2f 100644 --- a/vdslib/src/vespa/vdslib/container/mutabledocumentlist.cpp +++ b/vdslib/src/vespa/vdslib/container/mutabledocumentlist.cpp @@ -5,17 +5,17 @@ #include <vespa/document/datatype/documenttype.h> #include <vespa/vespalib/objects/nbostream.h> +using vespalib::compression::CompressionConfig; using vespalib::nbostream; namespace vdslib { -MutableDocumentList::MutableDocumentList(const document::DocumentTypeRepo::SP & repo, char* buffer, uint32_t bufferSize, bool keepexisting) +MutableDocumentList::MutableDocumentList(const document::DocumentTypeRepo::SP & repo, char* buffer, + uint32_t bufferSize, bool keepexisting) : DocumentList(repo, buffer, bufferSize, keepexisting) { } -MutableDocumentList::MutableDocumentList(const DocumentList& source, - char* buffer, - uint32_t bufferSize) +MutableDocumentList::MutableDocumentList(const DocumentList& source, char* buffer, uint32_t bufferSize) : DocumentList(source, buffer, bufferSize) { } @@ -41,8 +41,7 @@ MutableDocumentList::addOperationList(const OperationList& opl) } bool -MutableDocumentList::addPut(const document::Document& doc, Timestamp ts, - bool addBody) +MutableDocumentList::addPut(const document::Document& doc, Timestamp ts, bool addBody) { uint32_t freePos = _freePtr - _buffer; @@ -69,9 +68,7 @@ MutableDocumentList::addPut(const document::Document& doc, Timestamp ts, entry.bodyLen = bodySize; entry.flags = 0; - if (doc.getType().getFieldsType().getCompressionConfig().type - != document::CompressionConfig::NONE) - { + if (doc.getType().getFieldsType().getCompressionConfig().type != CompressionConfig::NONE) { entry.flags |= MetaEntry::COMPRESSED; } @@ -91,8 +88,7 @@ MutableDocumentList::addPut(const document::Document& doc, Timestamp ts, } bool -MutableDocumentList::addUpdate(const document::DocumentUpdate& update, - Timestamp ts) +MutableDocumentList::addUpdate(const document::DocumentUpdate& update, Timestamp ts) { vespalib::nbostream os; update.serialize42(os); diff --git a/vespa-hadoop/pom.xml b/vespa-hadoop/pom.xml index ea461e9783f..b9fbd589525 100644 --- a/vespa-hadoop/pom.xml +++ b/vespa-hadoop/pom.xml @@ -16,7 +16,7 @@ <properties> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> - <hadoop.version>2.7.3</hadoop.version> + <hadoop.version>2.8.0</hadoop.version> <pig.version>0.14.0</pig.version> </properties> diff --git a/vespaclient-core/src/main/resources/configdefinitions/feeder.def b/vespaclient-core/src/main/resources/configdefinitions/feeder.def index f6997d005f6..a376a4807a7 100644 --- a/vespaclient-core/src/main/resources/configdefinitions/feeder.def +++ b/vespaclient-core/src/main/resources/configdefinitions/feeder.def @@ -1,5 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -version=16 namespace=vespaclient.config ## Whether or not to abort if there are document-related errors. diff --git a/vespaclient/src/vespa/vespaclient/vesparoute/mynetwork.cpp b/vespaclient/src/vespa/vespaclient/vesparoute/mynetwork.cpp index ec7c1c623c9..d9d8d0c4056 100644 --- a/vespaclient/src/vespa/vespaclient/vesparoute/mynetwork.cpp +++ b/vespaclient/src/vespa/vespaclient/vesparoute/mynetwork.cpp @@ -3,6 +3,8 @@ #include "mynetwork.h" #include <vespa/messagebus/emptyreply.h> #include <vespa/messagebus/sendproxy.h> +#include <vespa/messagebus/network/oosmanager.h> + class MyServiceAddress : public mbus::IServiceAddress { private: diff --git a/vespajlib/src/main/java/com/yahoo/data/access/Inspector.java b/vespajlib/src/main/java/com/yahoo/data/access/Inspector.java index 5bf8016aad3..bb4487e68d6 100644 --- a/vespajlib/src/main/java/com/yahoo/data/access/Inspector.java +++ b/vespajlib/src/main/java/com/yahoo/data/access/Inspector.java @@ -12,7 +12,9 @@ import java.util.Map; * Instrospection methods are available, but you can also use accessors * with a default value if you expect a certain type and just want your * default value if some field doesn't exist or was of the wrong type. - **/ + * + * @author havardpe + */ public interface Inspector extends Inspectable { /** @@ -20,55 +22,55 @@ public interface Inspector extends Inspectable { * If you try to access a field or array entry that does not exist, * you will get an invalid Inspector returned. */ - public boolean valid(); + boolean valid(); /** Get the type of an inspector */ - public Type type(); + Type type(); /** Get the number of entries in an ARRAY (always returns 0 for non-arrays) */ - public int entryCount(); + int entryCount(); /** Get the number of fields in an OBJECT (always returns 0 for non-objects) */ - public int fieldCount(); + int fieldCount(); /** Access the inspector's value if it's a BOOLEAN; otherwise throws exception */ - public boolean asBool(); + boolean asBool(); /** Access the inspector's value if it's a LONG (or DOUBLE); otherwise throws exception */ - public long asLong(); + long asLong(); /** Access the inspector's value if it's a DOUBLE (or LONG); otherwise throws exception */ - public double asDouble(); + double asDouble(); /** Access the inspector's value if it's a STRING; otherwise throws exception */ - public String asString(); + String asString(); /** * Access the inspector's value (in utf-8 representation) if it's * a STRING; otherwise throws exception - **/ - public byte[] asUtf8(); + */ + byte[] asUtf8(); /** Access the inspector's value if it's DATA; otherwise throws exception */ - public byte[] asData(); + byte[] asData(); /** Get the inspector's value (or the supplied default), never throws */ - public boolean asBool(boolean defaultValue); + boolean asBool(boolean defaultValue); /** Get the inspector's value (or the supplied default), never throws */ - public long asLong(long defaultValue); + long asLong(long defaultValue); /** Get the inspector's value (or the supplied default), never throws */ - public double asDouble(double defaultValue); + double asDouble(double defaultValue); /** Get the inspector's value (or the supplied default), never throws */ - public String asString(String defaultValue); + String asString(String defaultValue); /** Get the inspector's value (or the supplied default), never throws */ - public byte[] asUtf8(byte[] defaultValue); + byte[] asUtf8(byte[] defaultValue); /** Get the inspector's value (or the supplied default), never throws */ - public byte[] asData(byte[] defaultValue); + byte[] asData(byte[] defaultValue); /** * Traverse an array value, performing callbacks for each entry. @@ -76,10 +78,11 @@ public interface Inspector extends Inspectable { * If the current Inspector is connected to an array value, * perform callbacks to the given traverser for each entry * contained in the array. Otherwise a no-op. - * @param at traverser callback object. - **/ + * + * @param at traverser callback object + */ @SuppressWarnings("overloads") - public void traverse(ArrayTraverser at); + void traverse(ArrayTraverser at); /** * Traverse an object value, performing callbacks for each field. @@ -87,10 +90,11 @@ public interface Inspector extends Inspectable { * If the current Inspector is connected to an object value, * perform callbacks to the given traverser for each field * contained in the object. Otherwise a no-op. - * @param ot traverser callback object. - **/ + * + * @param ot traverser callback object + */ @SuppressWarnings("overloads") - public void traverse(ObjectTraverser ot); + void traverse(ObjectTraverser ot); /** * Access an array entry. @@ -98,10 +102,11 @@ public interface Inspector extends Inspectable { * If the current Inspector doesn't connect to an array value, * or the given array index is out of bounds, the returned * Inspector will be invalid. - * @param idx array index. - * @return a new Inspector for the entry value. - **/ - public Inspector entry(int idx); + * + * @param idx array index + * @return a new Inspector for the entry value + */ + Inspector entry(int idx); /** * Access an field in an object. @@ -109,20 +114,22 @@ public interface Inspector extends Inspectable { * If the current Inspector doesn't connect to an object value, or * the object value does not contain a field with the given symbol * name, the returned Inspector will be invalid. - * @param name symbol name. - * @return a new Inspector for the field value. - **/ - public Inspector field(String name); + * + * @param name symbol name + * @return a new Inspector for the field value + */ + Inspector field(String name); /** * Convert an array to an iterable list. Other types will just * return an empty list. - **/ - public Iterable<Inspector> entries(); + */ + Iterable<Inspector> entries(); /** * Convert an object to an iterable list of (name, value) pairs. * Other types will just return an empty list. - **/ - public Iterable<Map.Entry<String,Inspector>> fields(); + */ + Iterable<Map.Entry<String,Inspector>> fields(); + } diff --git a/vespajlib/src/main/java/com/yahoo/lang/SettableOptional.java b/vespajlib/src/main/java/com/yahoo/lang/SettableOptional.java new file mode 100644 index 00000000000..00ff06b8f01 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/lang/SettableOptional.java @@ -0,0 +1,41 @@ +package com.yahoo.lang; + +import java.util.NoSuchElementException; +import java.util.Optional; + +/** + * An optional which contains a settable value + * + * @author bratseth + */ +public final class SettableOptional<T> { + + private T value = null; + + /** Creates a new empty settable optional */ + public SettableOptional() {} + + /** Creates a new settable optional with the given value */ + public SettableOptional(T value) { this.value = value; } + + public boolean isPresent() { + return value != null; + } + + public T get() { + if (value == null) + throw new NoSuchElementException("No value present"); + return value; + } + + public void set(T value) { + this.value = value; + } + + public Optional<T> asOptional() { + if (value == null) return Optional.empty(); + return Optional.of(value); + } + +} + diff --git a/vespajlib/src/main/java/com/yahoo/slime/Inspector.java b/vespajlib/src/main/java/com/yahoo/slime/Inspector.java index 89e7b3383fc..99489faca47 100644 --- a/vespajlib/src/main/java/com/yahoo/slime/Inspector.java +++ b/vespajlib/src/main/java/com/yahoo/slime/Inspector.java @@ -9,59 +9,61 @@ package com.yahoo.slime; * current Inspector is invalid or the type doesn't match your * accessor type. If you want to do something exceptional instead * when the types don't match, you must check using type() first. - **/ + * + * @author havardpe + */ public interface Inspector { /** check if this inspector is valid */ - public boolean valid(); + boolean valid(); /** return an enum describing value type */ - public Type type(); + Type type(); /** * Check how many entries or fields are contained in the current value. * Useful for arrays and objects; anything else always returns 0. * @return number of entries/fields contained. - **/ - public int children(); + */ + int children(); /** * Check how many entries are contained in the current value. * Useful for arrays; anything else always returns 0. * @return number of entries contained. - **/ - public int entries(); + */ + int entries(); /** * Check how many fields are contained in the current value. * Useful for objects; anything else always returns 0. * @return number of fields contained. - **/ - public int fields(); + */ + int fields(); /** the current value (for booleans); default: false */ - public boolean asBool(); + boolean asBool(); /** the current value (for integers); default: 0 */ - public long asLong(); + long asLong(); /** the current value (for floating-point values); default: 0.0 */ - public double asDouble(); + double asDouble(); /** the current value (for string values); default: empty string */ - public String asString(); + String asString(); /** the current value encoded into UTF-8 (for string values); default: empty array */ - public byte[] asUtf8(); + byte[] asUtf8(); /** the current value (for data values); default: empty array */ - public byte[] asData(); + byte[] asData(); /** * Use the visitor pattern to resolve the underlying type of this value. * @param v the visitor - **/ - public void accept(Visitor v); + */ + void accept(Visitor v); /** * Traverse an array value, performing callbacks for each entry. @@ -70,9 +72,9 @@ public interface Inspector { * perform callbacks to the given traverser for each entry * contained in the array. * @param at traverser callback object. - **/ + */ @SuppressWarnings("overloads") - public void traverse(ArrayTraverser at); + void traverse(ArrayTraverser at); /** * Traverse an object value, performing callbacks for each field. @@ -81,9 +83,9 @@ public interface Inspector { * perform callbacks to the given traverser for each field * contained in the object. * @param ot traverser callback object. - **/ + */ @SuppressWarnings("overloads") - public void traverse(ObjectSymbolTraverser ot); + void traverse(ObjectSymbolTraverser ot); /** * Traverse an object value, performing callbacks for each field. @@ -92,9 +94,9 @@ public interface Inspector { * perform callbacks to the given traverser for each field * contained in the object. * @param ot traverser callback object. - **/ + */ @SuppressWarnings("overloads") - public void traverse(ObjectTraverser ot); + void traverse(ObjectTraverser ot); /** * Access an array entry. @@ -104,8 +106,8 @@ public interface Inspector { * Inspector will be invalid. * @param idx array index. * @return a new Inspector for the entry value. - **/ - public Inspector entry(int idx); + */ + Inspector entry(int idx); /** * Access an field in an object by symbol id. @@ -115,8 +117,8 @@ public interface Inspector { * id, the returned Inspector will be invalid. * @param sym symbol id. * @return a new Inspector for the field value. - **/ - public Inspector field(int sym); + */ + Inspector field(int sym); /** * Access an field in an object by symbol name. @@ -126,6 +128,7 @@ public interface Inspector { * name, the returned Inspector will be invalid. * @param name symbol name. * @return a new Inspector for the field value. - **/ - public Inspector field(String name); + */ + Inspector field(String name); + } diff --git a/vespajlib/src/main/java/com/yahoo/slime/Slime.java b/vespajlib/src/main/java/com/yahoo/slime/Slime.java index b601d0b72ff..84f193caa4d 100644 --- a/vespajlib/src/main/java/com/yahoo/slime/Slime.java +++ b/vespajlib/src/main/java/com/yahoo/slime/Slime.java @@ -5,26 +5,29 @@ package com.yahoo.slime; * Top-level value class that contains one Value data object and a * symbol table (shared between all directly or indirectly contained * ObjectValue data objects). + * + * @author havardpe **/ -public final class Slime -{ +public final class Slime { + private final SymbolTable names = new SymbolTable(); private Value root = NixValue.instance(); /** * Construct an empty Slime with an empty top-level value. - **/ + */ public Slime() {} - /** return count of names in the symbol table. */ + /** Returns a count of names in the symbol table. */ public int symbols() { return names.symbols(); } /** * Return the symbol name associated with an id. + * * @param symbol the id, must be in range [0, symbols()-1] - **/ + */ public String inspect(int symbol) { return names.inspect(symbol); } @@ -32,9 +35,10 @@ public final class Slime /** * Add a name to the symbol table; if the name is already * in the symbol table just returns the id it already had. + * * @param name the name to insert * @return the id now associated with the name - **/ + */ public int insert(String name) { return names.insert(name); } @@ -43,7 +47,7 @@ public final class Slime * Find the id associated with a symbol name; if the * name was not in the symbol table returns the * constant Integer.MAX_VALUE instead. - **/ + */ public int lookup(String name) { return names.lookup(name); } @@ -53,7 +57,7 @@ public final class Slime /** * Create a new empty value and make it the new top-level data object. - **/ + */ public Cursor setNix() { root = NixValue.instance(); return root; @@ -61,8 +65,9 @@ public final class Slime /** * Create a new boolean value and make it the new top-level data object. + * * @param bit the actual boolean value for the new value - **/ + */ public Cursor setBool(boolean bit) { root = BoolValue.instance(bit); return root; @@ -70,8 +75,9 @@ public final class Slime /** * Create a new double value and make it the new top-level data object. + * * @param l the actual long value for the new value - **/ + */ public Cursor setLong(long l) { root = new LongValue(l); return root; @@ -79,8 +85,9 @@ public final class Slime /** * Create a new double value and make it the new top-level data object. + * * @param d the actual double value for the new value - **/ + */ public Cursor setDouble(double d) { root = new DoubleValue(d); return root; @@ -88,8 +95,9 @@ public final class Slime /** * Create a new string value and make it the new top-level data object. + * * @param str the actual string for the new value - **/ + */ public Cursor setString(String str) { root = StringValue.create(str); return root; @@ -97,8 +105,9 @@ public final class Slime /** * Create a new string value and make it the new top-level data object. + * * @param utf8 the actual string (encoded as UTF-8 data) for the new value - **/ + */ public Cursor setString(byte[] utf8) { root = Utf8Value.create(utf8); return root; @@ -106,8 +115,9 @@ public final class Slime /** * Create a new data value and make it the new top-level data object. + * * @param data the actual data to be put into the new value. - **/ + */ public Cursor setData(byte[] data) { root = DataValue.create(data); return root; @@ -115,7 +125,7 @@ public final class Slime /** * Create a new array value and make it the new top-level data object. - **/ + */ public Cursor setArray() { root = new ArrayValue(names); return root; @@ -123,7 +133,7 @@ public final class Slime /** * Create a new object value and make it the new top-level data object. - **/ + */ public Cursor setObject() { root = new ObjectValue(names); return root; @@ -133,7 +143,7 @@ public final class Slime * Take the current top-level data object and make it a field in a * new ObjectValue with the given symbol id as field id; the new * ObjectValue will also become the new top-level data object. - **/ + */ public Cursor wrap(int sym) { root = new ObjectValue(names, sym, root); return root; @@ -143,8 +153,9 @@ public final class Slime * Take the current top-level data object and make it a field in a * new ObjectValue with the given symbol name as field name; the new * ObjectValue will also become the new top-level data object. - **/ + */ public Cursor wrap(String name) { return wrap(names.insert(name)); } + } diff --git a/vespalib/CMakeLists.txt b/vespalib/CMakeLists.txt index 112f67f3a70..b6f8775b64b 100644 --- a/vespalib/CMakeLists.txt +++ b/vespalib/CMakeLists.txt @@ -6,6 +6,7 @@ vespa_define_module( EXTERNAL_DEPENDS lz4 + zstd APPS src/apps/make_fixture_macros @@ -22,6 +23,8 @@ vespa_define_module( src/tests/closure src/tests/component src/tests/compress + src/tests/compression + src/tests/data/databuffer src/tests/data/input_reader src/tests/data/lz4_encode_decode src/tests/data/memory_input @@ -65,6 +68,7 @@ vespa_define_module( src/tests/simple_thread_bundle src/tests/slaveproc src/tests/slime + src/tests/slime/external_data_value src/tests/slime/summary-feature-benchmark src/tests/stash src/tests/stllike diff --git a/vespalib/src/tests/compression/.gitignore b/vespalib/src/tests/compression/.gitignore new file mode 100644 index 00000000000..60ffd040a47 --- /dev/null +++ b/vespalib/src/tests/compression/.gitignore @@ -0,0 +1,4 @@ +*_test +.depend +Makefile +vespalib_compression_test_app diff --git a/vespalib/src/tests/compression/CMakeLists.txt b/vespalib/src/tests/compression/CMakeLists.txt new file mode 100644 index 00000000000..c5738733218 --- /dev/null +++ b/vespalib/src/tests/compression/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_compression_test_app TEST + SOURCES + compression_test.cpp + DEPENDS + vespalib +) +vespa_add_test(NAME vespalib_compression_test_app COMMAND vespalib_compression_test_app) diff --git a/document/src/tests/serialization/compression_test.cpp b/vespalib/src/tests/compression/compression_test.cpp index 6574b4ac34c..01cfe0af223 100644 --- a/document/src/tests/serialization/compression_test.cpp +++ b/vespalib/src/tests/compression/compression_test.cpp @@ -2,15 +2,14 @@ #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/stllike/string.h> -#include <vespa/document/util/compressor.h> +#include <vespa/vespalib/util/compressor.h> #include <vespa/vespalib/data/databuffer.h> #include <vespa/log/log.h> LOG_SETUP("compression_test"); -using namespace document; -using namespace document::compression; using namespace vespalib; +using namespace vespalib::compression; static vespalib::string _G_compressableText("AAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEEEEE" "AAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEEEEE" diff --git a/vespalib/src/tests/data/databuffer/.gitignore b/vespalib/src/tests/data/databuffer/.gitignore new file mode 100644 index 00000000000..f144796c66a --- /dev/null +++ b/vespalib/src/tests/data/databuffer/.gitignore @@ -0,0 +1 @@ +vespalib_data_databuffer_test_app diff --git a/vespalib/src/tests/data/databuffer/CMakeLists.txt b/vespalib/src/tests/data/databuffer/CMakeLists.txt new file mode 100644 index 00000000000..f1c6c7c1862 --- /dev/null +++ b/vespalib/src/tests/data/databuffer/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_data_databuffer_test_app TEST + SOURCES + databuffer_test.cpp + DEPENDS + vespalib +) +vespa_add_test(NAME vespalib_data_databuffer_test_app COMMAND vespalib_data_databuffer_test_app) diff --git a/staging_vespalib/src/tests/databuffer/databuffer_test.cpp b/vespalib/src/tests/data/databuffer/databuffer_test.cpp index f440ca1e15c..f440ca1e15c 100644 --- a/staging_vespalib/src/tests/databuffer/databuffer_test.cpp +++ b/vespalib/src/tests/data/databuffer/databuffer_test.cpp diff --git a/vespalib/src/tests/slime/external_data_value/CMakeLists.txt b/vespalib/src/tests/slime/external_data_value/CMakeLists.txt new file mode 100644 index 00000000000..df4cb8a23c3 --- /dev/null +++ b/vespalib/src/tests/slime/external_data_value/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_external_data_value_test_app TEST + SOURCES + external_data_value_test.cpp + DEPENDS + vespalib +) +vespa_add_test(NAME vespalib_external_data_value_test_app COMMAND vespalib_external_data_value_test_app) diff --git a/vespalib/src/tests/slime/external_data_value/external_data_value_test.cpp b/vespalib/src/tests/slime/external_data_value/external_data_value_test.cpp new file mode 100644 index 00000000000..acf937c840d --- /dev/null +++ b/vespalib/src/tests/slime/external_data_value/external_data_value_test.cpp @@ -0,0 +1,86 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/data/slime/slime.h> + +using namespace vespalib::slime::convenience; +using vespalib::slime::ExternalMemory; + +struct MyMem : ExternalMemory { + const std::vector<char> space; + MyMem(Memory memory) + : space(memory.data, memory.data + memory.size) {} + Memory get() const override { + return Memory(&space[0], space.size()); + } + static UP create(Memory memory) { + return std::make_unique<MyMem>(memory); + } +}; + +void verify_data(const Inspector &pos, Memory expect) { + EXPECT_TRUE(pos.valid()); + EXPECT_EQUAL(vespalib::slime::DATA::ID, pos.type().getId()); + EXPECT_EQUAL(pos.asString(), Memory()); + EXPECT_EQUAL(pos.asData(), expect); +} + +TEST("require that external memory can be used for data values") { + Slime slime; + TEST_DO(verify_data(slime.setData(MyMem::create("foo")), Memory("foo"))); + TEST_DO(verify_data(slime.get(), Memory("foo"))); +} + +TEST("require that nullptr external memory gives empty data value") { + Slime slime; + TEST_DO(verify_data(slime.setData(ExternalMemory::UP(nullptr)), Memory(""))); + TEST_DO(verify_data(slime.get(), Memory(""))); +} + +TEST("require that external memory can be used with array data values") { + Slime slime; + TEST_DO(verify_data(slime.setArray().addData(MyMem::create("foo")), Memory("foo"))); + TEST_DO(verify_data(slime.get()[0], Memory("foo"))); +} + +TEST("require that external memory can be used with object data values (name)") { + Slime slime; + TEST_DO(verify_data(slime.setObject().setData("field", MyMem::create("foo")), Memory("foo"))); + TEST_DO(verify_data(slime.get()["field"], Memory("foo"))); +} + +TEST("require that external memory can be used with object data values (symbol)") { + Slime slime; + TEST_DO(verify_data(slime.setObject().setData(Symbol(5), MyMem::create("foo")), Memory("foo"))); + TEST_DO(verify_data(slime.get()[Symbol(5)], Memory("foo"))); +} + +TEST("require that external memory can be used with slime inserter") { + Slime slime; + SlimeInserter inserter(slime); + TEST_DO(verify_data(inserter.insertData(MyMem::create("foo")), Memory("foo"))); + TEST_DO(verify_data(slime.get(), Memory("foo"))); +} + +TEST("require that external memory can be used with array inserter") { + Slime slime; + ArrayInserter inserter(slime.setArray()); + TEST_DO(verify_data(inserter.insertData(MyMem::create("foo")), Memory("foo"))); + TEST_DO(verify_data(slime.get()[0], Memory("foo"))); +} + +TEST("require that external memory can be used with object inserter") { + Slime slime; + ObjectInserter inserter(slime.setObject(), "field"); + TEST_DO(verify_data(inserter.insertData(MyMem::create("foo")), Memory("foo"))); + TEST_DO(verify_data(slime.get()["field"], Memory("foo"))); +} + +TEST("require that external memory can be used with object symbol inserter") { + Slime slime; + ObjectSymbolInserter inserter(slime.setObject(), Symbol(5)); + TEST_DO(verify_data(inserter.insertData(MyMem::create("foo")), Memory("foo"))); + TEST_DO(verify_data(slime.get()[Symbol(5)], Memory("foo"))); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/vespa/vespalib/component/version.cpp b/vespalib/src/vespa/vespalib/component/version.cpp index 3fab6d6f130..af38a675de8 100644 --- a/vespalib/src/vespa/vespalib/component/version.cpp +++ b/vespalib/src/vespa/vespalib/component/version.cpp @@ -7,7 +7,6 @@ namespace vespalib { - Version::Version(int major, int minor, int micro, const string & qualifier) : _major(major), _minor(minor), @@ -89,7 +88,7 @@ Version::Version(const string & versionString) _qualifier(), _stringValue(versionString) { - if (versionString != "") { + if ( ! versionString.empty()) { stringref r(versionString.c_str(), versionString.size()); stringref::size_type dot(r.find('.')); stringref majorS(r.substr(0, dot)); diff --git a/vespalib/src/vespa/vespalib/data/CMakeLists.txt b/vespalib/src/vespa/vespalib/data/CMakeLists.txt index 29c8055a0c0..3a94e00ae33 100644 --- a/vespalib/src/vespa/vespalib/data/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/data/CMakeLists.txt @@ -1,6 +1,7 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_library(vespalib_vespalib_data OBJECT SOURCES + databuffer.cpp input.cpp input_reader.cpp lz4_input_decoder.cpp diff --git a/staging_vespalib/src/vespa/vespalib/data/databuffer.cpp b/vespalib/src/vespa/vespalib/data/databuffer.cpp index 529e475e987..5558a371836 100644 --- a/staging_vespalib/src/vespa/vespalib/data/databuffer.cpp +++ b/vespalib/src/vespa/vespalib/data/databuffer.cpp @@ -33,6 +33,7 @@ DataBuffer::DataBuffer(size_t len, size_t alignment, const Alloc & initial) } } +DataBuffer::~DataBuffer() = default; void DataBuffer::moveFreeToData(size_t len) diff --git a/staging_vespalib/src/vespa/vespalib/data/databuffer.h b/vespalib/src/vespa/vespalib/data/databuffer.h index f7707a3ea56..28524f373b2 100644 --- a/staging_vespalib/src/vespa/vespalib/data/databuffer.h +++ b/vespalib/src/vespa/vespalib/data/databuffer.h @@ -81,6 +81,8 @@ public: _buffer(Alloc::alloc(0)) { } + ~DataBuffer(); + /** * @return a pointer to the dead part of this buffer. **/ diff --git a/vespalib/src/vespa/vespalib/data/memory.h b/vespalib/src/vespa/vespalib/data/memory.h index 74024549a0b..eae7c8d2f23 100644 --- a/vespalib/src/vespa/vespalib/data/memory.h +++ b/vespalib/src/vespa/vespalib/data/memory.h @@ -25,6 +25,7 @@ struct Memory Memory(const vespalib::stringref &str_ref) : data(str_ref.data()), size(str_ref.size()) {} vespalib::string make_string() const; + vespalib::stringref make_stringref() const { return stringref(data, size); } bool operator == (const Memory &rhs) const { return ((size == rhs.size) && ((data == rhs.data) || diff --git a/vespalib/src/vespa/vespalib/data/slime/CMakeLists.txt b/vespalib/src/vespa/vespalib/data/slime/CMakeLists.txt index c5aa9b595d2..46a8d410878 100644 --- a/vespalib/src/vespa/vespalib/data/slime/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/data/slime/CMakeLists.txt @@ -9,6 +9,9 @@ vespa_add_library(vespalib_vespalib_data_slime OBJECT convenience.cpp cursor.cpp empty_value_factory.cpp + external_data_value.cpp + external_data_value_factory.cpp + external_memory.cpp inject.cpp inserter.cpp inspector.cpp diff --git a/vespalib/src/vespa/vespalib/data/slime/cursor.h b/vespalib/src/vespa/vespalib/data/slime/cursor.h index 4242936a483..6815ad3ba83 100644 --- a/vespalib/src/vespa/vespalib/data/slime/cursor.h +++ b/vespalib/src/vespa/vespalib/data/slime/cursor.h @@ -3,6 +3,7 @@ #pragma once #include "inspector.h" +#include "external_memory.h" namespace vespalib { namespace slime { @@ -18,6 +19,7 @@ struct Cursor : public Inspector { virtual Cursor &addDouble(double d) = 0; virtual Cursor &addString(Memory str) = 0; virtual Cursor &addData(Memory data) = 0; + virtual Cursor &addData(ExternalMemory::UP data) = 0; virtual Cursor &addArray() = 0; virtual Cursor &addObject() = 0; @@ -27,6 +29,7 @@ struct Cursor : public Inspector { virtual Cursor &setDouble(Symbol sym, double d) = 0; virtual Cursor &setString(Symbol sym, Memory str) = 0; virtual Cursor &setData(Symbol sym, Memory data) = 0; + virtual Cursor &setData(Symbol sym, ExternalMemory::UP data) = 0; virtual Cursor &setArray(Symbol sym) = 0; virtual Cursor &setObject(Symbol sym) = 0; @@ -35,7 +38,8 @@ struct Cursor : public Inspector { virtual Cursor &setLong(Memory name, int64_t l) = 0; virtual Cursor &setDouble(Memory name, double d) = 0; virtual Cursor &setString(Memory name, Memory str) = 0; - virtual Cursor &setData(Memory name, Memory str) = 0; + virtual Cursor &setData(Memory name, Memory data) = 0; + virtual Cursor &setData(Memory name, ExternalMemory::UP data) = 0; virtual Cursor &setArray(Memory name) = 0; virtual Cursor &setObject(Memory name) = 0; diff --git a/vespalib/src/vespa/vespalib/data/slime/external_data_value.cpp b/vespalib/src/vespa/vespalib/data/slime/external_data_value.cpp new file mode 100644 index 00000000000..1004f8aab24 --- /dev/null +++ b/vespalib/src/vespa/vespalib/data/slime/external_data_value.cpp @@ -0,0 +1,7 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "external_data_value.h" + +namespace vespalib::slime { + +} // namespace vespalib::slime diff --git a/vespalib/src/vespa/vespalib/data/slime/external_data_value.h b/vespalib/src/vespa/vespalib/data/slime/external_data_value.h new file mode 100644 index 00000000000..3acfb1350bd --- /dev/null +++ b/vespalib/src/vespa/vespalib/data/slime/external_data_value.h @@ -0,0 +1,23 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "value.h" +#include "external_memory.h" + +namespace vespalib::slime { + +/** + * A data value backed by external memory. + **/ +class ExternalDataValue : public Value +{ +private: + ExternalMemory::UP _value; +public: + ExternalDataValue(ExternalMemory::UP data) : _value(std::move(data)) {} + Memory asData() const override { return _value->get(); } + Type type() const override { return DATA::instance; } +}; + +} // namespace vespalib::slime diff --git a/vespalib/src/vespa/vespalib/data/slime/external_data_value_factory.cpp b/vespalib/src/vespa/vespalib/data/slime/external_data_value_factory.cpp new file mode 100644 index 00000000000..592ddcdb519 --- /dev/null +++ b/vespalib/src/vespa/vespalib/data/slime/external_data_value_factory.cpp @@ -0,0 +1,18 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "external_data_value_factory.h" +#include "external_data_value.h" +#include "basic_value.h" + +namespace vespalib::slime { + +Value * +ExternalDataValueFactory::create(Stash &stash) const +{ + if (!input) { + return &stash.create<BasicDataValue>(Memory(), stash); + } + return &stash.create<ExternalDataValue>(std::move(input)); +} + +} // namespace vespalib::slime diff --git a/vespalib/src/vespa/vespalib/data/slime/external_data_value_factory.h b/vespalib/src/vespa/vespalib/data/slime/external_data_value_factory.h new file mode 100644 index 00000000000..be85f28aebe --- /dev/null +++ b/vespalib/src/vespa/vespalib/data/slime/external_data_value_factory.h @@ -0,0 +1,20 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "value_factory.h" +#include "external_memory.h" +#include <vespa/vespalib/util/stash.h> + +namespace vespalib::slime { + +/** + * Value factory for data values using external memory. + **/ +struct ExternalDataValueFactory : public ValueFactory { + mutable ExternalMemory::UP input; + ExternalDataValueFactory(ExternalMemory::UP in) : input(std::move(in)) {} + Value *create(Stash &stash) const override; +}; + +} // namespace vespalib::slime diff --git a/vespalib/src/vespa/vespalib/data/slime/external_memory.cpp b/vespalib/src/vespa/vespalib/data/slime/external_memory.cpp new file mode 100644 index 00000000000..ebbe833098a --- /dev/null +++ b/vespalib/src/vespa/vespalib/data/slime/external_memory.cpp @@ -0,0 +1,11 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "external_memory.h" + +namespace vespalib::slime { + +ExternalMemory::~ExternalMemory() +{ +} + +} // namespace vespalib::slime diff --git a/vespalib/src/vespa/vespalib/data/slime/external_memory.h b/vespalib/src/vespa/vespalib/data/slime/external_memory.h new file mode 100644 index 00000000000..a1d4510d0c8 --- /dev/null +++ b/vespalib/src/vespa/vespalib/data/slime/external_memory.h @@ -0,0 +1,22 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <memory> +#include <vespa/vespalib/data/memory.h> + +namespace vespalib::slime { + +/** + * Interface used to access external memory. External memory does not + * need to be copied when added to a Slime object. The Memory obtained + * by calling the get function must be valid until the object + * implementing this interface is destructed. + **/ +struct ExternalMemory { + using UP = std::unique_ptr<ExternalMemory>; + virtual Memory get() const = 0; + virtual ~ExternalMemory(); +}; + +} // namespace vespalib::slime diff --git a/vespalib/src/vespa/vespalib/data/slime/inserter.cpp b/vespalib/src/vespa/vespalib/data/slime/inserter.cpp index 81b8ffe334c..f9d80e74e6f 100644 --- a/vespalib/src/vespa/vespalib/data/slime/inserter.cpp +++ b/vespalib/src/vespa/vespalib/data/slime/inserter.cpp @@ -6,12 +6,15 @@ namespace vespalib { namespace slime { +using ExtMemUP = ExternalMemory::UP; + Cursor &SlimeInserter::insertNix() const { return slime.setNix(); } Cursor &SlimeInserter::insertBool(bool value) const { return slime.setBool(value); } Cursor &SlimeInserter::insertLong(int64_t value) const { return slime.setLong(value); } Cursor &SlimeInserter::insertDouble(double value) const { return slime.setDouble(value); } Cursor &SlimeInserter::insertString(Memory value) const { return slime.setString(value); } Cursor &SlimeInserter::insertData(Memory value) const { return slime.setData(value); } +Cursor &SlimeInserter::insertData(ExtMemUP value) const { return slime.setData(std::move(value)); } Cursor &SlimeInserter::insertArray() const { return slime.setArray(); } Cursor &SlimeInserter::insertObject() const { return slime.setObject(); } @@ -21,6 +24,7 @@ Cursor &ArrayInserter::insertLong(int64_t value) const { return cursor.addLong( Cursor &ArrayInserter::insertDouble(double value) const { return cursor.addDouble(value); } Cursor &ArrayInserter::insertString(Memory value) const { return cursor.addString(value); } Cursor &ArrayInserter::insertData(Memory value) const { return cursor.addData(value); } +Cursor &ArrayInserter::insertData(ExtMemUP value) const { return cursor.addData(std::move(value)); } Cursor &ArrayInserter::insertArray() const { return cursor.addArray(); } Cursor &ArrayInserter::insertObject() const { return cursor.addObject(); } @@ -30,6 +34,7 @@ Cursor &ObjectSymbolInserter::insertLong(int64_t value) const { return cursor.s Cursor &ObjectSymbolInserter::insertDouble(double value) const { return cursor.setDouble(symbol, value); } Cursor &ObjectSymbolInserter::insertString(Memory value) const { return cursor.setString(symbol, value); } Cursor &ObjectSymbolInserter::insertData(Memory value) const { return cursor.setData(symbol, value); } +Cursor &ObjectSymbolInserter::insertData(ExtMemUP value) const { return cursor.setData(symbol, std::move(value)); } Cursor &ObjectSymbolInserter::insertArray() const { return cursor.setArray(symbol); } Cursor &ObjectSymbolInserter::insertObject() const { return cursor.setObject(symbol); } @@ -39,6 +44,7 @@ Cursor &ObjectInserter::insertLong(int64_t value) const { return cursor.setLong Cursor &ObjectInserter::insertDouble(double value) const { return cursor.setDouble(name, value); } Cursor &ObjectInserter::insertString(Memory value) const { return cursor.setString(name, value); } Cursor &ObjectInserter::insertData(Memory value) const { return cursor.setData(name, value); } +Cursor &ObjectInserter::insertData(ExtMemUP value) const { return cursor.setData(name, std::move(value)); } Cursor &ObjectInserter::insertArray() const { return cursor.setArray(name); } Cursor &ObjectInserter::insertObject() const { return cursor.setObject(name); } diff --git a/vespalib/src/vespa/vespalib/data/slime/inserter.h b/vespalib/src/vespa/vespalib/data/slime/inserter.h index b8762ed3794..dff37183ac7 100644 --- a/vespalib/src/vespa/vespalib/data/slime/inserter.h +++ b/vespalib/src/vespa/vespalib/data/slime/inserter.h @@ -5,6 +5,7 @@ #include "type.h" #include <vespa/vespalib/data/memory.h> #include "symbol.h" +#include "external_memory.h" namespace vespalib { @@ -27,6 +28,7 @@ struct Inserter { virtual Cursor &insertDouble(double value) const = 0; virtual Cursor &insertString(Memory value) const = 0; virtual Cursor &insertData(Memory value) const = 0; + virtual Cursor &insertData(ExternalMemory::UP value) const = 0; virtual Cursor &insertArray() const = 0; virtual Cursor &insertObject() const = 0; virtual ~Inserter() {} @@ -44,6 +46,7 @@ struct SlimeInserter : Inserter { Cursor &insertDouble(double value) const override; Cursor &insertString(Memory value) const override; Cursor &insertData(Memory value) const override; + Cursor &insertData(ExternalMemory::UP value) const override; Cursor &insertArray() const override; Cursor &insertObject() const override; }; @@ -58,6 +61,7 @@ struct ArrayInserter : Inserter { Cursor &insertDouble(double value) const override; Cursor &insertString(Memory value) const override; Cursor &insertData(Memory value) const override; + Cursor &insertData(ExternalMemory::UP value) const override; Cursor &insertArray() const override; Cursor &insertObject() const override; }; @@ -73,6 +77,7 @@ struct ObjectSymbolInserter : Inserter { Cursor &insertDouble(double value) const override; Cursor &insertString(Memory value) const override; Cursor &insertData(Memory value) const override; + Cursor &insertData(ExternalMemory::UP value) const override; Cursor &insertArray() const override; Cursor &insertObject() const override; }; @@ -88,6 +93,7 @@ struct ObjectInserter : Inserter { Cursor &insertDouble(double value) const override; Cursor &insertString(Memory value) const override; Cursor &insertData(Memory value) const override; + Cursor &insertData(ExternalMemory::UP value) const override; Cursor &insertArray() const override; Cursor &insertObject() const override; }; diff --git a/vespalib/src/vespa/vespalib/data/slime/slime.h b/vespalib/src/vespa/vespalib/data/slime/slime.h index e5c1d6db1a8..af081887c9a 100644 --- a/vespalib/src/vespa/vespalib/data/slime/slime.h +++ b/vespalib/src/vespa/vespalib/data/slime/slime.h @@ -28,6 +28,7 @@ #include "type.h" #include "value.h" #include "value_factory.h" +#include "external_data_value_factory.h" #include <vespa/vespalib/data/input_reader.h> #include <vespa/vespalib/data/output_writer.h> #include <vespa/vespalib/data/simple_buffer.h> @@ -147,6 +148,9 @@ public: Cursor &setData(const Memory& data) { return _root.set(slime::DataValueFactory(data)); } + Cursor &setData(slime::ExternalMemory::UP data) { + return _root.set(slime::ExternalDataValueFactory(std::move(data))); + } Cursor &setArray() { return _root.set(slime::ArrayValueFactory(*_names)); } diff --git a/vespalib/src/vespa/vespalib/data/slime/value.cpp b/vespalib/src/vespa/vespalib/data/slime/value.cpp index 9b415080d7c..2eae660f431 100644 --- a/vespalib/src/vespa/vespalib/data/slime/value.cpp +++ b/vespalib/src/vespa/vespalib/data/slime/value.cpp @@ -5,6 +5,7 @@ #include "resolved_symbol.h" #include "empty_value_factory.h" #include "basic_value_factory.h" +#include "external_data_value_factory.h" #include <vespa/vespalib/data/simple_buffer.h> #include "json_format.h" @@ -80,7 +81,7 @@ Value::toString() const return buf.get().make_string(); } -// 6 x add +// 7 x add Cursor & Value::addNix() { return addLeaf(NixValueFactory()); } Cursor & @@ -93,8 +94,10 @@ Cursor & Value::addString(Memory str) { return addLeaf(StringValueFactory(str)); } Cursor & Value::addData(Memory data) { return addLeaf(DataValueFactory(data)); } +Cursor & +Value::addData(ExternalMemory::UP data) { return addLeaf(ExternalDataValueFactory(std::move(data))); } -// 6 x set (with numeric symbol id) +// 7 x set (with numeric symbol id) Cursor & Value::setNix(Symbol sym) { return setLeaf(sym, NixValueFactory()); } Cursor & @@ -107,8 +110,10 @@ Cursor & Value::setString(Symbol sym, Memory str) { return setLeaf(sym, StringValueFactory(str)); } Cursor & Value::setData(Symbol sym, Memory data) { return setLeaf(sym, DataValueFactory(data)); } +Cursor & +Value::setData(Symbol sym, ExternalMemory::UP data) { return setLeaf(sym, ExternalDataValueFactory(std::move(data))); } -// 6 x set (with symbol name) +// 7 x set (with symbol name) Cursor & Value::setNix(Memory name) { return setLeaf(name, NixValueFactory()); } Cursor & @@ -121,6 +126,8 @@ Cursor & Value::setString(Memory name, Memory str) { return setLeaf(name, StringValueFactory(str)); } Cursor & Value::setData(Memory name, Memory data) { return setLeaf(name, DataValueFactory(data)); } +Cursor & +Value::setData(Memory name, ExternalMemory::UP data) { return setLeaf(name, ExternalDataValueFactory(std::move(data))); } // nop defaults for array/objects Cursor & diff --git a/vespalib/src/vespa/vespalib/data/slime/value.h b/vespalib/src/vespa/vespalib/data/slime/value.h index dacc9b1800e..eaf14829ed9 100644 --- a/vespalib/src/vespa/vespalib/data/slime/value.h +++ b/vespalib/src/vespa/vespalib/data/slime/value.h @@ -57,6 +57,7 @@ public: Cursor &addDouble(double d) override; Cursor &addString(Memory str) override; Cursor &addData(Memory data) override; + Cursor &addData(ExternalMemory::UP data) override; Cursor &addArray() override; Cursor &addObject() override; @@ -66,6 +67,7 @@ public: Cursor &setDouble(Symbol sym, double d) override; Cursor &setString(Symbol sym, Memory str) override; Cursor &setData(Symbol sym, Memory data) override; + Cursor &setData(Symbol sym, ExternalMemory::UP data) override; Cursor &setArray(Symbol sym) override; Cursor &setObject(Symbol sym) override; @@ -75,6 +77,7 @@ public: Cursor &setDouble(Memory name, double d) override; Cursor &setString(Memory name, Memory str) override; Cursor &setData(Memory name, Memory str) override; + Cursor &setData(Memory name, ExternalMemory::UP data) override; Cursor &setArray(Memory name) override; Cursor &setObject(Memory name) override; diff --git a/vespalib/src/vespa/vespalib/util/CMakeLists.txt b/vespalib/src/vespa/vespalib/util/CMakeLists.txt index 310e1dde68d..6d08c3b1126 100644 --- a/vespalib/src/vespa/vespalib/util/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/util/CMakeLists.txt @@ -14,6 +14,7 @@ vespa_add_library(vespalib_vespalib_util OBJECT classname.cpp closuretask.cpp compress.cpp + compressor.cpp dual_merge_director.cpp error.cpp exception.cpp @@ -25,6 +26,7 @@ vespa_add_library(vespalib_vespalib_util OBJECT host_name.cpp joinable.cpp left_right_heap.cpp + lz4compressor.cpp md5.c printable.cpp priority_queue.cpp @@ -47,5 +49,6 @@ vespa_add_library(vespalib_vespalib_util OBJECT threadstackexecutorbase.cpp time_tracker.cpp valgrind.cpp + zstdcompressor.cpp DEPENDS ) diff --git a/vespalib/src/vespa/vespalib/util/compress.cpp b/vespalib/src/vespa/vespalib/util/compress.cpp index bf32fcfe593..617b632e0bf 100644 --- a/vespalib/src/vespa/vespalib/util/compress.cpp +++ b/vespalib/src/vespa/vespalib/util/compress.cpp @@ -4,8 +4,7 @@ #include "stringfmt.h" #include "exceptions.h" -namespace vespalib { -namespace compress { +namespace vespalib::compress { size_t Integer::compressedPositiveLength(uint64_t n) { @@ -84,4 +83,3 @@ size_t Integer::compress(int64_t n, void *destination) } } -} diff --git a/document/src/vespa/document/util/compressionconfig.h b/vespalib/src/vespa/vespalib/util/compressionconfig.h index 413bdc1fb3b..bb54a74ea41 100644 --- a/document/src/vespa/document/util/compressionconfig.h +++ b/vespalib/src/vespa/vespalib/util/compressionconfig.h @@ -4,8 +4,9 @@ #include <cmath> #include <cstdint> #include <cstddef> +#include <cstring> -namespace document { +namespace vespalib::compression { struct CompressionConfig { enum Type { @@ -51,6 +52,14 @@ struct CompressionConfig { default: return NONE; } } + static Type toType(const char * val) { + if (strncasecmp(val, "lz4", 3) == 0) { + return LZ4; + } if (strncasecmp(val, "zstd", 4) == 0) { + return ZSTD; + } + return NONE; + } static bool isCompressed(Type type) { return (type != CompressionConfig::NONE && type != CompressionConfig::UNCOMPRESSABLE); diff --git a/document/src/vespa/document/util/compressor.cpp b/vespalib/src/vespa/vespalib/util/compressor.cpp index cd45017dd69..6853c8375fd 100644 --- a/document/src/vespa/document/util/compressor.cpp +++ b/vespalib/src/vespa/vespalib/util/compressor.cpp @@ -7,11 +7,8 @@ #include <vespa/vespalib/data/databuffer.h> using vespalib::alloc::Alloc; -using vespalib::ConstBufferRef; -using vespalib::DataBuffer; -using vespalib::make_string; -namespace document::compression { +namespace vespalib::compression { CompressionConfig::Type compress(ICompressor & compressor, const CompressionConfig & compression, const ConstBufferRef & org, DataBuffer & dest) @@ -129,10 +126,10 @@ decompress(const CompressionConfig::Type & type, size_t uncompressedLen, const C size_t computeMaxCompressedsize(CompressionConfig::Type type, size_t payloadSize) { if (type == CompressionConfig::LZ4) { - document::LZ4Compressor lz4; + LZ4Compressor lz4; return lz4.adjustProcessLen(0, payloadSize); } else if (type == CompressionConfig::ZSTD) { - document::ZStdCompressor zstd; + ZStdCompressor zstd; return zstd.adjustProcessLen(0, payloadSize); } return payloadSize; diff --git a/document/src/vespa/document/util/compressor.h b/vespalib/src/vespa/vespalib/util/compressor.h index a8d4803e038..8f319a6735d 100644 --- a/document/src/vespa/document/util/compressor.h +++ b/vespalib/src/vespa/vespalib/util/compressor.h @@ -2,11 +2,11 @@ #pragma once #include "compressionconfig.h" -#include <vespa/vespalib/util/buffer.h> +#include "buffer.h" namespace vespalib { class DataBuffer; } -namespace document { +namespace vespalib::compression { class ICompressor { @@ -17,8 +17,6 @@ public: virtual size_t adjustProcessLen(uint16_t options, size_t len) const = 0; }; -namespace compression { - /** * Will try to compress a buffer according to the config. If the criteria can not * be met it will return NONE and dest will get the input buffer. @@ -43,9 +41,6 @@ CompressionConfig::Type compress(const CompressionConfig & compression, const ve */ void decompress(const CompressionConfig::Type & compression, size_t uncompressedLen, const vespalib::ConstBufferRef & org, vespalib::DataBuffer & dest, bool allowSwap); - size_t computeMaxCompressedsize(CompressionConfig::Type type, size_t uncompressedSize); } - -} diff --git a/document/src/vespa/document/util/lz4compressor.cpp b/vespalib/src/vespa/vespalib/util/lz4compressor.cpp index 04d68f394a1..366609ce7bf 100644 --- a/document/src/vespa/document/util/lz4compressor.cpp +++ b/vespalib/src/vespa/vespalib/util/lz4compressor.cpp @@ -8,7 +8,7 @@ using vespalib::alloc::Alloc; -namespace document { +namespace vespalib::compression { size_t LZ4Compressor::adjustProcessLen(uint16_t, size_t len) const { return LZ4_compressBound(len); } diff --git a/document/src/vespa/document/util/lz4compressor.h b/vespalib/src/vespa/vespalib/util/lz4compressor.h index 6dd8f8fd1cb..558088a914c 100644 --- a/document/src/vespa/document/util/lz4compressor.h +++ b/vespalib/src/vespa/vespalib/util/lz4compressor.h @@ -3,7 +3,7 @@ #include "compressor.h" -namespace document { +namespace vespalib::compression { class LZ4Compressor : public ICompressor { diff --git a/document/src/vespa/document/util/zstdcompressor.cpp b/vespalib/src/vespa/vespalib/util/zstdcompressor.cpp index 74a17212a2e..2f09abf4846 100644 --- a/document/src/vespa/document/util/zstdcompressor.cpp +++ b/vespalib/src/vespa/vespalib/util/zstdcompressor.cpp @@ -9,7 +9,7 @@ using vespalib::alloc::Alloc; -namespace document { +namespace vespalib::compression { namespace { diff --git a/document/src/vespa/document/util/zstdcompressor.h b/vespalib/src/vespa/vespalib/util/zstdcompressor.h index ba12ed2594c..0d84b8e7ca4 100644 --- a/document/src/vespa/document/util/zstdcompressor.h +++ b/vespalib/src/vespa/vespalib/util/zstdcompressor.h @@ -3,7 +3,7 @@ #include "compressor.h" -namespace document { +namespace vespalib::compression { class ZStdCompressor : public ICompressor { diff --git a/zkfacade/pom.xml b/zkfacade/pom.xml index b6dbbe63f71..45ba8c32372 100644 --- a/zkfacade/pom.xml +++ b/zkfacade/pom.xml @@ -65,11 +65,6 @@ <version>${project.version}</version> <scope>provided</scope> </dependency> - <!-- Needed to get org.jboss.netty.bootstrap, which zookeeper requires --> - <dependency> - <groupId>io.netty</groupId> - <artifactId>netty</artifactId> - </dependency> </dependencies> <build> <plugins> |