diff options
238 files changed, 2471 insertions, 1345 deletions
diff --git a/.travis.yml b/.travis.yml index 29aa44a4e9c..1144d01e5c6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,7 +22,6 @@ before_cache: - sudo rm -rf $HOME/.m2/repository/com/yahoo - du --summarize --human-readable $HOME/.m2/repository - du --summarize --human-readable $HOME/.ccache - - ccache --show-stats install: true diff --git a/README.md b/README.md index f934f16f011..866eb39bede 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ Code licensed under the Apache 2.0 license. See [LICENSE](LICENSE) for terms. ## Get started developing ### Setup build environment -C++ building is supported on CentOS 7. +C++ building is supported on CentOS 7. The Java source can be built on any platform having Java 8 and Maven installed. We recommend using the following environment: [Create C++ dev environment on CentOS using VirtualBox and Vagrant](vagrant/README.md). You can also setup CentOS 7 natively and install the following build dependencies: @@ -25,10 +25,9 @@ You can also setup CentOS 7 natively and install the following build dependencie yum-builddep -y <vespa-source>/dist/vespa.spec ### Build Java modules -Java modules can be built on any environment having Java 8 and Maven: sh bootstrap.sh - mvn install + mvn -T <num-threads> install ### Build C++ modules Replace `<build-dir>` with the name of the directory in which you'd like to build Vespa. diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/ApplicationPackageXmlFilesValidator.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/ApplicationPackageXmlFilesValidator.java index 29002d8a685..8e9c5c0b509 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/ApplicationPackageXmlFilesValidator.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/ApplicationPackageXmlFilesValidator.java @@ -11,7 +11,6 @@ import java.io.File; import java.io.FilenameFilter; import java.io.IOException; import java.util.List; -import java.util.Optional; /** * Validation of xml files in application package against RELAX NG schemas. @@ -22,31 +21,26 @@ public class ApplicationPackageXmlFilesValidator { private final AppSubDirs appDirs; - /** The Vespa version this package should be validated against */ - private final Version vespaVersion; + private final SchemaValidators validators; private static final FilenameFilter xmlFilter = (dir, name) -> name.endsWith(".xml"); public ApplicationPackageXmlFilesValidator(AppSubDirs appDirs, Version vespaVersion) { this.appDirs = appDirs; - this.vespaVersion = vespaVersion; + this.validators = new SchemaValidators(vespaVersion, new BaseDeployLogger()); } - public static ApplicationPackageXmlFilesValidator createDefaultXMLValidator(File appDir, Version vespaVersion) { - return new ApplicationPackageXmlFilesValidator(new AppSubDirs(appDir), vespaVersion); - } - - public static ApplicationPackageXmlFilesValidator createTestXmlValidator(File appDir, Version vespaVersion) { + public static ApplicationPackageXmlFilesValidator create(File appDir, Version vespaVersion) { return new ApplicationPackageXmlFilesValidator(new AppSubDirs(appDir), vespaVersion); } @SuppressWarnings("deprecation") public void checkApplication() throws IOException { - validate(SchemaValidator.servicesXmlSchemaName, servicesFileName()); - validateOptional(SchemaValidator.hostsXmlSchemaName, FilesApplicationPackage.HOSTS); - validateOptional(SchemaValidator.deploymentXmlSchemaName, FilesApplicationPackage.DEPLOYMENT_FILE.getName()); - validateOptional(SchemaValidator.validationOverridesXmlSchemaName, FilesApplicationPackage.VALIDATION_OVERRIDES.getName()); + validate(validators.servicesXmlValidator(), servicesFileName()); + validateOptional(validators.hostsXmlValidator(), FilesApplicationPackage.HOSTS); + validateOptional(validators.deploymentXmlValidator(), FilesApplicationPackage.DEPLOYMENT_FILE.getName()); + validateOptional(validators.validationOverridesXmlValidator(), FilesApplicationPackage.VALIDATION_OVERRIDES.getName()); if (appDirs.searchdefinitions().exists()) { if (FilesApplicationPackage.getSearchDefinitionFiles(appDirs.root()).isEmpty()) { @@ -55,26 +49,26 @@ public class ApplicationPackageXmlFilesValidator { } } - validate(appDirs.routingtables, "routing-standalone.rnc"); + validateRouting(appDirs.routingtables); } // For testing - public static void checkIncludedDirs(ApplicationPackage app, Version vespaVersion) throws IOException { + public void checkIncludedDirs(ApplicationPackage app) throws IOException { for (String includedDir : app.getUserIncludeDirs()) { List<NamedReader> includedFiles = app.getFiles(Path.fromString(includedDir), ".xml", true); for (NamedReader file : includedFiles) { - createSchemaValidator("container-include.rnc", vespaVersion).validate(file); + validators.containerIncludeXmlValidator().validate(file); } } } - private void validateOptional(String schema, String file) throws IOException { + private void validateOptional(SchemaValidator validator, String file) throws IOException { if ( ! appDirs.file(file).exists()) return; - validate(schema, file); + validate(validator, file); } - private void validate(String schema, String file) throws IOException { - createSchemaValidator(schema, vespaVersion).validate(appDirs.file(file)); + private void validate(SchemaValidator validator, String filename) throws IOException { + validator.validate(appDirs.file(filename)); } @SuppressWarnings("deprecation") @@ -87,26 +81,22 @@ public class ApplicationPackageXmlFilesValidator { return servicesFile; } - private void validate(Tuple2<File, String> directory, String schemaFile) throws IOException { + private void validateRouting(Tuple2<File, String> directory) throws IOException { if ( ! directory.first.isDirectory()) return; - validate(directory, createSchemaValidator(schemaFile, vespaVersion)); + validateRouting(validators.routingStandaloneXmlValidator(), directory); } - private void validate(Tuple2<File, String> directory, SchemaValidator validator) throws IOException { + private void validateRouting(SchemaValidator validator, Tuple2<File, String> directory) throws IOException { File dir = directory.first; if ( ! dir.isDirectory()) return; String directoryName = directory.second; for (File f : dir.listFiles(xmlFilter)) { if (f.isDirectory()) - validate(new Tuple2<>(f, directoryName + File.separator + f.getName()),validator); + validateRouting(validator, new Tuple2<>(f, directoryName + File.separator + f.getName())); else validator.validate(f, directoryName + File.separator + f.getName()); } } - private static SchemaValidator createSchemaValidator(String schemaFile, Version vespaVersion) { - return new SchemaValidator(schemaFile, new BaseDeployLogger(), vespaVersion); - } - } diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/DeployData.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/DeployData.java index d9a784a2dc5..f48dffecb27 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/DeployData.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/DeployData.java @@ -5,7 +5,6 @@ package com.yahoo.config.model.application.provider; * A class for holding values generated or computed during deployment * * @author hmusum - * @since 5.1.11 */ public class DeployData { @@ -57,4 +56,5 @@ public class DeployData { public String getApplicationName() { return applicationName; } + } diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java index 8b8921f50a1..f1565d1fa4b 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java @@ -631,10 +631,11 @@ public class FilesApplicationPackage implements ApplicationPackage { @Override public void validateXML(Optional<Version> vespaVersion) throws IOException { - com.yahoo.component.Version modelVersion = vespaVersion.map(v -> new com.yahoo.component.Version(vespaVersion.toString())).orElse(Vtag.currentVersion); - ApplicationPackageXmlFilesValidator xmlFilesValidator = ApplicationPackageXmlFilesValidator.createDefaultXMLValidator(appDir, modelVersion); - xmlFilesValidator.checkApplication(); - ApplicationPackageXmlFilesValidator.checkIncludedDirs(this, modelVersion); + com.yahoo.component.Version modelVersion = + vespaVersion.map(v -> new com.yahoo.component.Version(vespaVersion.toString())).orElse(Vtag.currentVersion); + ApplicationPackageXmlFilesValidator validator = ApplicationPackageXmlFilesValidator.create(appDir, modelVersion); + validator.checkApplication(); + validator.checkIncludedDirs(this); } @Override diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/SchemaValidator.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/SchemaValidator.java index 70da2f2e92e..d0cca38b375 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/SchemaValidator.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/SchemaValidator.java @@ -6,214 +6,40 @@ import com.thaiopensource.util.PropertyMapBuilder; import com.thaiopensource.validate.ValidateProperty; import com.thaiopensource.validate.ValidationDriver; import com.thaiopensource.validate.rng.CompactSchemaReader; -import com.yahoo.component.Version; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.io.IOUtils; import com.yahoo.io.reader.NamedReader; -import com.yahoo.log.LogLevel; -import static com.yahoo.vespa.defaults.Defaults.getDefaults; import com.yahoo.yolean.Exceptions; -import org.osgi.framework.Bundle; -import org.osgi.framework.FrameworkUtil; import org.xml.sax.ErrorHandler; import org.xml.sax.InputSource; import org.xml.sax.SAXException; import org.xml.sax.SAXParseException; + import java.io.File; import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; import java.io.Reader; -import java.net.JarURLConnection; -import java.net.URL; -import java.util.Enumeration; -import java.util.jar.JarEntry; -import java.util.jar.JarFile; import java.util.logging.Level; -import java.util.logging.Logger; /** - * Validates xml files against one schema. + * Validates xml files against a schema. * * @author tonytv */ public class SchemaValidator { - public static final String schemaDirBase = System.getProperty("java.io.tmpdir", File.separator + "tmp" + File.separator + "vespa"); - static final String servicesXmlSchemaName = "services.rnc"; - static final String hostsXmlSchemaName = "hosts.rnc"; - static final String deploymentXmlSchemaName = "deployment.rnc"; - static final String validationOverridesXmlSchemaName = "validation-overrides.rnc"; private final CustomErrorHandler errorHandler = new CustomErrorHandler(); private final ValidationDriver driver; - private DeployLogger deployLogger; - private static final Logger log = Logger.getLogger(SchemaValidator.class.getName()); - - /** - * Initializes the validator by using the given file as schema file - * @param schema a schema file in RNC format - * @param logger a logger - * @param vespaVersion the version of Vespa we should validate against - */ - public SchemaValidator(String schema, DeployLogger logger, Version vespaVersion) { - this.deployLogger = logger; - driver = new ValidationDriver(PropertyMap.EMPTY, instanceProperties(), CompactSchemaReader.getInstance()); - File schemaDir = new File(schemaDirBase); - try { - schemaDir = saveSchemasFromJar(new File(SchemaValidator.schemaDirBase), vespaVersion); - } catch (IOException e) { - throw new RuntimeException(e); - } - loadSchema(new File(schemaDir + File.separator + "schema" + File.separator + schema)); - IOUtils.recursiveDeleteDir(schemaDir); - } + private final DeployLogger deployLogger; /** * Initializes the validator by using the given file as schema file - * @param schema a schema file in RNC format - * @param vespaVersion the version we should validate against - * @throws IOException if it is not possible to read schema files - */ - public SchemaValidator(String schema, Version vespaVersion) throws IOException { - this(schema, new BaseDeployLogger(), vespaVersion); - } - - /** - * Create a validator for services.xml for tests - * @throws IOException if it is not possible to read schema files - */ - public static SchemaValidator createTestValidatorServices(Version vespaVersion) throws IOException { - return new SchemaValidator(servicesXmlSchemaName, vespaVersion); - } - - /** - * Create a validator for hosts.xml for tests - * @throws IOException if it is not possible to read schema files - */ - public static SchemaValidator createTestValidatorHosts(Version vespaVersion) throws IOException { - return new SchemaValidator(hostsXmlSchemaName, vespaVersion); - } - - /** - * Create a validator for deployment.xml for tests * + * @param schemaFile schema file * @throws IOException if it is not possible to read schema files */ - public static SchemaValidator createTestValidatorDeployment(Version vespaVersion) throws IOException { - return new SchemaValidator(deploymentXmlSchemaName, vespaVersion); - } - - private class CustomErrorHandler implements ErrorHandler { - volatile String fileName; - - public void warning(SAXParseException e) throws SAXException { - deployLogger.log(Level.WARNING, message(e)); - } - - public void error(SAXParseException e) throws SAXException { - throw new IllegalArgumentException(message(e)); - } - - public void fatalError(SAXParseException e) throws SAXException { - throw new IllegalArgumentException(message(e)); - } - - private String message(SAXParseException e) { - return "XML error in " + fileName + ": " + - Exceptions.toMessageString(e) - + " [" + e.getLineNumber() + ":" + e.getColumnNumber() + "]"; - } - } - - /** - * Look for the schema files that should be in vespa-model.jar and saves them on temp dir. - * - * @return the directory the schema files are stored in - * @throws IOException if it is not possible to read schema files - */ - private File saveSchemasFromJar(File tmpBase, Version vespaVersion) throws IOException { - final Class<? extends SchemaValidator> schemaValidatorClass = this.getClass(); - final ClassLoader classLoader = schemaValidatorClass.getClassLoader(); - Enumeration<URL> uris = classLoader.getResources("schema"); - if (uris==null) return null; - File tmpDir = java.nio.file.Files.createTempDirectory(tmpBase.toPath(), "vespa").toFile(); - log.log(LogLevel.DEBUG, "Saving schemas to " + tmpDir); - while(uris.hasMoreElements()) { - URL u = uris.nextElement(); - log.log(LogLevel.DEBUG, "uri for resource 'schema'=" + u.toString()); - if ("jar".equals(u.getProtocol())) { - JarURLConnection jarConnection = (JarURLConnection) u.openConnection(); - JarFile jarFile = jarConnection.getJarFile(); - for (Enumeration<JarEntry> entries = jarFile.entries(); - entries.hasMoreElements();) { - - JarEntry je=entries.nextElement(); - if (je.getName().startsWith("schema/") && je.getName().endsWith(".rnc")) { - writeContentsToFile(tmpDir, je.getName(), jarFile.getInputStream(je)); - } - } - jarFile.close(); - } else if ("bundle".equals(u.getProtocol())) { - Bundle bundle = FrameworkUtil.getBundle(schemaValidatorClass); - log.log(LogLevel.DEBUG, classLoader.toString()); - log.log(LogLevel.DEBUG, "bundle=" + bundle); - // TODO: Hack to handle cases where bundle=null - if (bundle == null) { - File schemaPath; - if (vespaVersion.getMajor() == 5) { - schemaPath = new File(getDefaults().underVespaHome("share/vespa/schema/version/5.x/schema/")); - } else { - schemaPath = new File(getDefaults().underVespaHome("share/vespa/schema/")); - } - log.log(LogLevel.DEBUG, "Using schemas found in " + schemaPath); - copySchemas(schemaPath, tmpDir); - } else { - log.log(LogLevel.DEBUG, String.format("Saving schemas for model bundle %s:%s", bundle.getSymbolicName(), bundle - .getVersion())); - for (Enumeration<URL> entries = bundle.findEntries("schema", "*.rnc", true); - entries.hasMoreElements(); ) { - - URL url = entries.nextElement(); - writeContentsToFile(tmpDir, url.getFile(), url.openStream()); - } - } - } else if ("file".equals(u.getProtocol())) { - File schemaPath = new File(u.getPath()); - copySchemas(schemaPath, tmpDir); - } - } - return tmpDir; - } - - private static void copySchemas(File from, File to) throws IOException { - // TODO: only copy .rnc files. - if (! from.exists()) throw new IOException("Could not find schema source directory '" + from + "'"); - if (! from.isDirectory()) throw new IOException("Schema source '" + from + "' is not a directory"); - File sourceFile = new File(from, servicesXmlSchemaName); - if (! sourceFile.exists()) throw new IOException("Schema source file '" + sourceFile + "' not found"); - IOUtils.copyDirectoryInto(from, to); - } - - private static void writeContentsToFile(File outDir, String outFile, InputStream inputStream) throws IOException { - String contents = IOUtils.readAll(new InputStreamReader(inputStream)); - File out = new File(outDir, outFile); - IOUtils.writeFile(out, contents, false); - } - - private void loadSchema(File schemaFile) { - try { - driver.loadSchema(ValidationDriver.fileInputSource(schemaFile)); - } catch (SAXException e) { - throw new RuntimeException("Invalid schema '" + schemaFile + "'", e); - } catch (IOException e) { - throw new RuntimeException("IO error reading schema '" + schemaFile + "'", e); - } - } - - private PropertyMap instanceProperties() { - PropertyMapBuilder builder = new PropertyMapBuilder(); - builder.put(ValidateProperty.ERROR_HANDLER, errorHandler); - return builder.toPropertyMap(); + SchemaValidator(File schemaFile, DeployLogger deployLogger) throws IOException, SAXException { + this.deployLogger = deployLogger; + this.driver = new ValidationDriver(PropertyMap.EMPTY, instanceProperties(), CompactSchemaReader.getInstance()); + driver.loadSchema(ValidationDriver.fileInputSource(schemaFile)); } public void validate(File file) throws IOException { @@ -246,4 +72,33 @@ public class SchemaValidator { "XML error in " + (fileName == null ? " input" : fileName) + ": " + Exceptions.toMessageString(e)); } } + + private PropertyMap instanceProperties() { + PropertyMapBuilder builder = new PropertyMapBuilder(); + builder.put(ValidateProperty.ERROR_HANDLER, errorHandler); + return builder.toPropertyMap(); + } + + private class CustomErrorHandler implements ErrorHandler { + volatile String fileName; + + public void warning(SAXParseException e) throws SAXException { + deployLogger.log(Level.WARNING, message(e)); + } + + public void error(SAXParseException e) throws SAXException { + throw new IllegalArgumentException(message(e)); + } + + public void fatalError(SAXParseException e) throws SAXException { + throw new IllegalArgumentException(message(e)); + } + + private String message(SAXParseException e) { + return "XML error in " + fileName + ": " + + Exceptions.toMessageString(e) + + " [" + e.getLineNumber() + ":" + e.getColumnNumber() + "]"; + } + } + } diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/SchemaValidators.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/SchemaValidators.java new file mode 100644 index 00000000000..e7e65751ee8 --- /dev/null +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/SchemaValidators.java @@ -0,0 +1,195 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.application.provider; + +import com.yahoo.component.Version; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.io.IOUtils; +import com.yahoo.log.LogLevel; +import org.osgi.framework.Bundle; +import org.osgi.framework.FrameworkUtil; +import org.xml.sax.SAXException; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.JarURLConnection; +import java.net.URL; +import java.util.Enumeration; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; +import java.util.logging.Logger; + +import static com.yahoo.vespa.defaults.Defaults.getDefaults; + +/** + * Wrapper class for schema validators for application package xml files + * + * @author hmusum + */ +public class SchemaValidators { + + private static final String schemaDirBase = System.getProperty("java.io.tmpdir", File.separator + "tmp" + File.separator + "vespa"); + private static final Logger log = Logger.getLogger(SchemaValidators.class.getName()); + + private static final String servicesXmlSchemaName = "services.rnc"; + private static final String hostsXmlSchemaName = "hosts.rnc"; + private static final String deploymentXmlSchemaName = "deployment.rnc"; + private static final String validationOverridesXmlSchemaName = "validation-overrides.rnc"; + private static final String containerIncludeXmlSchemaName = "container-include.rnc"; + private static final String routingStandaloneXmlSchemaName = "routing-standalone.rnc"; + + + private final DeployLogger deployLogger; + + private final SchemaValidator servicesXmlValidator; + private final SchemaValidator hostsXmlValidator; + private final SchemaValidator deploymentXmlValidator; + private final SchemaValidator validationOverridesXmlValidator; + private final SchemaValidator containerIncludeXmlValidator; + private final SchemaValidator routingStandaloneXmlValidator; + + /** + * Initializes the validator by using the given file as schema file + * + * @param vespaVersion the version of Vespa we should validate against + */ + public SchemaValidators(Version vespaVersion, DeployLogger logger) { + this.deployLogger = logger; + File schemaDir; + try { + schemaDir = saveSchemasFromJar(new File(SchemaValidators.schemaDirBase), vespaVersion); + } catch (IOException e) { + throw new RuntimeException(e); + } + + servicesXmlValidator = createValidator(schemaDir, servicesXmlSchemaName); + hostsXmlValidator = createValidator(schemaDir, hostsXmlSchemaName); + deploymentXmlValidator = createValidator(schemaDir, deploymentXmlSchemaName); + validationOverridesXmlValidator = createValidator(schemaDir, validationOverridesXmlSchemaName); + containerIncludeXmlValidator = createValidator(schemaDir, containerIncludeXmlSchemaName); + routingStandaloneXmlValidator = createValidator(schemaDir, routingStandaloneXmlSchemaName); + IOUtils.recursiveDeleteDir(schemaDir); + } + + /** + * Initializes the validator by using the given file as schema file + * + * @param vespaVersion the version of Vespa we should validate against + */ + public SchemaValidators(Version vespaVersion) { + this(vespaVersion, new BaseDeployLogger()); + } + + public SchemaValidator servicesXmlValidator() throws IOException { + return servicesXmlValidator; + } + + public SchemaValidator hostsXmlValidator() throws IOException { + return hostsXmlValidator; + } + + public SchemaValidator deploymentXmlValidator() throws IOException { + return deploymentXmlValidator; + } + + SchemaValidator validationOverridesXmlValidator() throws IOException { + return validationOverridesXmlValidator; + } + + SchemaValidator containerIncludeXmlValidator() { + return containerIncludeXmlValidator; + } + + public SchemaValidator routingStandaloneXmlValidator() { + return routingStandaloneXmlValidator; + } + + /** + * Look for the schema files that should be in vespa-model.jar and saves them on temp dir. + * + * @return the directory the schema files are stored in + * @throws IOException if it is not possible to read schema files + */ + File saveSchemasFromJar(File tmpBase, Version vespaVersion) throws IOException { + final Class<? extends SchemaValidators> schemaValidatorClass = this.getClass(); + final ClassLoader classLoader = schemaValidatorClass.getClassLoader(); + Enumeration<URL> uris = classLoader.getResources("schema"); + if (uris == null) return null; + File tmpDir = java.nio.file.Files.createTempDirectory(tmpBase.toPath(), "vespa").toFile(); + log.log(LogLevel.DEBUG, "Will save all XML schemas to " + tmpDir); + while (uris.hasMoreElements()) { + URL u = uris.nextElement(); + log.log(LogLevel.DEBUG, "uri for resource 'schema'=" + u.toString()); + if ("jar".equals(u.getProtocol())) { + JarURLConnection jarConnection = (JarURLConnection) u.openConnection(); + JarFile jarFile = jarConnection.getJarFile(); + for (Enumeration<JarEntry> entries = jarFile.entries(); + entries.hasMoreElements(); ) { + + JarEntry je = entries.nextElement(); + if (je.getName().startsWith("schema/") && je.getName().endsWith(".rnc")) { + writeContentsToFile(tmpDir, je.getName(), jarFile.getInputStream(je)); + } + } + jarFile.close(); + } else if ("bundle".equals(u.getProtocol())) { + Bundle bundle = FrameworkUtil.getBundle(schemaValidatorClass); + log.log(LogLevel.DEBUG, classLoader.toString()); + log.log(LogLevel.DEBUG, "bundle=" + bundle); + // TODO: Hack to handle cases where bundle=null + if (bundle == null) { + File schemaPath; + if (vespaVersion.getMajor() == 5) { + schemaPath = new File(getDefaults().underVespaHome("share/vespa/schema/version/5.x/schema/")); + } else { + schemaPath = new File(getDefaults().underVespaHome("share/vespa/schema/")); + } + log.log(LogLevel.DEBUG, "Using schemas found in " + schemaPath); + copySchemas(schemaPath, tmpDir); + } else { + log.log(LogLevel.DEBUG, String.format("Saving schemas for model bundle %s:%s", bundle.getSymbolicName(), bundle + .getVersion())); + for (Enumeration<URL> entries = bundle.findEntries("schema", "*.rnc", true); + entries.hasMoreElements(); ) { + + URL url = entries.nextElement(); + writeContentsToFile(tmpDir, url.getFile(), url.openStream()); + } + } + } else if ("file".equals(u.getProtocol())) { + File schemaPath = new File(u.getPath()); + copySchemas(schemaPath, tmpDir); + } + } + return tmpDir; + } + + // TODO: This only copies schema for services.xml. Why? + private static void copySchemas(File from, File to) throws IOException { + // TODO: only copy .rnc files. + if (! from.exists()) throw new IOException("Could not find schema source directory '" + from + "'"); + if (! from.isDirectory()) throw new IOException("Schema source '" + from + "' is not a directory"); + File sourceFile = new File(from, servicesXmlSchemaName); + if (! sourceFile.exists()) throw new IOException("Schema source file '" + sourceFile + "' not found"); + IOUtils.copyDirectoryInto(from, to); + } + + private static void writeContentsToFile(File outDir, String outFile, InputStream inputStream) throws IOException { + String contents = IOUtils.readAll(new InputStreamReader(inputStream)); + File out = new File(outDir, outFile); + IOUtils.writeFile(out, contents, false); + } + + private SchemaValidator createValidator(File schemaDir, String schemaFile) { + try { + File file = new File(schemaDir + File.separator + "schema" + File.separator + schemaFile); + return new SchemaValidator(file, deployLogger); + } catch (SAXException e) { + throw new RuntimeException("Invalid schema '" + schemaFile + "'", e); + } catch (IOException e) { + throw new RuntimeException("IO error reading schema '" + schemaFile + "'", e); + } + } + +} diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/SimpleApplicationValidator.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/SimpleApplicationValidator.java index f94cf3e31a0..9db254bc742 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/SimpleApplicationValidator.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/SimpleApplicationValidator.java @@ -14,6 +14,6 @@ import java.io.Reader; public class SimpleApplicationValidator { public static void checkServices(Reader reader, Version version) throws IOException { - SchemaValidator.createTestValidatorServices(version).validate(reader); + new SchemaValidators(version, new BaseDeployLogger()).servicesXmlValidator().validate(reader); } } diff --git a/config-application-package/src/test/java/com/yahoo/config/application/OverrideProcessorTest.java b/config-application-package/src/test/java/com/yahoo/config/application/OverrideProcessorTest.java index b80372ec9ff..32a1c4f847f 100644 --- a/config-application-package/src/test/java/com/yahoo/config/application/OverrideProcessorTest.java +++ b/config-application-package/src/test/java/com/yahoo/config/application/OverrideProcessorTest.java @@ -15,8 +15,7 @@ import java.io.IOException; import java.io.StringReader; /** - * @author lulf - * @since 5.22 + * @author Ulf Lilleengen */ public class OverrideProcessorTest { diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationFile.java b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationFile.java index bcc8d222ca5..0384a5c7a1c 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationFile.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationFile.java @@ -11,7 +11,7 @@ import java.util.List; * An application file represents a file within an application package. This class can be used to traverse the entire * application package file structure, as well as read and write files to it, and create directories. * - * @author Ulf Lillengen + * @author Ulf Lilleengen */ public abstract class ApplicationFile implements Comparable<ApplicationFile> { diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java index 795e87b7690..c1a786194a2 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.application.api; -import com.yahoo.config.provision.ProvisionInfo; +import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.Version; import com.yahoo.config.provision.Zone; import com.yahoo.path.Path; @@ -28,7 +28,7 @@ import java.util.jar.JarFile; * * Anyone wanting to access application data should use this interface. * - * @author vegardh + * @author Vegard Havdal */ public interface ApplicationPackage { @@ -228,10 +228,22 @@ public interface ApplicationPackage { throw new UnsupportedOperationException("This application package cannot write its metadata"); } - default Map<Version, ProvisionInfo> getProvisionInfoMap() { + /** + * Returns the single host allocation info of this, or an empty map if no allocation is available + * + * @deprecated please use #getAllocatedHosts + */ + // TODO: Remove on Vespa 7 + @Deprecated + default Map<Version, AllocatedHosts> getProvisionInfoMap() { return Collections.emptyMap(); } + /** Returns the host allocation info of this, or empty if no allocation is available */ + default Optional<AllocatedHosts> getAllocatedHosts() { + return Optional.empty(); + } + default Map<Version, FileRegistry> getFileRegistryMap() { return Collections.emptyMap(); } diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java index 2842ac5bd30..68c1a897dc3 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java @@ -346,11 +346,6 @@ public class DeploymentSpec { return true; } - // TODO: Remove when no version older than 6.111 is deployed anywhere - public boolean matches(Environment environment, Optional<RegionName> region) { - return deploysTo(environment, region); - } - @Override public int hashCode() { return Objects.hash(environment, region); diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/HostInfo.java b/config-model-api/src/main/java/com/yahoo/config/model/api/HostInfo.java index dcb875d02b5..589aee50d7c 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/HostInfo.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/HostInfo.java @@ -7,7 +7,6 @@ import java.util.Collection; * Contains information about a host and what services are running on it. * * @author lulf - * @since 5.37 */ public class HostInfo { diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/Model.java b/config-model-api/src/main/java/com/yahoo/config/model/api/Model.java index 0c038077fe4..f8f749ef070 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/Model.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/Model.java @@ -1,12 +1,11 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.model.api; -import com.yahoo.config.provision.ProvisionInfo; +import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.ConfigPayload; import com.yahoo.vespa.config.buildergen.ConfigDefinition; -import java.time.Clock; import java.time.Instant; import java.util.Optional; import java.util.Set; @@ -16,8 +15,7 @@ import java.util.Collection; * A {@link Model} represents the interface towards the model of an entire tenant, and defines methods * for querying this model. * - * @author lulf - * @since 5.1 + * @author Ulf Lilleengen */ public interface Model { @@ -60,9 +58,20 @@ public interface Model { /** * Get the provisioning info for this model. - * @return {@link ProvisionInfo} instance, if available. + * + * @return {@link AllocatedHosts} instance, if available. + * @deprecated use allocatedHosts */ - Optional<ProvisionInfo> getProvisionInfo(); + @Deprecated + // TODO: Remove this (and the implementation below) when no version older than 6.143 is deployed anywhere + default Optional<AllocatedHosts> getProvisionInfo() { + return Optional.of(allocatedHosts()); + } + + @SuppressWarnings("deprecation") + default AllocatedHosts allocatedHosts() { + return getProvisionInfo().get(); + } /** * Returns whether this application allows serving config request for a different version. @@ -71,11 +80,6 @@ public interface Model { */ default boolean allowModelVersionMismatch(Instant now) { return false; } - /** @deprecated pass now. */ - // TODO: Remove this when no version older than 6.115 is deployed anywhere - @Deprecated - default boolean allowModelVersionMismatch() { return allowModelVersionMismatch(Clock.systemUTC().instant()); } - /** * Returns whether old config models should be loaded (default) or not. * Skipping old config models is a validation override which is useful when the old model @@ -88,9 +92,4 @@ public interface Model { */ default boolean skipOldConfigModels(Instant now) { return false; } - /** @deprecated pass now. */ - // TODO: Remove this when no version older than 6.115 is deployed anywhere - @Deprecated - default boolean skipOldConfigModels() { return skipOldConfigModels(Clock.systemUTC().instant()); } - } diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 5afc570d81b..5b79415c132 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -17,7 +17,7 @@ import java.util.Set; /** * Model context containing state provided to model factories. * - * @author lulf + * @author Ulf Lilleengen */ public interface ModelContext { @@ -31,11 +31,6 @@ public interface ModelContext { Properties properties(); default Optional<File> appDir() { return Optional.empty();} - /** @deprecated TODO: Remove this when no config models older than 6.98 are used */ - @SuppressWarnings("unused") - @Deprecated - default Optional<com.yahoo.config.provision.Version> vespaVersion() { return Optional.empty(); } - /** The Vespa version this model is built for */ Version modelVespaVersion(); @@ -50,4 +45,5 @@ public interface ModelContext { Zone zone(); Set<Rotation> rotations(); } + } diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelState.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelState.java index 10153ca07df..c6ac913ad14 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelState.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelState.java @@ -5,5 +5,7 @@ package com.yahoo.config.model.api; * @author lulf */ public interface ModelState { + Model getModel(); + } diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/HostsXmlProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/HostsXmlProvisioner.java index 85f05116e5b..f909f3864da 100644 --- a/config-model/src/main/java/com/yahoo/config/model/provision/HostsXmlProvisioner.java +++ b/config-model/src/main/java/com/yahoo/config/model/provision/HostsXmlProvisioner.java @@ -14,7 +14,6 @@ import java.util.List; * application if one exists. Pre-condition: A valid hosts file. * * @author hmusum - * @since 5.11 */ public class HostsXmlProvisioner implements HostProvisioner { diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java index 133d94c745b..8b97eb2503e 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java @@ -25,6 +25,7 @@ import java.util.*; * @author tonytv */ public class MockApplicationPackage implements ApplicationPackage { + public static final String MUSIC_SEARCHDEFINITION = createSearchDefinition("music", "foo"); public static final String BOOK_SEARCHDEFINITION = createSearchDefinition("book", "bar"); diff --git a/config-model/src/main/java/com/yahoo/config/model/test/TestDriver.java b/config-model/src/main/java/com/yahoo/config/model/test/TestDriver.java index 55c522025fb..b538468d0bc 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/TestDriver.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/TestDriver.java @@ -5,7 +5,8 @@ import com.google.common.annotations.Beta; import com.yahoo.component.Version; import com.yahoo.config.model.MapConfigModelRegistry; import com.yahoo.config.application.api.ApplicationPackage; -import com.yahoo.config.model.application.provider.SchemaValidator; +import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.config.model.application.provider.SchemaValidators; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.builder.xml.ConfigModelBuilder; import com.yahoo.vespa.model.VespaModel; @@ -102,11 +103,10 @@ public class TestDriver { if (!validate) { return; } - SchemaValidator validator = SchemaValidator.createTestValidatorHosts(new Version(6)); + SchemaValidators schemaValidators = new SchemaValidators(new Version(6), new BaseDeployLogger()); if (appPkg.getHosts() != null) { - validator.validate(appPkg.getHosts()); + schemaValidators.hostsXmlValidator().validate(appPkg.getHosts()); } - validator = SchemaValidator.createTestValidatorServices(new Version(6)); - validator.validate(appPkg.getServices()); + schemaValidators.servicesXmlValidator().validate(appPkg.getServices()); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java b/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java index bc890755ca9..5e74a2ebc8a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java @@ -22,7 +22,7 @@ import java.util.stream.Collectors; * TODO: Merge with {@link Host} * Host resources are ordered by their host order. * - * @author Ulf Lillengen + * @author Ulf Lilleengen */ public class HostResource implements Comparable<HostResource> { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java index c61435ca831..53cc8be9e96 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java @@ -20,7 +20,7 @@ import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.config.model.producer.AbstractConfigProducerRoot; import com.yahoo.config.model.producer.UserConfigRepo; -import com.yahoo.config.provision.ProvisionInfo; +import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.log.LogLevel; import com.yahoo.vespa.config.ConfigDefinitionKey; import com.yahoo.vespa.config.ConfigKey; @@ -83,7 +83,7 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri public static final Logger log = Logger.getLogger(VespaModel.class.getPackage().toString()); private ConfigModelRepo configModelRepo = new ConfigModelRepo(); - private final Optional<ProvisionInfo> info; + private final AllocatedHosts allocatedHosts; /** * The config id for the root config producer @@ -146,7 +146,7 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri if (complete) { // create a a completed, frozen model configModelRepo.readConfigModels(deployState, builder, root, configModelRegistry); addServiceClusters(deployState.getApplicationPackage(), builder); - this.info = Optional.of(createProvisionInfo()); // must happen after the two lines above + this.allocatedHosts = AllocatedHosts.withHosts(root.getHostSystem().getHostSpecs()); // must happen after the two lines above setupRouting(); this.fileDistributor = root.getFileDistributionConfigProducer().getFileDistributor(); getAdmin().addPerHostServices(getHostSystem().getHosts(), deployState.getProperties()); @@ -157,7 +157,7 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri this.deployState = null; } else { // create a model with no services instantiated and the given file distributor - this.info = Optional.of(createProvisionInfo()); + this.allocatedHosts = AllocatedHosts.withHosts(root.getHostSystem().getHostSpecs()); this.fileDistributor = fileDistributor; } } @@ -167,10 +167,6 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri return new VespaModel(new NullConfigModelRegistry(), deployState, false, new FileDistributor(deployState.getFileRegistry())); } - private ProvisionInfo createProvisionInfo() { - return ProvisionInfo.withHosts(root.getHostSystem().getHostSpecs()); - } - private void validateWrapExceptions() { try { validate(); @@ -421,8 +417,8 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri } @Override - public Optional<ProvisionInfo> getProvisionInfo() { - return info; + public AllocatedHosts allocatedHosts() { + return allocatedHosts; } private static Set<ConfigKey<?>> configsProduced(ConfigProducer cp) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java index 215aa6c2f7f..fc27f9e8dc7 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java @@ -34,7 +34,7 @@ import java.util.logging.Logger; /** * Factory for creating {@link VespaModel} instances. * - * @author lulf + * @author Ulf Lilleengen */ public class VespaModelFactory implements ModelFactory { @@ -42,9 +42,17 @@ public class VespaModelFactory implements ModelFactory { private final ConfigModelRegistry configModelRegistry; private final Zone zone; private final Clock clock; + private final Version version; + /** Creates a factory for vespa models for this version of the source */ @Inject public VespaModelFactory(ComponentRegistry<ConfigModelPlugin> pluginRegistry, Zone zone) { + this(Version.fromIntValues(VespaVersion.major, VespaVersion.minor, VespaVersion.micro), pluginRegistry, zone); + } + + /** Creates a factory for vespa models of a particular version */ + public VespaModelFactory(Version version, ComponentRegistry<ConfigModelPlugin> pluginRegistry, Zone zone) { + this.version = version; List<ConfigModelBuilder> modelBuilders = new ArrayList<>(); for (ConfigModelPlugin plugin : pluginRegistry.allComponents()) { if (plugin instanceof ConfigModelBuilder) { @@ -55,11 +63,15 @@ public class VespaModelFactory implements ModelFactory { this.zone = zone; this.clock = Clock.systemUTC(); } - + public VespaModelFactory(ConfigModelRegistry configModelRegistry) { this(configModelRegistry, Clock.systemUTC()); } public VespaModelFactory(ConfigModelRegistry configModelRegistry, Clock clock) { + this(Version.fromIntValues(VespaVersion.major, VespaVersion.minor, VespaVersion.micro), configModelRegistry, clock); + } + public VespaModelFactory(Version version, ConfigModelRegistry configModelRegistry, Clock clock) { + this.version = version; if (configModelRegistry == null) { this.configModelRegistry = new NullConfigModelRegistry(); log.info("Will not load config models from plugins, as no registry is available"); @@ -72,9 +84,7 @@ public class VespaModelFactory implements ModelFactory { /** Returns the version this model is build for */ @Override - public Version getVersion() { - return Version.fromIntValues(VespaVersion.major, VespaVersion.minor, VespaVersion.micro); - } + public Version getVersion() { return version; } @Override public Model createModel(ModelContext modelContext) { @@ -82,8 +92,7 @@ public class VespaModelFactory implements ModelFactory { } @Override - public ModelCreateResult createAndValidateModel(ModelContext modelContext, - boolean ignoreValidationErrors) { + public ModelCreateResult createAndValidateModel(ModelContext modelContext, boolean ignoreValidationErrors) { validateXml(modelContext, ignoreValidationErrors); DeployState deployState = createDeployState(modelContext); VespaModel model = buildModel(deployState); @@ -94,18 +103,16 @@ public class VespaModelFactory implements ModelFactory { private void validateXml(ModelContext modelContext, boolean ignoreValidationErrors) { if (modelContext.appDir().isPresent()) { ApplicationPackageXmlFilesValidator validator = - ApplicationPackageXmlFilesValidator.createDefaultXMLValidator(modelContext.appDir().get(), - modelContext.modelVespaVersion()); + ApplicationPackageXmlFilesValidator.create(modelContext.appDir().get(), + modelContext.modelVespaVersion()); try { validator.checkApplication(); - ApplicationPackageXmlFilesValidator.checkIncludedDirs(modelContext.applicationPackage(), - modelContext.modelVespaVersion()); + validator.checkIncludedDirs(modelContext.applicationPackage()); } catch (IllegalArgumentException e) { rethrowUnlessIgnoreErrors(e, ignoreValidationErrors); } catch (Exception e) { throw new RuntimeException(e); } - } else { validateXML(modelContext.applicationPackage(), ignoreValidationErrors); } @@ -164,7 +171,6 @@ public class VespaModelFactory implements ModelFactory { private List<ConfigChangeAction> validateModel(VespaModel model, DeployState deployState, boolean ignoreValidationErrors) { try { - deployState.getApplicationPackage().validateXML(); return Validation.validate(model, ignoreValidationErrors, deployState); } catch (IllegalArgumentException e) { rethrowUnlessIgnoreErrors(e, ignoreValidationErrors); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Slobrok.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Slobrok.java index 0867fc2a299..cefc08981a4 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/Slobrok.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Slobrok.java @@ -41,7 +41,7 @@ public class Slobrok extends AbstractService { } public String getStartupCommand() { - return "exec $ROOT/bin/vespa-slobrok -p " + getPort() + + return "exec $ROOT/sbin/vespa-slobrok -p " + getPort() + " -s " + getStatePort() + " -c " + getConfigId(); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index 4481c53e248..b54978f52d3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -276,7 +276,7 @@ public class ContentSearchCluster extends AbstractConfigProducer implements Prot @Override public void getConfig(ProtonConfig.Builder builder) { double visibilityDelay = hasIndexedCluster() ? getIndexed().getVisibilityDelay() : 0.0; - for (NewDocumentType type : documentDefinitions.values()) { + for (NewDocumentType type : TopologicalDocumentTypeSorter.sort(documentDefinitions.values())) { ProtonConfig.Documentdb.Builder ddbB = new ProtonConfig.Documentdb.Builder(); String docTypeName = type.getFullName().getName(); boolean globalDocType = isGloballyDistributed(type); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/TopologicalDocumentTypeSorter.java b/config-model/src/main/java/com/yahoo/vespa/model/content/TopologicalDocumentTypeSorter.java new file mode 100644 index 00000000000..2f749a28f2d --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/TopologicalDocumentTypeSorter.java @@ -0,0 +1,46 @@ +package com.yahoo.vespa.model.content; + +import com.yahoo.documentmodel.NewDocumentType; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Class that sorts a list of document types in topological order + * according to the document references between the types. + * + * Document types without any outgoing document references are considered + * to be first in the topological order. + * + * @author geirst + */ +public class TopologicalDocumentTypeSorter { + + private final Map<String, NewDocumentType> unsortedTypes = new LinkedHashMap<>(); + private final Map<String, NewDocumentType> sortedTypes = new LinkedHashMap<>(); + + private TopologicalDocumentTypeSorter(Collection<NewDocumentType> documentTypes) { + documentTypes.forEach(docType -> unsortedTypes.put(docType.getName(), docType)); + unsortedTypes.values().forEach(docType -> depthFirstTraverse(docType)); + } + + private void depthFirstTraverse(NewDocumentType docType) { + // Note that cycles are not allowed and detected earlier in DocumentGraphValidator. + if (sortedTypes.containsKey(docType.getName())) { + return; + } + for (NewDocumentType.Name referenceDocTypeName : docType.getDocumentReferences()) { + NewDocumentType referenceDocType = unsortedTypes.get(referenceDocTypeName.getName()); + depthFirstTraverse(referenceDocType); + } + sortedTypes.put(docType.getName(), docType); + } + + public static List<NewDocumentType> sort(Collection<NewDocumentType> documentTypes) { + TopologicalDocumentTypeSorter sorter = new TopologicalDocumentTypeSorter(documentTypes); + return new ArrayList<>(sorter.sortedTypes.values()); + } +} diff --git a/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java b/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java index ee2e6ffcc74..1e1b8cd2ac8 100644 --- a/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java @@ -242,9 +242,10 @@ public class ApplicationDeployTest { public FilesApplicationPackage createAppPkg(String appPkg, boolean validateXml) throws IOException { final FilesApplicationPackage filesApplicationPackage = FilesApplicationPackage.fromFile(new File(appPkg)); if (validateXml) { - ApplicationPackageXmlFilesValidator validator = ApplicationPackageXmlFilesValidator.createTestXmlValidator(new File(appPkg), new Version(6)); + ApplicationPackageXmlFilesValidator validator = + ApplicationPackageXmlFilesValidator.create(new File(appPkg), new Version(6)); validator.checkApplication(); - ApplicationPackageXmlFilesValidator.checkIncludedDirs(filesApplicationPackage, new Version(6)); + validator.checkIncludedDirs(filesApplicationPackage); } return filesApplicationPackage; } diff --git a/config-model/src/test/java/com/yahoo/config/model/application/provider/SchemaValidatorTest.java b/config-model/src/test/java/com/yahoo/config/model/application/provider/SchemaValidatorTest.java index 42659755186..fda230e22ab 100644 --- a/config-model/src/test/java/com/yahoo/config/model/application/provider/SchemaValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/application/provider/SchemaValidatorTest.java @@ -65,6 +65,6 @@ public class SchemaValidatorTest { } private SchemaValidator createValidator() throws IOException { - return SchemaValidator.createTestValidatorServices(new Version(6)); + return new SchemaValidators(new Version(6)).servicesXmlValidator(); } } diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/HostSpecTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/HostSpecTest.java index 6a17f314d26..51b039a7532 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/HostSpecTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/HostSpecTest.java @@ -11,10 +11,10 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; /** - * @author lulf - * @since 5.11 + * @author Ulf Lilleengen */ public class HostSpecTest { + @Test public void testEquals() { HostSpec h1 = new HostSpec("foo", Collections.<String>emptyList()); @@ -42,4 +42,5 @@ public class HostSpecTest { assertFalse(h4.equals(h3)); assertTrue(h4.equals(h4)); } + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java index d2166b170da..5f18b28d6ce 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java @@ -4,10 +4,14 @@ package com.yahoo.vespa.model.content; import com.yahoo.vespa.config.search.core.ProtonConfig; import com.yahoo.vespa.model.content.cluster.ContentCluster; import com.yahoo.vespa.model.content.utils.ContentClusterBuilder; +import com.yahoo.vespa.model.content.utils.SearchDefinitionBuilder; import org.junit.Test; +import java.util.ArrayList; import java.util.Arrays; +import java.util.List; +import static com.yahoo.config.model.test.TestUtil.joinLines; import static com.yahoo.vespa.model.content.utils.ContentClusterUtils.createCluster; import static com.yahoo.vespa.model.content.utils.SearchDefinitionBuilder.createSearchDefinitions; import static junit.framework.TestCase.assertEquals; @@ -86,4 +90,28 @@ public class ContentSearchClusterTest { assertEquals(expGlobal, db.global()); } + @Test + public void require_that_document_types_with_references_are_topologically_sorted() throws Exception { + ProtonConfig cfg = getProtonConfig(createClusterWithThreeDocumentTypes()); + assertEquals(3, cfg.documentdb().size()); + assertDocumentDb("c", true, cfg.documentdb(0)); + assertDocumentDb("b", true, cfg.documentdb(1)); + assertDocumentDb("a", false, cfg.documentdb(2)); + } + + private static ContentCluster createClusterWithThreeDocumentTypes() throws Exception { + List<String> searchDefinitions = new ArrayList<>(); + searchDefinitions.add(new SearchDefinitionBuilder().name("a") + .content(joinLines("field ref_to_b type reference<b> { indexing: attribute }", + "field ref_to_c type reference<c> { indexing: attribute }")).build()); + searchDefinitions.add(new SearchDefinitionBuilder().name("b") + .content("field ref_to_c type reference<c> { indexing: attribute }").build()); + searchDefinitions.add(new SearchDefinitionBuilder().name("c").build()); + return createCluster(new ContentClusterBuilder().docTypes(Arrays.asList( + new ContentClusterBuilder.DocType("a"), + new ContentClusterBuilder.DocType("b", true), + new ContentClusterBuilder.DocType("c", true))).getXml(), + searchDefinitions); + } + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/TopologicalDocumentTypeSorterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/TopologicalDocumentTypeSorterTest.java new file mode 100644 index 00000000000..ddac6562612 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/TopologicalDocumentTypeSorterTest.java @@ -0,0 +1,68 @@ +package com.yahoo.vespa.model.content; + +import com.yahoo.documentmodel.NewDocumentType; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.junit.Assert.assertEquals; + +/** + * @author geirst + */ +public class TopologicalDocumentTypeSorterTest { + + @Test + public void require_that_types_without_references_are_returned_in_input_order() { + assertOrder(Arrays.asList("a"), new DocumentTypesBuilder().add("a")); + assertOrder(Arrays.asList("a", "c", "b"), + new DocumentTypesBuilder().add("a").add("c").add("b")); + } + + @Test + public void require_that_types_with_references_are_sorted_in_topological_order() { + assertOrder(Arrays.asList("b", "a"), new DocumentTypesBuilder() + .add("a", Arrays.asList("b")) + .add("b")); + assertOrder(Arrays.asList("c", "b", "a"), new DocumentTypesBuilder() + .add("a", Arrays.asList("b", "c")) + .add("b", Arrays.asList("c")) + .add("c")); + assertOrder(Arrays.asList("b", "a", "d", "c"), new DocumentTypesBuilder() + .add("a", Arrays.asList("b")) + .add("b") + .add("c", Arrays.asList("d")) + .add("d")); + } + + private void assertOrder(List<String> expOrder, DocumentTypesBuilder builder) { + List<NewDocumentType> sortedDocTypes = TopologicalDocumentTypeSorter.sort(builder.build()); + List<String> actOrder = sortedDocTypes.stream().map(NewDocumentType::getName).collect(Collectors.toList()); + assertEquals(expOrder, actOrder); + } + + private static class DocumentTypesBuilder { + + private final List<NewDocumentType> result = new ArrayList<>(); + + public DocumentTypesBuilder add(String docTypeName) { + return add(docTypeName, Collections.emptyList()); + } + + public DocumentTypesBuilder add(String docTypeName, List<String> docTypeNameReferences) { + Set<NewDocumentType.Name> documentReferences = + docTypeNameReferences.stream().map(NewDocumentType.Name::new).collect(Collectors.toSet()); + result.add(new NewDocumentType(new NewDocumentType.Name(docTypeName), documentReferences)); + return this; + } + + public List<NewDocumentType> build() { + return result; + } + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java index 8102f358830..cc3f4a22966 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java @@ -18,7 +18,7 @@ import com.yahoo.config.model.provision.InMemoryProvisioner; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.config.model.test.TestDriver; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ProvisionInfo; +import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.messagebus.MessagebusConfig; import com.yahoo.net.HostName; @@ -286,7 +286,7 @@ public class VespaModelTestCase { .build()) .build(); VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); - ProvisionInfo info = model.getProvisionInfo().get(); + AllocatedHosts info = model.allocatedHosts(); assertEquals("Admin version 3 is ignored, and there are no other hosts to borrow for admin services", 0, info.getHosts().size()); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithFilePkg.java b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithFilePkg.java index 7802b6f51cd..c89d3098c4d 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithFilePkg.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithFilePkg.java @@ -45,8 +45,10 @@ public class VespaModelCreatorWithFilePkg { } public void validate() throws IOException { - ApplicationPackageXmlFilesValidator.createTestXmlValidator(applicationPkg.getAppDir(), new Version(6)).checkApplication(); - ApplicationPackageXmlFilesValidator.checkIncludedDirs(applicationPkg, new Version(6)); + ApplicationPackageXmlFilesValidator validator = + ApplicationPackageXmlFilesValidator.create(applicationPkg.getAppDir(), new Version(6)); + validator.checkApplication(); + validator.checkIncludedDirs(applicationPkg); } public VespaModel create(boolean validateApplicationWithSchema) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithMockPkg.java b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithMockPkg.java index 98e9fd7b166..8bb500906f3 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithMockPkg.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithMockPkg.java @@ -6,8 +6,8 @@ import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.ConfigModelRegistry; import com.yahoo.config.model.NullConfigModelRegistry; import com.yahoo.config.model.api.ConfigChangeAction; +import com.yahoo.config.model.application.provider.SchemaValidators; import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.config.model.application.provider.SchemaValidator; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.application.validation.Validation; @@ -56,14 +56,15 @@ public class VespaModelCreatorWithMockPkg { VespaModel model = new VespaModel(configModelRegistry, deployState); Version vespaVersion = new Version(6); if (validate) { + SchemaValidators validators = new SchemaValidators(vespaVersion); try { if (appPkg.getHosts() != null) { - SchemaValidator.createTestValidatorHosts(vespaVersion).validate(appPkg.getHosts()); + validators.hostsXmlValidator().validate(appPkg.getHosts()); } if (appPkg.getDeployment().isPresent()) { - SchemaValidator.createTestValidatorDeployment(vespaVersion).validate(appPkg.getDeployment().get()); + validators.deploymentXmlValidator().validate(appPkg.getDeployment().get()); } - SchemaValidator.createTestValidatorServices(vespaVersion).validate(appPkg.getServices()); + validators.servicesXmlValidator().validate(appPkg.getServices()); } catch (Exception e) { System.err.println(e.getClass()); throw e instanceof RuntimeException ? (RuntimeException) e : new RuntimeException(e); diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/AllocatedHosts.java b/config-provisioning/src/main/java/com/yahoo/config/provision/AllocatedHosts.java new file mode 100644 index 00000000000..13efc2b3337 --- /dev/null +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/AllocatedHosts.java @@ -0,0 +1,116 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.provision; + +import com.google.common.collect.ImmutableSet; +import com.yahoo.slime.ArrayTraverser; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.config.SlimeUtils; + +import java.io.IOException; +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.Optional; +import java.util.Set; + +/** + * The hosts allocated to an application. + * This can be serialized to/from JSON. + * This is immutable. + * + * @author Ulf Lilleengen + * @author bratseth + */ +public class AllocatedHosts { + + private static final String mappingKey = "mapping"; + private static final String hostSpecKey = "hostSpec"; + private static final String hostSpecHostName = "hostName"; + private static final String hostSpecMembership = "membership"; + private static final String hostSpecFlavor = "flavor"; + private static final String hostSpecVespaVersion = "vespaVersion"; + + private final ImmutableSet<HostSpec> hosts; + + AllocatedHosts(Set<HostSpec> hosts) { + this.hosts = ImmutableSet.copyOf(hosts); + } + + public static AllocatedHosts withHosts(Set<HostSpec> hosts) { + return new AllocatedHosts(hosts); + } + + private void toSlime(Cursor cursor) { + Cursor array = cursor.setArray(mappingKey); + for (HostSpec host : hosts) + toSlime(host, array.addObject().setObject(hostSpecKey)); + } + + private void toSlime(HostSpec host, Cursor cursor) { + cursor.setString(hostSpecHostName, host.hostname()); + if (host.membership().isPresent()) { + cursor.setString(hostSpecMembership, host.membership().get().stringValue()); + cursor.setString(hostSpecVespaVersion, host.membership().get().cluster().vespaVersion().toString()); + } + if (host.flavor().isPresent()) + cursor.setString(hostSpecFlavor, host.flavor().get().name()); + } + + /** Returns the hosts of this allocation */ + public Set<HostSpec> getHosts() { return hosts; } + + private static AllocatedHosts fromSlime(Inspector inspector, Optional<NodeFlavors> nodeFlavors) { + Inspector array = inspector.field(mappingKey); + Set<HostSpec> hosts = new LinkedHashSet<>(); + array.traverse(new ArrayTraverser() { + @Override + public void entry(int i, Inspector inspector) { + hosts.add(hostsFromSlime(inspector.field(hostSpecKey), nodeFlavors)); + } + }); + return new AllocatedHosts(hosts); + } + + static HostSpec hostsFromSlime(Inspector object, Optional<NodeFlavors> nodeFlavors) { + Optional<ClusterMembership> membership = + object.field(hostSpecMembership).valid() ? Optional.of(membershipFromSlime(object)) : Optional.empty(); + Optional<Flavor> flavor = + object.field(hostSpecFlavor).valid() ? flavorFromSlime(object, nodeFlavors) : Optional.empty(); + + return new HostSpec(object.field(hostSpecHostName).asString(),Collections.emptyList(), flavor, membership); + } + + private static ClusterMembership membershipFromSlime(Inspector object) { + return ClusterMembership.from(object.field(hostSpecMembership).asString(), + com.yahoo.component.Version.fromString(object.field(hostSpecVespaVersion).asString())); + } + + private static Optional<Flavor> flavorFromSlime(Inspector object, Optional<NodeFlavors> nodeFlavors) { + return nodeFlavors.map(flavorMapper -> flavorMapper.getFlavor(object.field(hostSpecFlavor).asString())) + .orElse(Optional.empty()); + } + + public byte[] toJson() throws IOException { + Slime slime = new Slime(); + toSlime(slime.setObject()); + return SlimeUtils.toJsonBytes(slime); + } + + public static AllocatedHosts fromJson(byte[] json, Optional<NodeFlavors> nodeFlavors) { + return fromSlime(SlimeUtils.jsonToSlime(json).get(), nodeFlavors); + } + + @Override + public boolean equals(Object other) { + if (other == this) return true; + if ( ! (other instanceof AllocatedHosts)) return false; + return ((AllocatedHosts) other).hosts.equals(this.hosts); + } + + @Override + public int hashCode() { + return hosts.hashCode(); + } + +} diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/HostSpec.java b/config-provisioning/src/main/java/com/yahoo/config/provision/HostSpec.java index 5d6b3fcaca4..dd8bc311939 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/HostSpec.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/HostSpec.java @@ -5,11 +5,12 @@ import com.google.common.collect.ImmutableList; import java.util.ArrayList; import java.util.List; +import java.util.Objects; import java.util.Optional; /** * A specification of a host and its role. - * The identity of a host is determined by its name. + * Equality and order is determined by the host name. * * @author hmusum */ @@ -19,7 +20,7 @@ public class HostSpec implements Comparable<HostSpec> { private final String hostname; /** Aliases of this host */ - private final List<String> aliases; + private final ImmutableList<String> aliases; /** The current membership role of this host in the cluster it belongs to */ private final Optional<ClusterMembership> membership; @@ -69,10 +70,11 @@ public class HostSpec implements Comparable<HostSpec> { } @Override - public boolean equals(Object o) { - if ( ! (o instanceof HostSpec)) return false; - HostSpec other = (HostSpec) o; - return this.hostname().equals(other.hostname()); + public boolean equals(Object other) { + if (other == this) return true; + if ( ! (other instanceof HostSpec)) return false; + + return ((HostSpec)other).hostname.equals(this.hostname); } @Override diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ProvisionInfo.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ProvisionInfo.java index 8bef1f7c9b7..dbb1b55aeb9 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/ProvisionInfo.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ProvisionInfo.java @@ -1,63 +1,35 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.provision; import com.yahoo.slime.ArrayTraverser; -import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; -import com.yahoo.slime.Slime; import com.yahoo.vespa.config.SlimeUtils; -import java.io.IOException; -import java.util.Collections; import java.util.LinkedHashSet; import java.util.Optional; import java.util.Set; /** - * Information about hosts provisioned for an application, and (de)serialization of this information to/from JSON. - * - * @author lulf - * @since 5.12 + * @author bratseth + * @deprecated use AllocatedHosts */ -public class ProvisionInfo { +// TODO: Remove when no version older than 6.143 is in production anywhere +@Deprecated +@SuppressWarnings("unused") +public class ProvisionInfo extends AllocatedHosts { private static final String mappingKey = "mapping"; private static final String hostSpecKey = "hostSpec"; - private static final String hostSpecHostName = "hostName"; - private static final String hostSpecMembership = "membership"; - private static final String hostSpecFlavor = "flavor"; - private static final String hostSpecVespaVersion = "vespaVersion"; - - private final Set<HostSpec> hosts = new LinkedHashSet<>(); private ProvisionInfo(Set<HostSpec> hosts) { - this.hosts.addAll(hosts); + super(hosts); } public static ProvisionInfo withHosts(Set<HostSpec> hosts) { return new ProvisionInfo(hosts); } - private void toSlime(Cursor cursor) { - Cursor array = cursor.setArray(mappingKey); - for (HostSpec host : hosts) { - Cursor object = array.addObject(); - serializeHostSpec(object.setObject(hostSpecKey), host); - } - } - - private void serializeHostSpec(Cursor cursor, HostSpec host) { - cursor.setString(hostSpecHostName, host.hostname()); - if (host.membership().isPresent()) { - cursor.setString(hostSpecMembership, host.membership().get().stringValue()); - cursor.setString(hostSpecVespaVersion, host.membership().get().cluster().vespaVersion().toString()); - } - if (host.flavor().isPresent()) - cursor.setString(hostSpecFlavor, host.flavor().get().name()); - } - - public Set<HostSpec> getHosts() { - return Collections.unmodifiableSet(hosts); + public static ProvisionInfo fromJson(byte[] json, Optional<NodeFlavors> nodeFlavors) { + return fromSlime(SlimeUtils.jsonToSlime(json).get(), nodeFlavors); } private static ProvisionInfo fromSlime(Inspector inspector, Optional<NodeFlavors> nodeFlavors) { @@ -66,46 +38,10 @@ public class ProvisionInfo { array.traverse(new ArrayTraverser() { @Override public void entry(int i, Inspector inspector) { - hosts.add(deserializeHostSpec(inspector.field(hostSpecKey), nodeFlavors)); + hosts.add(hostsFromSlime(inspector.field(hostSpecKey), nodeFlavors)); } }); return new ProvisionInfo(hosts); } - private static HostSpec deserializeHostSpec(Inspector object, Optional<NodeFlavors> nodeFlavors) { - Optional<ClusterMembership> membership = - object.field(hostSpecMembership).valid() ? Optional.of(readMembership(object)) : Optional.empty(); - Optional<Flavor> flavor = - object.field(hostSpecFlavor).valid() ? readFlavor(object, nodeFlavors) : Optional.empty(); - - return new HostSpec(object.field(hostSpecHostName).asString(),Collections.emptyList(), flavor, membership); - } - - private static ClusterMembership readMembership(Inspector object) { - return ClusterMembership.from(object.field(hostSpecMembership).asString(), - com.yahoo.component.Version.fromString(object.field(hostSpecVespaVersion).asString())); - } - - private static Optional<Flavor> readFlavor(Inspector object, Optional<NodeFlavors> nodeFlavors) { - return nodeFlavors.map(flavorMapper -> flavorMapper.getFlavor(object.field(hostSpecFlavor).asString())) - .orElse(Optional.empty()); - } - - public byte[] toJson() throws IOException { - Slime slime = new Slime(); - toSlime(slime.setObject()); - return SlimeUtils.toJsonBytes(slime); - } - - public static ProvisionInfo fromJson(byte[] json, Optional<NodeFlavors> nodeFlavors) { - return fromSlime(SlimeUtils.jsonToSlime(json).get(), nodeFlavors); - } - - public ProvisionInfo merge(ProvisionInfo provisionInfo) { - Set<HostSpec> mergedSet = new LinkedHashSet<>(); - mergedSet.addAll(this.hosts); - mergedSet.addAll(provisionInfo.getHosts()); - return ProvisionInfo.withHosts(mergedSet); - } - } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/Provisioner.java b/config-provisioning/src/main/java/com/yahoo/config/provision/Provisioner.java index 980cd4a00b9..6be1d49ebd3 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/Provisioner.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/Provisioner.java @@ -9,8 +9,7 @@ import java.util.List; /** * Interface used by the config system to acquire hosts. * - * @author lulf - * @since 5.11 + * @author Ulf Lilleengen */ public interface Provisioner { diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/AllocatedHostsTest.java b/config-provisioning/src/test/java/com/yahoo/config/provision/AllocatedHostsTest.java new file mode 100644 index 00000000000..675af88596a --- /dev/null +++ b/config-provisioning/src/test/java/com/yahoo/config/provision/AllocatedHostsTest.java @@ -0,0 +1,52 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.provision; + +import org.junit.Test; + +import java.io.IOException; +import java.util.LinkedHashSet; +import java.util.Optional; +import java.util.Set; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * @author Ulf Lilleengen + */ +public class AllocatedHostsTest { + + private final HostSpec h1 = new HostSpec("host1", Optional.empty()); + private final HostSpec h2 = new HostSpec("host2", Optional.empty()); + private final HostSpec h3 = new HostSpec("host3", Optional.of(ClusterMembership.from("container/test/0", com.yahoo.component.Version.fromString("6.73.1")))); + + @Test + public void testAllocatedHostsSerialization() throws IOException { + Set<HostSpec> hosts = new LinkedHashSet<>(); + hosts.add(h1); + hosts.add(h2); + hosts.add(h3); + AllocatedHosts info = AllocatedHosts.withHosts(hosts); + assertAllocatedHosts(info); + } + + private void assertAllocatedHosts(AllocatedHosts info) throws IOException { + AllocatedHosts serializedAllocatedHosts = AllocatedHosts.fromJson(info.toJson(), Optional.empty()); + assertEquals(info.getHosts().size(), serializedAllocatedHosts.getHosts().size()); + assertTrue(serializedAllocatedHosts.getHosts().contains(h1)); + assertTrue(serializedAllocatedHosts.getHosts().contains(h2)); + assertTrue(serializedAllocatedHosts.getHosts().contains(h3)); + assertTrue(!getHost(h1.hostname(), serializedAllocatedHosts.getHosts()).membership().isPresent()); + assertEquals("container/test/0", getHost(h3.hostname(), serializedAllocatedHosts.getHosts()).membership().get().stringValue()); + assertEquals(h3.membership().get().cluster().vespaVersion(), getHost(h3.hostname(), + serializedAllocatedHosts.getHosts()).membership().get().cluster().vespaVersion()); + } + + private HostSpec getHost(String hostname, Set<HostSpec> hosts) { + for (HostSpec host : hosts) + if (host.hostname().equals(hostname)) + return host; + throw new IllegalArgumentException("No host " + hostname + " is present"); + } + +} diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/ProvisionInfoTest.java b/config-provisioning/src/test/java/com/yahoo/config/provision/ProvisionInfoTest.java deleted file mode 100644 index 4fa69eb77e0..00000000000 --- a/config-provisioning/src/test/java/com/yahoo/config/provision/ProvisionInfoTest.java +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.config.provision; - -import org.junit.Test; - -import java.io.IOException; -import java.util.Collections; -import java.util.LinkedHashSet; -import java.util.Optional; -import java.util.Set; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -/** - * @author lulf - * @since 5.12 - */ -public class ProvisionInfoTest { - - private final HostSpec h1 = new HostSpec("host1", Optional.empty()); - private final HostSpec h2 = new HostSpec("host2", Optional.empty()); - private final HostSpec h3 = new HostSpec("host3", Optional.of(ClusterMembership.from("container/test/0", com.yahoo.component.Version.fromString("6.73.1")))); - - @Test - public void testProvisionInfoSerialization() throws IOException { - Set<HostSpec> hosts = new LinkedHashSet<>(); - hosts.add(h1); - hosts.add(h2); - hosts.add(h3); - ProvisionInfo info = ProvisionInfo.withHosts(hosts); - assertProvisionInfo(info); - } - - @Test - public void testProvisionInfoMerging() throws IOException { - Set<HostSpec> hostsA = new LinkedHashSet<>(Collections.singleton(h1)); - Set<HostSpec> hostsB = new LinkedHashSet<>(); - hostsB.add(h2); - hostsB.add(h3); - - ProvisionInfo infoA = ProvisionInfo.withHosts(hostsA); - ProvisionInfo infoB = ProvisionInfo.withHosts(hostsB); - assertProvisionInfo(infoA.merge(infoB)); - assertProvisionInfo(infoB.merge(infoA)); - } - - private void assertProvisionInfo(ProvisionInfo info) throws IOException { - ProvisionInfo serializedInfo = ProvisionInfo.fromJson(info.toJson(), Optional.empty()); - assertEquals(info.getHosts().size(), serializedInfo.getHosts().size()); - assertTrue(serializedInfo.getHosts().contains(h1)); - assertTrue(serializedInfo.getHosts().contains(h2)); - assertTrue(serializedInfo.getHosts().contains(h3)); - assertTrue(!getHost(h1.hostname(), serializedInfo.getHosts()).membership().isPresent()); - assertEquals("container/test/0", getHost(h3.hostname(), serializedInfo.getHosts()).membership().get().stringValue()); - assertEquals(h3.membership().get().cluster().vespaVersion(), getHost(h3.hostname(), serializedInfo.getHosts()).membership().get().cluster().vespaVersion()); - } - - private HostSpec getHost(String hostname, Set<HostSpec> hosts) { - for (HostSpec host : hosts) - if (host.hostname().equals(hostname)) - return host; - throw new IllegalArgumentException("No host " + hostname + " is present"); - } - -} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/GlobalComponentRegistry.java b/configserver/src/main/java/com/yahoo/vespa/config/server/GlobalComponentRegistry.java index 1bda8dcb69a..4f26cfa265b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/GlobalComponentRegistry.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/GlobalComponentRegistry.java @@ -19,8 +19,7 @@ import java.util.Optional; /** * Interface representing all global config server components used within the config server. * - * @author lulf - * @since 5.1 + * @author Ulf Lilleengen */ public interface GlobalComponentRegistry { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistry.java b/configserver/src/main/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistry.java index f88a0ef1a2d..fa5224732f6 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistry.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistry.java @@ -22,8 +22,7 @@ import java.util.Optional; /** * Registry containing all the "static"/"global" components in a config server in one place. * - * @author lulf - * @since 5.1 + * @author Ulf Lilleengen */ public class InjectedGlobalComponentRegistry implements GlobalComponentRegistry { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java index 94abbc10046..0435c8e59db 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java @@ -9,8 +9,7 @@ import java.util.List; /** * The applications of a tenant * - * @author lulf - * @since 5.1 + * @author Ulf Lilleengen */ public interface TenantApplications { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ZKTenantApplications.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ZKTenantApplications.java index ac318aba8e8..5260dd9228c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ZKTenantApplications.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ZKTenantApplications.java @@ -29,8 +29,7 @@ import java.util.logging.Logger; * Each application is stored as a single file, named the same as the application id and containing the id * of the session storing the content of the application. * - * @author lulf - * @since 5.1 + * @author Ulf Lilleengen */ // TODO: Merge into interface and separate out curator layer instead public class ZKTenantApplications implements TenantApplications, PathChildrenCacheListener { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java index 61382af6a30..51995eb98cf 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java @@ -136,7 +136,7 @@ public class Deployment implements com.yahoo.config.provision.Deployment { transaction.add(deactivateCurrentActivateNew(localSessionRepo.getActiveSession(session.getApplicationId()), session, ignoreSessionStaleFailure)); if (hostProvisioner.isPresent()) { - hostProvisioner.get().activate(transaction, session.getApplicationId(), session.getProvisionInfo().getHosts()); + hostProvisioner.get().activate(transaction, session.getApplicationId(), session.getAllocatedHosts().getHosts()); } transaction.commit(); session.waitUntilActivated(timeoutBudget); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 820ad8e530c..70b677b4057 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -18,7 +18,7 @@ import java.util.Set; /** * Implementation of {@link ModelContext} for configserver. * - * @author lulf + * @author Ulf Lilleengen */ public class ModelContextImpl implements ModelContext { @@ -83,6 +83,11 @@ public class ModelContextImpl implements ModelContext { return permanentApplicationPackage; } + /** + * Returns the host provisioner to use, or empty to use the default provisioner, + * creating hosts from the application package defined hosts + */ + // TODO: Don't allow empty here but create the right provisioner when this is set up instead @Override public Optional<HostProvisioner> hostProvisioner() { return hostProvisioner; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java index ea278596e80..69266620e45 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java @@ -6,7 +6,7 @@ import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.UnparsedConfigDefinition; -import com.yahoo.config.provision.ProvisionInfo; +import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.Version; import com.yahoo.io.reader.NamedReader; import com.yahoo.log.LogLevel; @@ -25,13 +25,12 @@ import java.util.*; * A class used for reading and writing application data to zookeeper. * * @author hmusum - * @since 5.1 */ public class ZooKeeperClient { private final ConfigCurator configCurator; private final DeployLogger logger; - private final boolean trace; + private final boolean logFine; /* This is the generation that will be used for reading and writing application data. (1 more than last deployed application) */ private final Path rootPath; @@ -42,10 +41,10 @@ public class ZooKeeperClient { } }; - public ZooKeeperClient(ConfigCurator configCurator, DeployLogger logger, boolean trace, Path rootPath) { + public ZooKeeperClient(ConfigCurator configCurator, DeployLogger logger, boolean logFine, Path rootPath) { this.configCurator = configCurator; this.logger = logger; - this.trace = trace; + this.logFine = logFine; this.rootPath = rootPath; } @@ -62,7 +61,7 @@ public class ZooKeeperClient { try { while (retries > 0) { try { - trace("Setting up ZooKeeper nodes for this application"); + logFine("Setting up ZooKeeper nodes for this application"); createZooKeeperNodes(); break; } catch (RuntimeException e) { @@ -105,28 +104,28 @@ public class ZooKeeperClient { * * @param app the application package to feed to zookeeper */ - void feedZooKeeper(ApplicationPackage app) { - trace("Feeding application config into ZooKeeper"); + void write(ApplicationPackage app) { + logFine("Feeding application config into ZooKeeper"); // gives lots and lots of debug output: // BasicConfigurator.configure(); try { - trace("zk operations: " + configCurator.getNumberOfOperations()); - trace("zk operations: " + configCurator.getNumberOfOperations()); - trace("Feeding user def files into ZooKeeper"); - feedZKUserDefs(app); - trace("zk operations: " + configCurator.getNumberOfOperations()); - trace("Feeding application package into ZooKeeper"); + logFine("zk operations: " + configCurator.getNumberOfOperations()); + logFine("zk operations: " + configCurator.getNumberOfOperations()); + logFine("Feeding user def files into ZooKeeper"); + writeUserDefs(app); + logFine("zk operations: " + configCurator.getNumberOfOperations()); + logFine("Feeding application package into ZooKeeper"); // TODO 1200 zk operations done in the below method - feedZKAppPkg(app); - feedSearchDefinitions(app); - feedZKUserIncludeDirs(app, app.getUserIncludeDirs()); - trace("zk operations: " + configCurator.getNumberOfOperations()); - trace("zk read operations: " + configCurator.getNumberOfReadOperations()); - trace("zk write operations: " + configCurator.getNumberOfWriteOperations()); - trace("Feeding sd from docproc bundle into ZooKeeper"); - trace("zk operations: " + configCurator.getNumberOfOperations()); - trace("Write application metadata into ZooKeeper"); - feedZKAppMetaData(app.getMetaData()); - trace("zk operations: " + configCurator.getNumberOfOperations()); + writeSomeOf(app); + writeSearchDefinitions(app); + writeUserIncludeDirs(app, app.getUserIncludeDirs()); + logFine("zk operations: " + configCurator.getNumberOfOperations()); + logFine("zk read operations: " + configCurator.getNumberOfReadOperations()); + logFine("zk write operations: " + configCurator.getNumberOfWriteOperations()); + logFine("Feeding sd from docproc bundle into ZooKeeper"); + logFine("zk operations: " + configCurator.getNumberOfOperations()); + logFine("Write application metadata into ZooKeeper"); + write(app.getMetaData()); + logFine("zk operations: " + configCurator.getNumberOfOperations()); } catch (Exception e) { throw new IllegalStateException("Unable to write vespa model to config server(s) " + System.getProperty("configsources") + "\n" + "Please ensure that cloudconfig_server is started on the config server node(s), " + @@ -134,7 +133,7 @@ public class ZooKeeperClient { } } - private void feedSearchDefinitions(ApplicationPackage app) throws IOException { + private void writeSearchDefinitions(ApplicationPackage app) throws IOException { Collection<NamedReader> sds = app.getSearchDefinitions(); if (sds.isEmpty()) { return; @@ -142,7 +141,7 @@ public class ZooKeeperClient { Path zkPath = getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH).append(ApplicationPackage.SEARCH_DEFINITIONS_DIR); configCurator.createNode(zkPath.getAbsolute()); // Ensures that ranking expressions and other files are also fed. - feedDirZooKeeper(app.getFile(ApplicationPackage.SEARCH_DEFINITIONS_DIR), zkPath, false); + writeDir(app.getFile(ApplicationPackage.SEARCH_DEFINITIONS_DIR), zkPath, false); for (NamedReader sd : sds) { String name = sd.getName(); Reader reader = sd.getReader(); @@ -153,12 +152,12 @@ public class ZooKeeperClient { } /** - * Puts the application package files into ZK + * Puts some of the application package files into ZK - see write(app). * * @param app The application package to use as input. * @throws java.io.IOException if not able to write to Zookeeper */ - void feedZKAppPkg(ApplicationPackage app) throws IOException { + void writeSomeOf(ApplicationPackage app) throws IOException { ApplicationFile.PathFilter srFilter = new ApplicationFile.PathFilter() { @Override public boolean accept(Path path) { @@ -167,40 +166,40 @@ public class ZooKeeperClient { }; // Copy app package files and subdirs into zk // TODO: We should have a way of doing this which doesn't require repeating all the content - feedFileZooKeeper(app.getFile(Path.fromString(ApplicationPackage.SERVICES)), - getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH)); - feedFileZooKeeper(app.getFile(Path.fromString(ApplicationPackage.HOSTS)), - getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH)); - feedFileZooKeeper(app.getFile(Path.fromString(ApplicationPackage.DEPLOYMENT_FILE.getName())), - getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH)); - feedFileZooKeeper(app.getFile(Path.fromString(ApplicationPackage.VALIDATION_OVERRIDES.getName())), - getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH)); + writeFile(app.getFile(Path.fromString(ApplicationPackage.SERVICES)), + getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH)); + writeFile(app.getFile(Path.fromString(ApplicationPackage.HOSTS)), + getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH)); + writeFile(app.getFile(Path.fromString(ApplicationPackage.DEPLOYMENT_FILE.getName())), + getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH)); + writeFile(app.getFile(Path.fromString(ApplicationPackage.VALIDATION_OVERRIDES.getName())), + getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH)); - feedDirZooKeeper(app.getFile(Path.fromString(ApplicationPackage.TEMPLATES_DIR)), - getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH), - true); - feedDirZooKeeper(app.getFile(ApplicationPackage.RULES_DIR), - getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH).append(ApplicationPackage.RULES_DIR), - srFilter, true); - feedDirZooKeeper(app.getFile(ApplicationPackage.QUERY_PROFILES_DIR), - getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH).append(ApplicationPackage.QUERY_PROFILES_DIR), - xmlFilter, true); - feedDirZooKeeper(app.getFile(ApplicationPackage.PAGE_TEMPLATES_DIR), - getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH).append(ApplicationPackage.PAGE_TEMPLATES_DIR), - xmlFilter, true); - feedDirZooKeeper(app.getFile(Path.fromString(ApplicationPackage.SEARCHCHAINS_DIR)), - getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH).append(ApplicationPackage.SEARCHCHAINS_DIR), - xmlFilter, true); - feedDirZooKeeper(app.getFile(Path.fromString(ApplicationPackage.DOCPROCCHAINS_DIR)), - getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH).append(ApplicationPackage.DOCPROCCHAINS_DIR), - xmlFilter, true); - feedDirZooKeeper(app.getFile(Path.fromString(ApplicationPackage.ROUTINGTABLES_DIR)), - getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH).append(ApplicationPackage.ROUTINGTABLES_DIR), - xmlFilter, true); + writeDir(app.getFile(Path.fromString(ApplicationPackage.TEMPLATES_DIR)), + getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH), + true); + writeDir(app.getFile(ApplicationPackage.RULES_DIR), + getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH).append(ApplicationPackage.RULES_DIR), + srFilter, true); + writeDir(app.getFile(ApplicationPackage.QUERY_PROFILES_DIR), + getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH).append(ApplicationPackage.QUERY_PROFILES_DIR), + xmlFilter, true); + writeDir(app.getFile(ApplicationPackage.PAGE_TEMPLATES_DIR), + getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH).append(ApplicationPackage.PAGE_TEMPLATES_DIR), + xmlFilter, true); + writeDir(app.getFile(Path.fromString(ApplicationPackage.SEARCHCHAINS_DIR)), + getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH).append(ApplicationPackage.SEARCHCHAINS_DIR), + xmlFilter, true); + writeDir(app.getFile(Path.fromString(ApplicationPackage.DOCPROCCHAINS_DIR)), + getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH).append(ApplicationPackage.DOCPROCCHAINS_DIR), + xmlFilter, true); + writeDir(app.getFile(Path.fromString(ApplicationPackage.ROUTINGTABLES_DIR)), + getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH).append(ApplicationPackage.ROUTINGTABLES_DIR), + xmlFilter, true); } - private void feedDirZooKeeper(ApplicationFile file, Path zooKeeperAppPath, boolean recurse) throws IOException { - feedDirZooKeeper(file, zooKeeperAppPath, new ApplicationFile.PathFilter() { + private void writeDir(ApplicationFile file, Path zooKeeperAppPath, boolean recurse) throws IOException { + writeDir(file, zooKeeperAppPath, new ApplicationFile.PathFilter() { @Override public boolean accept(Path path) { return true; @@ -208,7 +207,7 @@ public class ZooKeeperClient { }, recurse); } - private void feedDirZooKeeper(ApplicationFile dir, Path path, ApplicationFile.PathFilter filenameFilter, boolean recurse) throws IOException { + private void writeDir(ApplicationFile dir, Path path, ApplicationFile.PathFilter filenameFilter, boolean recurse) throws IOException { if (!dir.isDirectory()) { logger.log(LogLevel.FINE, dir.getPath().getAbsolute()+" is not a directory. Not feeding the files into ZooKeeper."); return; @@ -220,10 +219,10 @@ public class ZooKeeperClient { if (file.isDirectory()) { configCurator.createNode(path.append(name).getAbsolute()); if (recurse) { - feedDirZooKeeper(file, path.append(name), filenameFilter, recurse); + writeDir(file, path.append(name), filenameFilter, recurse); } } else { - feedFileZooKeeper(file, path); + writeFile(file, path); } } } @@ -248,7 +247,7 @@ public class ZooKeeperClient { return ret; } - private void feedFileZooKeeper(ApplicationFile file, Path zkPath) throws IOException { + private void writeFile(ApplicationFile file, Path zkPath) throws IOException { if (!file.exists()) { return; } @@ -260,7 +259,7 @@ public class ZooKeeperClient { } } - private void feedZKUserIncludeDirs(ApplicationPackage applicationPackage, List<String> userIncludeDirs) throws IOException { + private void writeUserIncludeDirs(ApplicationPackage applicationPackage, List<String> userIncludeDirs) throws IOException { // User defined include directories for (String userInclude : userIncludeDirs) { ApplicationFile dir = applicationPackage.getFile(Path.fromString(userInclude)); @@ -268,9 +267,9 @@ public class ZooKeeperClient { if (files == null || files.isEmpty()) { configCurator.createNode(getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + userInclude).getAbsolute()); } - feedDirZooKeeper(dir, - getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + userInclude), - xmlFilter, true); + writeDir(dir, + getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + userInclude), + xmlFilter, true); } } @@ -278,22 +277,22 @@ public class ZooKeeperClient { * Feeds all user-defined .def file from the application package into ZooKeeper (both into * /defconfigs and /userdefconfigs */ - private void feedZKUserDefs(ApplicationPackage applicationPackage) { + private void writeUserDefs(ApplicationPackage applicationPackage) { Map<ConfigDefinitionKey, UnparsedConfigDefinition> configDefs = applicationPackage.getAllExistingConfigDefs(); for (Map.Entry<ConfigDefinitionKey, UnparsedConfigDefinition> entry : configDefs.entrySet()) { ConfigDefinitionKey key = entry.getKey(); String contents = entry.getValue().getUnparsedContent(); - feedDefToZookeeper(key.getName(), key.getNamespace(), getZooKeeperAppPath(ConfigCurator.USER_DEFCONFIGS_ZK_SUBPATH).getAbsolute(), contents); - feedDefToZookeeper(key.getName(), key.getNamespace(), getZooKeeperAppPath(ConfigCurator.DEFCONFIGS_ZK_SUBPATH).getAbsolute(), contents); + write(key.getName(), key.getNamespace(), getZooKeeperAppPath(ConfigCurator.USER_DEFCONFIGS_ZK_SUBPATH).getAbsolute(), contents); + write(key.getName(), key.getNamespace(), getZooKeeperAppPath(ConfigCurator.DEFCONFIGS_ZK_SUBPATH).getAbsolute(), contents); } logger.log(LogLevel.FINE, configDefs.size() + " user config definitions"); } - private void feedDefToZookeeper(String name, String namespace, String path, String data) { - feedDefToZookeeper(name, namespace, "", path, com.yahoo.text.Utf8.toBytes(data)); + private void write(String name, String namespace, String path, String data) { + write(name, namespace, "", path, com.yahoo.text.Utf8.toBytes(data)); } - private void feedDefToZookeeper(String name, String namespace, String version, String path, byte[] data) { + private void write(String name, String namespace, String version, String path, byte[] data) { configCurator.putDefData( ("".equals(namespace)) ? name : (namespace + "." + name), version, @@ -301,8 +300,8 @@ public class ZooKeeperClient { data); } - private void feedZKFileRegistry(Version vespaVersion, FileRegistry fileRegistry) { - trace("Feeding file registry data into ZooKeeper"); + private void write(Version vespaVersion, FileRegistry fileRegistry) { + logFine("Feeding file registry data into ZooKeeper"); String exportedRegistry = PreGeneratedFileRegistry.exportRegistry(fileRegistry); configCurator.putData(getZooKeeperAppPath(null).append(ZKApplicationPackage.fileRegistryNode).getAbsolute(), @@ -316,12 +315,12 @@ public class ZooKeeperClient { * * @param metaData The application metadata. */ - private void feedZKAppMetaData(ApplicationMetaData metaData) { + private void write(ApplicationMetaData metaData) { configCurator.putData(getZooKeeperAppPath(ConfigCurator.META_ZK_PATH).getAbsolute(), metaData.asJsonString()); } void cleanupZooKeeper() { - trace("Exception occurred. Cleaning up ZooKeeper"); + logFine("Exception occurred. Cleaning up ZooKeeper"); try { for (String subPath : Arrays.asList( ConfigCurator.DEFCONFIGS_ZK_SUBPATH, @@ -350,26 +349,20 @@ public class ZooKeeperClient { } } - void trace(String msg) { - if (trace) { + void logFine(String msg) { + if (logFine) { logger.log(LogLevel.FINE, msg); } } - private void feedProvisionInfo(Version version, ProvisionInfo info) throws IOException { - byte[] json = info.toJson(); - configCurator.putData(rootPath.append(ZKApplicationPackage.allocatedHostsNode).append(version.toSerializedForm()).getAbsolute(), json); + public void write(AllocatedHosts info) throws IOException { + configCurator.putData(rootPath.append(ZKApplicationPackage.allocatedHostsNode).getAbsolute(), info.toJson()); } - public void feedZKFileRegistries(Map<Version, FileRegistry> fileRegistryMap) { + public void write(Map<Version, FileRegistry> fileRegistryMap) { for (Map.Entry<Version, FileRegistry> versionFileRegistryEntry : fileRegistryMap.entrySet()) { - feedZKFileRegistry(versionFileRegistryEntry.getKey(), versionFileRegistryEntry.getValue()); + write(versionFileRegistryEntry.getKey(), versionFileRegistryEntry.getValue()); } } - public void feedProvisionInfos(Map<Version, ProvisionInfo> provisionInfoMap) throws IOException { - for (Map.Entry<Version, ProvisionInfo> versionProvisionInfoEntry : provisionInfoMap.entrySet()) { - feedProvisionInfo(versionProvisionInfoEntry.getKey(), versionProvisionInfoEntry.getValue()); - } - } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperDeployer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperDeployer.java index 246d7226cfd..22ce952481d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperDeployer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperDeployer.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.config.server.deploy; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.FileRegistry; -import com.yahoo.config.provision.ProvisionInfo; +import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.Version; import java.io.IOException; @@ -14,7 +14,6 @@ import java.util.Map; * Initialize must be called before each deploy. * * @author lulf - * @since 5.1 */ public class ZooKeeperDeployer { @@ -28,15 +27,16 @@ public class ZooKeeperDeployer { * Deploys an application package to zookeeper. initialize() must be called before calling this method. * * @param applicationPackage The application package to persist. - * @param fileRegistryMap The file registries to persist. - * @param provisionInfoMap The provisioning infos to persist. + * @param fileRegistryMap the file registries to persist. + * @param allocatedHosts the provisioning info to persist. * @throws IOException if deploying fails */ - public void deploy(ApplicationPackage applicationPackage, Map<Version, FileRegistry> fileRegistryMap, Map<Version, ProvisionInfo> provisionInfoMap) throws IOException { + public void deploy(ApplicationPackage applicationPackage, Map<Version, FileRegistry> fileRegistryMap, + AllocatedHosts allocatedHosts) throws IOException { zooKeeperClient.setupZooKeeper(); - zooKeeperClient.feedZooKeeper(applicationPackage); - zooKeeperClient.feedZKFileRegistries(fileRegistryMap); - zooKeeperClient.feedProvisionInfos(provisionInfoMap); + zooKeeperClient.write(applicationPackage); + zooKeeperClient.write(fileRegistryMap); + zooKeeperClient.write(allocatedHosts); } public void cleanup() { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/host/HostRegistry.java b/configserver/src/main/java/com/yahoo/vespa/config/server/host/HostRegistry.java index 36e7737163a..e61fc124d53 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/host/HostRegistry.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/host/HostRegistry.java @@ -16,7 +16,6 @@ import com.yahoo.log.LogLevel; * TODO: Is there a generalized version of this pattern? Need some sort mix of Bimap and Multimap * * @author lulf - * @since 5.3 */ public class HostRegistry<T> implements HostValidator<T> { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java index bbd1e189e29..1301f24788f 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java @@ -5,12 +5,11 @@ import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.api.ConfigDefinitionRepo; -import com.yahoo.config.model.api.HostProvisioner; import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.api.ModelFactory; import com.yahoo.config.model.application.provider.MockFileRegistry; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ProvisionInfo; +import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Version; import com.yahoo.config.provision.Zone; @@ -24,7 +23,6 @@ import com.yahoo.vespa.config.server.application.PermanentApplicationPackage; import com.yahoo.vespa.config.server.deploy.ModelContextImpl; import com.yahoo.vespa.config.server.monitoring.MetricUpdater; import com.yahoo.vespa.config.server.monitoring.Metrics; -import com.yahoo.vespa.config.server.provision.StaticProvisioner; import com.yahoo.vespa.config.server.session.SessionZooKeeperClient; import com.yahoo.vespa.config.server.session.SilentDeployLogger; import com.yahoo.vespa.curator.Curator; @@ -47,7 +45,6 @@ public class ActivatedModelsBuilder extends ModelsBuilder<Application> { private final long appGeneration; private final SessionZooKeeperClient zkClient; private final Optional<PermanentApplicationPackage> permanentApplicationPackage; - private final Optional<com.yahoo.config.provision.Provisioner> hostProvisioner; private final ConfigserverConfig configserverConfig; private final ConfigDefinitionRepo configDefinitionRepo; private final Metrics metrics; @@ -56,7 +53,8 @@ public class ActivatedModelsBuilder extends ModelsBuilder<Application> { private final DeployLogger logger; public ActivatedModelsBuilder(TenantName tenant, long appGeneration, SessionZooKeeperClient zkClient, GlobalComponentRegistry globalComponentRegistry) { - super(globalComponentRegistry.getModelFactoryRegistry()); + super(globalComponentRegistry.getModelFactoryRegistry(), + globalComponentRegistry.getHostProvisioner().isPresent()); this.tenant = tenant; this.appGeneration = appGeneration; this.zkClient = zkClient; @@ -64,17 +62,17 @@ public class ActivatedModelsBuilder extends ModelsBuilder<Application> { this.configserverConfig = globalComponentRegistry.getConfigserverConfig(); this.configDefinitionRepo = globalComponentRegistry.getConfigDefinitionRepo(); this.metrics = globalComponentRegistry.getMetrics(); - this.hostProvisioner = globalComponentRegistry.getHostProvisioner(); this.curator = globalComponentRegistry.getCurator(); this.zone = globalComponentRegistry.getZone(); this.logger = new SilentDeployLogger(); } @Override - protected Application buildModelVersion(ModelFactory modelFactory, + protected Application buildModelVersion(ModelFactory modelFactory, ApplicationPackage applicationPackage, - ApplicationId applicationId, + ApplicationId applicationId, com.yahoo.component.Version wantedNodeVespaVersion, + Optional<AllocatedHosts> ignored, // Ignored since we have this in the app package for activated models Instant now) { log.log(LogLevel.DEBUG, String.format("Loading model version %s for session %s application %s", modelFactory.getVersion(), appGeneration, applicationId)); @@ -86,7 +84,7 @@ public class ActivatedModelsBuilder extends ModelsBuilder<Application> { logger, configDefinitionRepo, getForVersionOrLatest(applicationPackage.getFileRegistryMap(), modelFactory.getVersion()).orElse(new MockFileRegistry()), - createHostProvisioner(getForVersionOrLatest(applicationPackage.getProvisionInfoMap(), modelFactory.getVersion())), + createStaticProvisioner(applicationPackage.getAllocatedHosts()), createModelContextProperties(applicationId), Optional.empty(), new com.yahoo.component.Version(modelFactory.getVersion().toString()), @@ -96,17 +94,6 @@ public class ActivatedModelsBuilder extends ModelsBuilder<Application> { applicationMetricUpdater, applicationId); } - private Optional<HostProvisioner> createHostProvisioner(Optional<ProvisionInfo> provisionInfo) { - if (hostProvisioner.isPresent() && provisionInfo.isPresent()) { - return Optional.of(createStaticProvisioner(provisionInfo.get())); - } - return Optional.empty(); - } - - private HostProvisioner createStaticProvisioner(ProvisionInfo provisionInfo) { - return new StaticProvisioner(provisionInfo); - } - private static <T> Optional<T> getForVersionOrLatest(Map<Version, T> map, Version version) { if (map.isEmpty()) { return Optional.empty(); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelFactoryRegistry.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelFactoryRegistry.java index ab1c84eb5dd..1c2c9c731d1 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelFactoryRegistry.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelFactoryRegistry.java @@ -13,7 +13,7 @@ import java.util.*; * A registry of model factories. Allows querying for a specific version of a {@link ModelFactory} or * simply returning all of them. Keeps track of the latest {@link com.yahoo.config.provision.Version} supported. * - * @author lulf + * @author Ulf Lilleengen */ public class ModelFactoryRegistry { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java index cc7a12801d3..6a4ab40d843 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java @@ -3,16 +3,20 @@ package com.yahoo.vespa.config.server.modelfactory; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.api.HostProvisioner; import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.api.ModelFactory; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.OutOfCapacityException; +import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.Rotation; import com.yahoo.config.provision.Version; import com.yahoo.config.provision.Zone; +import com.yahoo.lang.SettableOptional; import com.yahoo.vespa.config.server.ConfigServerSpec; import com.yahoo.vespa.config.server.deploy.ModelContextImpl; import com.yahoo.vespa.config.server.http.UnknownVespaVersionException; +import com.yahoo.vespa.config.server.provision.StaticProvisioner; import java.time.Instant; import java.util.ArrayList; @@ -37,13 +41,24 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> { private final ModelFactoryRegistry modelFactoryRegistry; - protected ModelsBuilder(ModelFactoryRegistry modelFactoryRegistry) { + /** True if we are running in hosted mode */ + private final boolean hosted; + + protected ModelsBuilder(ModelFactoryRegistry modelFactoryRegistry, boolean hosted) { this.modelFactoryRegistry = modelFactoryRegistry; + this.hosted = hosted; } + /** + * Builds all applicable model versions + * + * @param allocatedHosts the newest version (major and minor) (which is loaded first) decides the allocated hosts + * and assigns to this SettableOptional such that it can be used after this method returns + */ public List<MODELRESULT> buildModels(ApplicationId applicationId, com.yahoo.component.Version wantedNodeVespaVersion, ApplicationPackage applicationPackage, + SettableOptional<AllocatedHosts> allocatedHosts, Instant now) { Set<Version> versions = modelFactoryRegistry.allVersions(); @@ -52,7 +67,7 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> { if (requestedMajorVersion.isPresent()) versions = filterByMajorVersion(requestedMajorVersion.get(), versions); - // Load models by one major version at the time as new major versions are allowed to be unloadable + // Load models by one major version at the time as new major versions are allowed to be non-loadable // in the case where an existing application is incompatible with a new major version // (which is possible by the definition of major) List<Integer> majorVersions = versions.stream() @@ -65,7 +80,8 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> { for (int i = 0; i < majorVersions.size(); i++) { try { allApplicationModels.addAll(buildModelVersion(filterByMajorVersion(majorVersions.get(i), versions), - applicationId, wantedNodeVespaVersion, applicationPackage, now)); + applicationId, wantedNodeVespaVersion, applicationPackage, + allocatedHosts, now)); // skip old config models if requested after we have found a major version which works if (allApplicationModels.size() > 0 && allApplicationModels.get(0).getModel().skipOldConfigModels(now)) @@ -91,30 +107,43 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> { private List<MODELRESULT> buildModelVersion(Set<Version> versions, ApplicationId applicationId, com.yahoo.component.Version wantedNodeVespaVersion, ApplicationPackage applicationPackage, + SettableOptional<AllocatedHosts> allocatedHosts, Instant now) { Version latest = findLatest(versions); // load latest application version - MODELRESULT latestApplicationVersion = buildModelVersion(modelFactoryRegistry.getFactory(latest), + MODELRESULT latestModelVersion = buildModelVersion(modelFactoryRegistry.getFactory(latest), applicationPackage, applicationId, wantedNodeVespaVersion, + allocatedHosts.asOptional(), now); - if (latestApplicationVersion.getModel().skipOldConfigModels(now)) { - return Collections.singletonList(latestApplicationVersion); - } - else { // load old model versions - List<MODELRESULT> allApplicationVersions = new ArrayList<>(); - allApplicationVersions.add(latestApplicationVersion); - for (Version version : versions) { - if (version.equals(latest)) continue; // already loaded - allApplicationVersions.add(buildModelVersion(modelFactoryRegistry.getFactory(version), - applicationPackage, - applicationId, - wantedNodeVespaVersion, - now)); - } - return allApplicationVersions; + allocatedHosts.set(latestModelVersion.getModel().allocatedHosts()); // Update with additional clusters allocated + + if (latestModelVersion.getModel().skipOldConfigModels(now)) + return Collections.singletonList(latestModelVersion); + + // load old model versions + List<MODELRESULT> allApplicationVersions = new ArrayList<>(); + allApplicationVersions.add(latestModelVersion); + + // TODO: We use the allocated hosts from the newest version when building older model versions. + // This is correct except for the case where an old model specifies a cluster which the new version + // does not. In that case we really want to extend the set of allocated hosts to include those of that + // cluster as well. To do that, create a new provisioner which uses static provisioning for known + // clusters and the node repository provisioner as fallback. + for (Version version : versions) { + if (version.equals(latest)) continue; // already loaded + + MODELRESULT modelVersion = buildModelVersion(modelFactoryRegistry.getFactory(version), + applicationPackage, + applicationId, + wantedNodeVespaVersion, + allocatedHosts.asOptional(), + now); + allocatedHosts.set(modelVersion.getModel().allocatedHosts()); // Update with additional clusters allocated + allApplicationVersions.add(modelVersion); } + return allApplicationVersions; } private Set<Version> filterByMajorVersion(int majorVersion, Set<Version> versions) { @@ -133,6 +162,7 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> { protected abstract MODELRESULT buildModelVersion(ModelFactory modelFactory, ApplicationPackage applicationPackage, ApplicationId applicationId, com.yahoo.component.Version wantedNodeVespaVersion, + Optional<AllocatedHosts> allocatedHosts, Instant now); protected ModelContext.Properties createModelContextProperties(ApplicationId applicationId, @@ -147,4 +177,15 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> { rotations); } + /** + * Returns a host provisioner returning the previously allocated hosts if available and when on hosted Vespa, + * returns empty otherwise, which may either mean that no hosts are allocated or that we are running + * non-hosted and should default to use hosts defined in the application package, depending on context + */ + protected Optional<HostProvisioner> createStaticProvisioner(Optional<AllocatedHosts> allocatedHosts) { + if (hosted && allocatedHosts.isPresent()) + return Optional.of(new StaticProvisioner(allocatedHosts.get())); + return Optional.empty(); + } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java index b2dca075c4f..78d660b347e 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java @@ -12,6 +12,7 @@ import com.yahoo.config.model.api.ModelCreateResult; import com.yahoo.config.model.api.ModelFactory; import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.Version; import com.yahoo.log.LogLevel; import com.yahoo.vespa.config.server.application.ApplicationSet; @@ -21,6 +22,7 @@ import com.yahoo.vespa.config.server.deploy.ModelContextImpl; import com.yahoo.vespa.config.server.filedistribution.FileDistributionProvider; import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; import com.yahoo.vespa.config.server.provision.ProvisionerAdapter; +import com.yahoo.vespa.config.server.provision.StaticProvisioner; import com.yahoo.vespa.config.server.session.FileDistributionFactory; import com.yahoo.vespa.config.server.session.PrepareParams; import com.yahoo.vespa.config.server.session.SessionContext; @@ -60,7 +62,7 @@ public class PreparedModelsBuilder extends ModelsBuilder<PreparedModelsBuilder.P PrepareParams params, Optional<ApplicationSet> currentActiveApplicationSet, ModelContext.Properties properties) { - super(modelFactoryRegistry); + super(modelFactoryRegistry, properties.hostedVespa()); this.permanentApplicationPackage = permanentApplicationPackage; this.configDefinitionRepo = configDefinitionRepo; @@ -79,14 +81,17 @@ public class PreparedModelsBuilder extends ModelsBuilder<PreparedModelsBuilder.P protected PreparedModelResult buildModelVersion(ModelFactory modelFactory, ApplicationPackage applicationPackage, ApplicationId applicationId, - com.yahoo.component.Version wantedNodeVespaVersion, Instant now) { + com.yahoo.component.Version wantedNodeVespaVersion, + Optional<AllocatedHosts> allocatedHosts, + Instant now) { Version modelVersion = modelFactory.getVersion(); log.log(LogLevel.DEBUG, "Building model " + modelVersion + " for " + applicationId); FileDistributionProvider fileDistributionProvider = fileDistributionFactory.createProvider( context.getServerDBSessionDir(), applicationId); - Optional<HostProvisioner> hostProvisioner = createHostProvisionerAdapter(properties); + // Use empty on non-hosted systems, use already allocated hosts if available, create connection to a host provisioner otherwise + Optional<HostProvisioner> hostProvisioner = createHostProvisioner(allocatedHosts); Optional<Model> previousModel = currentActiveApplicationSet .map(set -> set.getForVersionOrLatest(Optional.of(modelVersion), now).getModel()); ModelContext modelContext = new ModelContextImpl( @@ -109,6 +114,24 @@ public class PreparedModelsBuilder extends ModelsBuilder<PreparedModelsBuilder.P return new PreparedModelsBuilder.PreparedModelResult(modelVersion, result.getModel(), fileDistributionProvider, result.getConfigChangeActions()); } + // This method is an excellent demonstration of what happens when one is too liberal with Optional + // -bratseth, who had to write the below :-\ + private Optional<HostProvisioner> createHostProvisioner(Optional<AllocatedHosts> allocatedHosts) { + Optional<HostProvisioner> nodeRepositoryProvisioner = createNodeRepositoryProvisioner(properties); + if ( ! allocatedHosts.isPresent()) return nodeRepositoryProvisioner; + + Optional<HostProvisioner> staticProvisioner = createStaticProvisioner(allocatedHosts); + if ( ! staticProvisioner.isPresent()) return Optional.empty(); // Since we have hosts allocated this means we are on non-hosted + + // The following option should not be possible, but since there is a right action for it we can take it + if ( ! nodeRepositoryProvisioner.isPresent()) + return Optional.of(new StaticProvisioner(allocatedHosts.get())); + + // Nodes are already allocated by a model and we should use them unless this model requests hosts from a + // previously unallocated cluster. This allows future models to stop allocate certain clusters. + return Optional.of(new StaticProvisioner(allocatedHosts.get(), nodeRepositoryProvisioner.get())); + } + private Optional<File> getAppDir(ApplicationPackage applicationPackage) { try { return applicationPackage instanceof FilesApplicationPackage ? @@ -124,7 +147,7 @@ public class PreparedModelsBuilder extends ModelsBuilder<PreparedModelsBuilder.P .collect(Collectors.toList())); } - private Optional<HostProvisioner> createHostProvisionerAdapter(ModelContext.Properties properties) { + private Optional<HostProvisioner> createNodeRepositoryProvisioner(ModelContext.Properties properties) { return hostProvisionerProvider.getHostProvisioner().map( provisioner -> new ProvisionerAdapter(provisioner, properties.applicationId())); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/provision/HostProvisionerProvider.java b/configserver/src/main/java/com/yahoo/vespa/config/server/provision/HostProvisionerProvider.java index 619ee7499e2..32a641a3bc7 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/provision/HostProvisionerProvider.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/provision/HostProvisionerProvider.java @@ -13,21 +13,18 @@ import java.util.logging.Logger; /** * This class is necessary to support both having and not having a host provisioner. We inject * a component registry here, which then enables us to check whether or not we have a provisioner available. + * We only have a provisioner if we are running in hosted mode. * - * @author lulf - * @since 5.15 + * @author Ulf Lilleengen */ public class HostProvisionerProvider { - private static final Logger log = Logger.getLogger(HostProvisionerProvider.class.getName()); private final Optional<Provisioner> hostProvisioner; public HostProvisionerProvider(ComponentRegistry<Provisioner> hostProvisionerRegistry, ConfigserverConfig configserverConfig) { if (hostProvisionerRegistry.allComponents().isEmpty() || ! configserverConfig.hostedVespa()) { - log.info("Host provisioner is missing, provisioner component count: " + hostProvisionerRegistry.allComponents().size() + ", is hosted Vespa: " + configserverConfig.hostedVespa()); hostProvisioner = Optional.empty(); } else { - log.log(LogLevel.DEBUG, "Host provisioner injected. Will be used for all deployments"); hostProvisioner = Optional.of(hostProvisionerRegistry.allComponents().get(0)); } } @@ -36,6 +33,7 @@ public class HostProvisionerProvider { this(componentRegistry, new ConfigserverConfig(new ConfigserverConfig.Builder())); } + /** Returns the host provisioner, or empty if we are not in hosted mode */ public Optional<Provisioner> getHostProvisioner() { return hostProvisioner; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/provision/ProvisionerAdapter.java b/configserver/src/main/java/com/yahoo/vespa/config/server/provision/ProvisionerAdapter.java index ad0ee3b3eb9..32380b296dd 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/provision/ProvisionerAdapter.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/provision/ProvisionerAdapter.java @@ -16,8 +16,7 @@ import java.util.*; * behavior to the config model. Adapts interface from a {@link HostProvisioner} to a * {@link Provisioner}. * - * @author lulf - * @since 5.11 + * @author Ulf Lilleengen */ public class ProvisionerAdapter implements HostProvisioner { @@ -31,6 +30,7 @@ public class ProvisionerAdapter implements HostProvisioner { @Override public HostSpec allocateHost(String alias) { + // Wow. Such mess. TODO: Actually support polymorphy or stop pretending to, see also ModelContextImpl.getHostProvisioner throw new UnsupportedOperationException("Allocating a single host in a hosted environment is not possible"); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/provision/StaticProvisioner.java b/configserver/src/main/java/com/yahoo/vespa/config/server/provision/StaticProvisioner.java index 1cad735879a..7e97690331f 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/provision/StaticProvisioner.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/provision/StaticProvisioner.java @@ -5,19 +5,35 @@ import com.yahoo.config.model.api.HostProvisioner; import com.yahoo.config.provision.*; import java.util.List; +import java.util.Optional; import java.util.stream.Collectors; /** - * Host provisioning from an existing {@link ProvisionInfo} instance. + * Host provisioning from an existing {@link AllocatedHosts} instance. * * @author bratseth */ public class StaticProvisioner implements HostProvisioner { - private final ProvisionInfo provisionInfo; + private final AllocatedHosts allocatedHosts; + + /** The fallback provisioner to use for unknown clusters, or null to not fall back */ + private final HostProvisioner fallback; - public StaticProvisioner(ProvisionInfo provisionInfo) { - this.provisionInfo = provisionInfo; + /** + * Creates a static host provisioner with no fallback + */ + public StaticProvisioner(AllocatedHosts allocatedHosts) { + this(allocatedHosts, null); + } + + /** + * Creates a static host provisioner which will fall back to using the given provisioner + * if a request is made for nodes in a cluster which is not present in this allocation. + */ + public StaticProvisioner(AllocatedHosts allocatedHosts, HostProvisioner fallback) { + this.allocatedHosts = allocatedHosts; + this.fallback = fallback; } @Override @@ -27,9 +43,14 @@ public class StaticProvisioner implements HostProvisioner { @Override public List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, int groups, ProvisionLogger logger) { - return provisionInfo.getHosts().stream() - .filter(host -> host.membership().isPresent() && matches(host.membership().get().cluster(), cluster)) - .collect(Collectors.toList()); + List<HostSpec> hostsAlreadyAllocatedToCluster = + allocatedHosts.getHosts().stream() + .filter(host -> host.membership().isPresent() && matches(host.membership().get().cluster(), cluster)) + .collect(Collectors.toList()); + if ( ! hostsAlreadyAllocatedToCluster.isEmpty()) + return hostsAlreadyAllocatedToCluster; + else + return fallback.prepare(cluster, capacity, groups, logger); } private boolean matches(ClusterSpec nodeCluster, ClusterSpec requestedCluster) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java index a222182f87a..308ca31f278 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java @@ -6,7 +6,7 @@ import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.ApplicationMetaData; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.config.provision.ProvisionInfo; +import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.transaction.AbstractTransaction; import com.yahoo.transaction.NestedTransaction; import com.yahoo.transaction.Transaction; @@ -30,11 +30,10 @@ import java.util.Optional; * prepared. Deleting a local session will ensure that the local filesystem state and global zookeeper state is * cleaned for this session. * - * @author lulf - * @since 5.1 + * @author Ulf Lilleengen */ // This is really the store of an application, whether it is active or in an edit session -// TODO: Separate the "application store" and "session" aspects - the latter belongs in the HTTP layer +// TODO: Separate the "application store" and "session" aspects - the latter belongs in the HTTP layer -bratseth public class LocalSession extends Session implements Comparable<LocalSession> { private final ApplicationPackage applicationPackage; @@ -172,8 +171,8 @@ public class LocalSession extends Session implements Comparable<LocalSession> { public Version getVespaVersion() { return zooKeeperClient.readVespaVersion(); } - public ProvisionInfo getProvisionInfo() { - return zooKeeperClient.getProvisionInfo(); + public AllocatedHosts getAllocatedHosts() { + return zooKeeperClient.getAllocatedHosts(); } @Override diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionRepo.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionRepo.java index 7fff132dd7d..ae15be83062 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionRepo.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionRepo.java @@ -18,7 +18,6 @@ import java.util.logging.Logger; * File-based session repository for LocalSessions. Contains state for the local instance of the configserver. * * @author lulf - * @since 5.1 */ public class LocalSessionRepo extends SessionRepo<LocalSession> { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java index c20f6e0b853..5f3cf46e0de 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.config.server.session; import com.yahoo.config.provision.*; +import com.yahoo.lang.SettableOptional; import com.yahoo.vespa.config.server.*; import com.yahoo.log.LogLevel; import com.yahoo.vespa.config.server.application.ApplicationSet; @@ -19,8 +20,7 @@ import java.util.logging.Logger; * A RemoteSession represents a session created on another config server. This session can * be regarded as read only, and this interface only allows reading information about a session. * - * @author lulf - * @since 5.1 + * @author Ulf Lilleengen */ public class RemoteSession extends Session { @@ -57,6 +57,7 @@ public class RemoteSession extends Session { return ApplicationSet.fromList(applicationLoader.buildModels(zooKeeperClient.readApplicationId(), zooKeeperClient.readVespaVersion(), zooKeeperClient.loadApplicationPackage(), + new SettableOptional<>(), clock.instant())); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionFactory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionFactory.java index 0f1e65aaf2b..298acaca901 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionFactory.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionFactory.java @@ -12,8 +12,7 @@ import com.yahoo.vespa.curator.Curator; import java.time.Clock; /** - * @author lulf - * @since 5.1.24 + * @author Ulf Lilleengen */ public class RemoteSessionFactory { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionContext.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionContext.java index a840e5d5a97..323c2667d30 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionContext.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionContext.java @@ -12,8 +12,7 @@ import java.io.File; /** * The dependencies needed for a local session to be edited and prepared. * - * @author lulf - * @since 5.1 + * @author Ulf Lilleengen */ public class SessionContext { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java index 19651041ea4..c2655d1a1b3 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java @@ -170,7 +170,8 @@ public class SessionFactoryImpl implements SessionFactory, LocalSessionLoader { defRepo, serverId, nodeFlavors); - SessionContext context = new SessionContext(applicationPackage, sessionZKClient, sessionDir, applicationRepo, hostRegistry, superModelGenerationCounter); + SessionContext context = new SessionContext(applicationPackage, sessionZKClient, sessionDir, applicationRepo, + hostRegistry, superModelGenerationCounter); return new LocalSession(tenant, sessionId, sessionPreparer, context); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java index 0b76c57e142..beb62cf3ac9 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java @@ -10,6 +10,7 @@ import com.yahoo.config.application.api.FileRegistry; import com.yahoo.config.model.api.ConfigDefinitionRepo; import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.provision.*; +import com.yahoo.lang.SettableOptional; import com.yahoo.log.LogLevel; import com.yahoo.path.Path; import com.yahoo.vespa.config.server.application.ApplicationSet; @@ -42,8 +43,7 @@ import javax.xml.transform.TransformerException; /** * A SessionPreparer is responsible for preparing a session given an application package. * - * @author lulf - * @since 5.1 + * @author Ulf Lilleengen */ public class SessionPreparer { @@ -92,8 +92,8 @@ public class SessionPreparer { Preparation preparation = new Preparation(context, logger, params, currentActiveApplicationSet, tenantPath); preparation.preprocess(); try { - preparation.buildModels(now); - preparation.makeResult(); + AllocatedHosts allocatedHosts = preparation.buildModels(now); + preparation.makeResult(allocatedHosts); if ( ! params.isDryRun()) { preparation.writeStateZK(); preparation.writeRotZK(); @@ -177,13 +177,16 @@ public class SessionPreparer { checkTimeout("preprocess"); } - void buildModels(Instant now) { - this.modelResultList = preparedModelsBuilder.buildModels(applicationId, vespaVersion, applicationPackage, now); + AllocatedHosts buildModels(Instant now) { + SettableOptional<AllocatedHosts> allocatedHosts = new SettableOptional<>(); + this.modelResultList = preparedModelsBuilder.buildModels(applicationId, vespaVersion, + applicationPackage, allocatedHosts, now); checkTimeout("build models"); + return allocatedHosts.get(); } - void makeResult() { - this.prepareResult = new PrepareResult(modelResultList); + void makeResult(AllocatedHosts allocatedHosts) { + this.prepareResult = new PrepareResult(allocatedHosts, modelResultList); checkTimeout("making result from models"); } @@ -195,7 +198,7 @@ public class SessionPreparer { vespaVersion, logger, prepareResult.getFileRegistries(), - prepareResult.getProvisionInfos()); + prepareResult.allocatedHosts()); checkTimeout("write state to zookeeper"); } @@ -236,10 +239,10 @@ public class SessionPreparer { com.yahoo.component.Version vespaVersion, DeployLogger deployLogger, Map<Version, FileRegistry> fileRegistryMap, - Map<Version, ProvisionInfo> provisionInfoMap) { + AllocatedHosts allocatedHosts) { ZooKeeperDeployer zkDeployer = zooKeeperClient.createDeployer(deployLogger); try { - zkDeployer.deploy(applicationPackage, fileRegistryMap, provisionInfoMap); + zkDeployer.deploy(applicationPackage, fileRegistryMap, allocatedHosts); zooKeeperClient.writeApplicationId(applicationId); zooKeeperClient.writeVespaVersion(vespaVersion); } catch (RuntimeException | IOException e) { @@ -251,21 +254,19 @@ public class SessionPreparer { /** The result of preparation over all model versions */ private static class PrepareResult { + private final AllocatedHosts allocatedHosts; private final ImmutableList<PreparedModelsBuilder.PreparedModelResult> results; - - public PrepareResult(List<PreparedModelsBuilder.PreparedModelResult> results) { + + public PrepareResult(AllocatedHosts allocatedHosts, List<PreparedModelsBuilder.PreparedModelResult> results) { + this.allocatedHosts = allocatedHosts; this.results = ImmutableList.copyOf(results); } /** Returns the results for each model as an immutable list */ public List<PreparedModelsBuilder.PreparedModelResult> asList() { return results; } - public Map<Version, ProvisionInfo> getProvisionInfos() { - return results.stream() - .filter(result -> result.model.getProvisionInfo().isPresent()) - .collect(Collectors.toMap((prepareResult -> prepareResult.version), - (prepareResult -> prepareResult.model.getProvisionInfo().get()))); - } + /** Returns the host allocations resulting from this preparation. */ + public AllocatedHosts allocatedHosts() { return allocatedHosts; } public Map<Version, FileRegistry> getFileRegistries() { return results.stream() @@ -290,4 +291,31 @@ public class SessionPreparer { } + /** + * During model building each model version will request nodes allocated (from the node allocator) + * for each cluster specified by that model. As allocations are stable this should usually + * result in the same allocations for the same clusters across all model versions, + * otherwise we should fail this preparation as such inconsistencies lead to undefined behavior, + * and there is really just one true allocation (for a given cluster) to be activated in the node repository. + * + * However, these disagreements between allocations in each model version are allowed: + * - A node may be retired in some model version but not another. This allows model versions to change cluster sizes, + * and is ok because the system will converge on the latest version's opinion + * - Clusters may be present on some version but not on another. This does not lead to inconsistency + * and allows new model versions to introduce new clusters. + * + * For each cluster, the newest model version which has that cluster decides the correct retirement status of nodes + * (and all model versions having the cluster must have the same nodes). + * + * This class ensures these constraints and returns a reconciliated set of nodes which should be activated, + * given a set of model activation results. + */ + private static final class ReconciliatedHostAllocations { + + public ReconciliatedHostAllocations(List<PreparedModelsBuilder.PreparedModelResult> results) { + + } + + } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java index a677c5cb7f9..09fc83e225d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java @@ -6,8 +6,7 @@ import com.yahoo.component.Vtag; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.provision.NodeFlavors; -import com.yahoo.config.provision.ProvisionInfo; -import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.transaction.Transaction; import com.yahoo.log.LogLevel; import com.yahoo.path.Path; @@ -31,8 +30,7 @@ import java.util.concurrent.TimeUnit; * Zookeeper client for a specific session. Can be used to read and write session status * and create and get prepare and active barrier. * - * @author lulf - * @since 5.1 + * @author Ulf Lilleengen */ public class SessionZooKeeperClient { @@ -189,10 +187,9 @@ public class SessionZooKeeperClient { return rootPath.append(CREATE_TIME_PATH).getAbsolute(); } - ProvisionInfo getProvisionInfo() { - return loadApplicationPackage().getProvisionInfoMap().values().stream() - .reduce((infoA, infoB) -> infoA.merge(infoB)) - .orElseThrow(() -> new IllegalStateException("Trying to read provision info, but no provision info exists")); + AllocatedHosts getAllocatedHosts() { + return loadApplicationPackage().getAllocatedHosts() + .orElseThrow(() -> new IllegalStateException("Allocated hosts does not exists")); } public ZooKeeperDeployer createDeployer(DeployLogger logger) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantBuilder.java index 144bbf10dc6..f4a5e941ac2 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantBuilder.java @@ -27,8 +27,7 @@ import java.util.concurrent.Executors; /** * Builder for helping out with tenant creation. Each of a tenants dependencies may be overridden for testing. * - * @author lulf - * @since 5.1 + * @author Ulf Lilleengen */ public class TenantBuilder { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenants.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenants.java index 28bae587e2a..377b3997e43 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenants.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/Tenants.java @@ -48,12 +48,12 @@ import java.util.logging.Logger; * Once a tenant is deleted from zookeeper, the zookeeper watcher thread will get notified on all configservers, and * shutdown and delete any per-configserver state. * - * @author vegardh - * @author lulf - * @since 5.1.26 + * @author Vegard Havdal + * @author Ulf Lilleengen */ //TODO Rename to TenantRepository public class Tenants implements ConnectionStateListener, PathChildrenCacheListener { + public static final TenantName HOSTED_VESPA_TENANT = TenantName.from("hosted-vespa"); private static final TenantName DEFAULT_TENANT = TenantName.defaultName(); private static final List<TenantName> SYSTEM_TENANT_NAMES = Arrays.asList( diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ConfigCurator.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ConfigCurator.java index 80c1c44546f..c09fe5ae4bb 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ConfigCurator.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ConfigCurator.java @@ -146,7 +146,7 @@ public class ConfigCurator { /** * Returns the data at a path, or null if the path does not exist. * - * @param path a String with a pathname. + * @param path a String with a pathname. * @return a byte array with data. */ public byte[] getBytes(String path) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java index 4f591278a38..05c301ddba8 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java @@ -2,6 +2,8 @@ package com.yahoo.vespa.config.server.zookeeper; import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableSet; +import com.yahoo.component.Version; import com.yahoo.config.application.api.ApplicationMetaData; import com.yahoo.config.application.api.ComponentInfo; import com.yahoo.config.application.api.FileRegistry; @@ -10,9 +12,9 @@ import com.yahoo.config.codegen.DefParser; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.application.provider.*; +import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeFlavors; -import com.yahoo.config.provision.ProvisionInfo; -import com.yahoo.config.provision.Version; +import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.io.IOUtils; import com.yahoo.path.Path; import com.yahoo.io.reader.NamedReader; @@ -24,20 +26,26 @@ import com.yahoo.vespa.config.util.ConfigUtils; import java.io.File; import java.io.Reader; import java.io.StringReader; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; /** * Represents an application residing in zookeeper. * - * @author tonytv + * @author Tony Vaagenes */ public class ZKApplicationPackage implements ApplicationPackage { private ZKLiveApp liveApp; - private final Map<Version, PreGeneratedFileRegistry> fileRegistryMap = new HashMap<>(); - private final Map<Version, ProvisionInfo> provisionInfoMap = new HashMap<>(); - private static final Version legacyVersion = Version.fromIntValues(0, 0, 0); + private final Map<com.yahoo.config.provision.Version, PreGeneratedFileRegistry> fileRegistryMap = new HashMap<>(); + private final Optional<AllocatedHosts> allocatedHosts; + private static final com.yahoo.config.provision.Version legacyVersion = com.yahoo.config.provision.Version.fromIntValues(0, 0, 0); public static final String fileRegistryNode = "fileregistry"; public static final String allocatedHostsNode = "allocatedHosts"; @@ -48,34 +56,56 @@ public class ZKApplicationPackage implements ApplicationPackage { liveApp = new ZKLiveApp(zk, appPath); metaData = readMetaDataFromLiveApp(liveApp); importFileRegistries(fileRegistryNode); - importProvisionInfos(allocatedHostsNode, nodeFlavors); + allocatedHosts = importAllocatedHosts(allocatedHostsNode, nodeFlavors); } - private void importProvisionInfos(String allocatedHostsNode, Optional<NodeFlavors> nodeFlavors) { - List<String> provisionInfoNodes = liveApp.getChildren(allocatedHostsNode); - if (provisionInfoNodes.isEmpty()) { - Optional<ProvisionInfo> provisionInfo = importProvisionInfo(allocatedHostsNode, nodeFlavors); - provisionInfo.ifPresent(info -> provisionInfoMap.put(legacyVersion, info)); - } else { - provisionInfoNodes.stream() - .forEach(versionStr -> { - Version version = Version.fromString(versionStr); - Optional<ProvisionInfo> provisionInfo = importProvisionInfo(Joiner.on("/").join(allocatedHostsNode, versionStr), - nodeFlavors); - provisionInfo.ifPresent(info -> provisionInfoMap.put(version, info)); - }); + private Optional<AllocatedHosts> importAllocatedHosts(String allocatedHostsPath, Optional<NodeFlavors> nodeFlavors) { + if ( ! liveApp.exists(allocatedHostsPath)) return Optional.empty(); + Optional<AllocatedHosts> allocatedHosts = readAllocatedHosts(allocatedHostsPath, nodeFlavors); + if ( ! allocatedHosts.isPresent()) { // Read from legacy location. TODO: Remove when 6.143 is in production everywhere + List<String> allocatedHostsByVersionNodes = liveApp.getChildren(allocatedHostsPath); + allocatedHosts = merge(readAllocatedHostsByVersion(allocatedHostsByVersionNodes, nodeFlavors)); + } + return allocatedHosts; + } + + private Map<Version, AllocatedHosts> readAllocatedHostsByVersion(List<String> allocatedHostsByVersionNodes, + Optional<NodeFlavors> nodeFlavors) { + Map<Version, AllocatedHosts> allocatedHostsByVersion = new HashMap<>(); + allocatedHostsByVersionNodes.stream() + .forEach(versionStr -> { + Version version = Version.fromString(versionStr); + Optional<AllocatedHosts> allocatedHosts = readAllocatedHosts(Joiner.on("/").join(allocatedHostsNode, versionStr), + nodeFlavors); + allocatedHosts.ifPresent(info -> allocatedHostsByVersion.put(version, info)); + }); + return allocatedHostsByVersion; + } + + private Optional<AllocatedHosts> merge(Map<Version, AllocatedHosts> allocatedHostsByVersion) { + // Merge the allocated hosts in any order. This is wrong but preserves current behavior (modulo order differences) + if (allocatedHostsByVersion.isEmpty()) return Optional.empty(); + + Map<String, HostSpec> merged = new HashMap<>(); + for (Map.Entry<Version, AllocatedHosts> entry : allocatedHostsByVersion.entrySet()) { + for (HostSpec host : entry.getValue().getHosts()) + merged.put(host.hostname(), host); } + return Optional.of(AllocatedHosts.withHosts(ImmutableSet.copyOf(merged.values()))); } - private Optional<ProvisionInfo> importProvisionInfo(String provisionInfoNode, Optional<NodeFlavors> nodeFlavors) { + /** + * Reads allocated hosts at the given node. + * + * @return the allocated hosts at this node or empty if there is no data at this path + */ + private Optional<AllocatedHosts> readAllocatedHosts(String allocatedHostsPath, Optional<NodeFlavors> nodeFlavors) { try { - if (liveApp.exists(provisionInfoNode)) { - return Optional.of(ProvisionInfo.fromJson(liveApp.getBytes(provisionInfoNode), nodeFlavors)); - } else { - return Optional.empty(); - } + byte[] data = liveApp.getBytes(allocatedHostsPath); + if (data.length == 0) return Optional.empty(); // TODO: Remove this line (and make return non-optional) when 6.143 is in production everywhere + return Optional.of(AllocatedHosts.fromJson(data, nodeFlavors)); } catch (Exception e) { - throw new RuntimeException("Unable to read provision info", e); + throw new RuntimeException("Unable to read allocated hosts", e); } } @@ -85,9 +115,9 @@ public class ZKApplicationPackage implements ApplicationPackage { fileRegistryMap.put(legacyVersion, importFileRegistry(fileRegistryNode)); } else { fileRegistryNodes.stream() - .forEach(versionStr -> { - Version version = Version.fromString(versionStr); - fileRegistryMap.put(version, importFileRegistry(Joiner.on("/").join(fileRegistryNode, versionStr))); + .forEach(version -> { + fileRegistryMap.put(com.yahoo.config.provision.Version.fromString(version), + importFileRegistry(Joiner.on("/").join(fileRegistryNode, version))); }); } } @@ -147,16 +177,17 @@ public class ZKApplicationPackage implements ApplicationPackage { return ret; } - public Map<Version, ProvisionInfo> getProvisionInfoMap() { - return Collections.unmodifiableMap(provisionInfoMap); + @Override + public Optional<AllocatedHosts> getAllocatedHosts() { + return allocatedHosts; } @Override - public Map<Version, FileRegistry> getFileRegistryMap() { + public Map<com.yahoo.config.provision.Version, FileRegistry> getFileRegistryMap() { return Collections.unmodifiableMap(fileRegistryMap); } - private Optional<PreGeneratedFileRegistry> getPreGeneratedFileRegistry(Version vespaVersion) { + private Optional<PreGeneratedFileRegistry> getPreGeneratedFileRegistry(com.yahoo.config.provision.Version vespaVersion) { // Assumes at least one file registry, which we always have. Optional<PreGeneratedFileRegistry> fileRegistry = Optional.ofNullable(fileRegistryMap.get(vespaVersion)); if (!fileRegistry.isPresent()) { @@ -243,7 +274,7 @@ public class ZKApplicationPackage implements ApplicationPackage { } @Override - public List<ComponentInfo> getComponentsInfo(Version vespaVersion) { + public List<ComponentInfo> getComponentsInfo(com.yahoo.config.provision.Version vespaVersion) { List<ComponentInfo> components = new ArrayList<>(); PreGeneratedFileRegistry fileRegistry = getPreGeneratedFileRegistry(vespaVersion).get(); for (String path : fileRegistry.getPaths()) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKLiveApp.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKLiveApp.java index 88748cb7689..8084be1cefa 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKLiveApp.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKLiveApp.java @@ -173,14 +173,11 @@ public class ZKLiveApp { * Returns the full list of children (file names) in the given path. * * @param path a path relative to the currently active application - * @return a list of file names + * @return a list of file names, which is empty (never null) if the path does not exist */ public List<String> getChildren(String path) { String fullPath = getFullPath(path); - if (! zk.exists(fullPath)) { - log.fine("ZKApplicationPackage: " + fullPath + " is not a valid dir"); - return Collections.emptyList(); - } + if (! zk.exists(fullPath)) return Collections.emptyList(); return zk.getChildren(fullPath); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ModelStub.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ModelStub.java index b892abc67af..2ccce9727a3 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ModelStub.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ModelStub.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.config.server; import com.yahoo.config.model.api.FileDistribution; import com.yahoo.config.model.api.HostInfo; import com.yahoo.config.model.api.Model; -import com.yahoo.config.provision.ProvisionInfo; +import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.ConfigPayload; import com.yahoo.vespa.config.buildergen.ConfigDefinition; @@ -44,7 +44,7 @@ public class ModelStub implements Model { } @Override - public Optional<ProvisionInfo> getProvisionInfo() { + public AllocatedHosts allocatedHosts() { return null; } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/MockModel.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/MockModel.java index 5806c7991fc..cf5463b7f4c 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/MockModel.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/MockModel.java @@ -7,7 +7,7 @@ import com.yahoo.config.model.api.Model; import com.yahoo.config.model.api.PortInfo; import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.ProvisionInfo; +import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.ConfigPayload; import com.yahoo.vespa.config.buildergen.ConfigDefinition; @@ -93,7 +93,7 @@ class MockModel implements Model { } @Override - public Optional<ProvisionInfo> getProvisionInfo() { + public AllocatedHosts allocatedHosts() { throw new UnsupportedOperationException(); } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java index c9556425dda..5fea15a7b10 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java @@ -20,7 +20,7 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.InstanceName; -import com.yahoo.config.provision.ProvisionInfo; +import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.ProvisionLogger; import com.yahoo.config.provision.Provisioner; import com.yahoo.config.provision.Version; @@ -69,7 +69,7 @@ public class DeployTester { private ApplicationId id; public DeployTester(String appPath) { - this(appPath, Collections.singletonList(createDefaultModelFactory(Clock.systemUTC()))); + this(appPath, Collections.singletonList(createModelFactory(Clock.systemUTC()))); } public DeployTester(String appPath, List<ModelFactory> modelFactories) { @@ -80,7 +80,7 @@ public class DeployTester { public DeployTester(String appPath, ConfigserverConfig configserverConfig) { this(appPath, - Collections.singletonList(createDefaultModelFactory(Clock.systemUTC())), + Collections.singletonList(createModelFactory(Clock.systemUTC())), configserverConfig); } @@ -99,9 +99,16 @@ public class DeployTester { public Tenant tenant() { return tenants.defaultTenant(); } - /** Create the model factory which will be used in production */ - public static ModelFactory createDefaultModelFactory(Clock clock) { return new VespaModelFactory(new NullConfigModelRegistry(), clock); } - + /** Create a model factory for the version of this source*/ + public static ModelFactory createModelFactory(Clock clock) { + return new VespaModelFactory(new NullConfigModelRegistry(), clock); + } + + /** Create a model factory for a particular version */ + public static ModelFactory createModelFactory(Version version, Clock clock) { + return new VespaModelFactory(version, new NullConfigModelRegistry(), clock); + } + /** Create a model factory which always fails validation */ public static ModelFactory createFailingModelFactory(Version version) { return new FailingModelFactory(version); } @@ -109,20 +116,19 @@ public class DeployTester { * Do the initial "deploy" with the existing API-less code as the deploy API doesn't support first deploys yet. */ public ApplicationId deployApp(String appName, Instant now) { - return deployApp(appName, Optional.empty(), now); + return deployApp(appName, null, now); } /** * Do the initial "deploy" with the existing API-less code as the deploy API doesn't support first deploys yet. */ - public ApplicationId deployApp(String appName, Optional<String> vespaVersion, Instant now) { + public ApplicationId deployApp(String appName, String vespaVersion, Instant now) { Tenant tenant = tenant(); LocalSession session = tenant.getSessionFactory().createSession(testApp, appName, new TimeoutBudget(Clock.systemUTC(), Duration.ofSeconds(60))); ApplicationId id = ApplicationId.from(tenant.getName(), ApplicationName.from(appName), InstanceName.defaultName()); - PrepareParams.Builder paramsBuilder = new PrepareParams.Builder() - .applicationId(id); - if (vespaVersion.isPresent()) - paramsBuilder.vespaVersion(vespaVersion.get()); + PrepareParams.Builder paramsBuilder = new PrepareParams.Builder().applicationId(id); + if (vespaVersion != null) + paramsBuilder.vespaVersion(vespaVersion); session.prepare(new SilentDeployLogger(), paramsBuilder.build(), Optional.empty(), @@ -134,11 +140,11 @@ public class DeployTester { return id; } - public ProvisionInfo getProvisionInfoFromDeployedApp(ApplicationId applicationId) { + public AllocatedHosts getAllocatedHostsOf(ApplicationId applicationId) { Tenant tenant = tenant(); LocalSession session = tenant.getLocalSessionRepo().getSession(tenant.getApplicationRepo() .getSessionIdForApplication(applicationId)); - return session.getProvisionInfo(); + return session.getAllocatedHosts(); } public ApplicationId applicationId() { return id; } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java index 31e92cc9f93..301ae63fb8c 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java @@ -32,7 +32,7 @@ public class HostedDeployTest { @Test public void testRedeployWithVersion() throws InterruptedException, IOException { DeployTester tester = new DeployTester("src/test/apps/hosted/", createConfigserverConfig()); - tester.deployApp("myApp", Optional.of("4.5.6"), Instant.now()); + tester.deployApp("myApp", "4.5.6", Instant.now()); Optional<com.yahoo.config.provision.Deployment> deployment = tester.redeployFromLocalActive(); assertTrue(deployment.isPresent()); @@ -53,11 +53,23 @@ public class HostedDeployTest { } @Test + public void testDeployMultipleVersions() throws InterruptedException, IOException { + ManualClock clock = new ManualClock("2016-10-09T00:00:00"); + List<ModelFactory> modelFactories = new ArrayList<>(); + modelFactories.add(DeployTester.createModelFactory(Version.fromString("6.1.0"), clock)); + modelFactories.add(DeployTester.createModelFactory(Version.fromString("6.2.0"), clock)); + modelFactories.add(DeployTester.createModelFactory(Version.fromString("7.0.0"), clock)); + DeployTester tester = new DeployTester("src/test/apps/hosted/", modelFactories, createConfigserverConfig()); + ApplicationId app = tester.deployApp("myApp", Instant.now()); + assertEquals(3, tester.getAllocatedHostsOf(app).getHosts().size()); + } + + @Test public void testRedeployAfterExpiredValidationOverride() throws InterruptedException, IOException { // Old version of model fails, but application disables loading old models until 2016-10-10, so deployment works ManualClock clock = new ManualClock("2016-10-09T00:00:00"); List<ModelFactory> modelFactories = new ArrayList<>(); - modelFactories.add(DeployTester.createDefaultModelFactory(clock)); + modelFactories.add(DeployTester.createModelFactory(clock)); modelFactories.add(DeployTester.createFailingModelFactory(Version.fromIntValues(1, 0, 0))); // older than default DeployTester tester = new DeployTester("src/test/apps/validationOverride/", modelFactories, createConfigserverConfig()); tester.deployApp("myApp", clock.instant()); @@ -97,19 +109,19 @@ public class HostedDeployTest { public void testDeployWithDockerImage() throws InterruptedException, IOException { final String vespaVersion = "6.51.1"; DeployTester tester = new DeployTester("src/test/apps/hosted/", createConfigserverConfig()); - ApplicationId applicationId = tester.deployApp("myApp", Optional.of(vespaVersion), Instant.now()); - assertProvisionInfo(vespaVersion, tester, applicationId); + ApplicationId applicationId = tester.deployApp("myApp", vespaVersion, Instant.now()); + assertAllocatedHosts(vespaVersion, tester, applicationId); System.out.println("Redeploy"); Optional<com.yahoo.config.provision.Deployment> deployment = tester.redeployFromLocalActive(); assertTrue(deployment.isPresent()); deployment.get().prepare(); deployment.get().activate(); - //assertProvisionInfo(vespaVersion, tester, applicationId); + //assertAllocatedHosts(vespaVersion, tester, applicationId); } - private void assertProvisionInfo(String vespaVersion, DeployTester tester, ApplicationId applicationId) { - tester.getProvisionInfoFromDeployedApp(applicationId).getHosts().stream() + private void assertAllocatedHosts(String vespaVersion, DeployTester tester, ApplicationId applicationId) { + tester.getAllocatedHostsOf(applicationId).getHosts().stream() .forEach(h -> assertEquals(vespaVersion, h.membership().get().cluster().vespaVersion())); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/RedeployTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/RedeployTest.java index 5658e0fb2aa..49e40321321 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/RedeployTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/RedeployTest.java @@ -47,7 +47,7 @@ public class RedeployTest { @Test public void testNoRedeploy() { List<ModelFactory> modelFactories = new ArrayList<>(); - modelFactories.add(DeployTester.createDefaultModelFactory(Clock.systemUTC())); + modelFactories.add(DeployTester.createModelFactory(Clock.systemUTC())); modelFactories.add(DeployTester.createFailingModelFactory(Version.fromIntValues(1, 0, 0))); DeployTester tester = new DeployTester("ignored/app/path", modelFactories); ApplicationId id = ApplicationId.from(TenantName.from("default"), diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java index ac029c12b17..bf7f7038c1a 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.deploy; +import com.google.common.collect.ImmutableSet; import com.yahoo.config.application.api.ApplicationMetaData; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.FileRegistry; @@ -46,8 +47,8 @@ public class ZooKeeperClientTest extends TestWithCurator { Map<Version, FileRegistry> fileRegistries = createFileRegistries(); app.writeMetaData(); zkc.setupZooKeeper(); - zkc.feedZooKeeper(app); - zkc.feedZKFileRegistries(fileRegistries); + zkc.write(app); + zkc.write(fileRegistries); } private Map<Version, FileRegistry> createFileRegistries() { @@ -174,23 +175,15 @@ public class ZooKeeperClientTest extends TestWithCurator { Path app = Path.fromString("/1"); ZooKeeperClient zooKeeperClient = new ZooKeeperClient(zk, logger, true, app); zooKeeperClient.setupZooKeeper(); - zooKeeperClient.feedProvisionInfos(createProvisionInfos()); + HostSpec host1 = new HostSpec("host1.yahoo.com", Collections.emptyList()); + HostSpec host2 = new HostSpec("host2.yahoo.com", Collections.emptyList()); + ImmutableSet<HostSpec> hosts = ImmutableSet.of(host1, host2); + zooKeeperClient.write(AllocatedHosts.withHosts(hosts)); Path hostsPath = app.append(ZKApplicationPackage.allocatedHostsNode); assertTrue(zk.exists(hostsPath.getAbsolute())); - assertEquals(0, zk.getBytes(hostsPath.getAbsolute()).length); // Changed from null - assertTrue(zk.exists(hostsPath.append("1.2.3").getAbsolute())); - assertTrue(zk.exists(hostsPath.append("3.2.1").getAbsolute())); - assertTrue(zk.getBytes(hostsPath.append("1.2.3").getAbsolute()).length > 0); - assertTrue(zk.getBytes(hostsPath.append("3.2.1").getAbsolute()).length > 0); - } - - private Map<Version, ProvisionInfo> createProvisionInfos() { - Map<Version, ProvisionInfo> provisionInfoMap = new HashMap<>(); - ProvisionInfo a = ProvisionInfo.withHosts(Collections.singleton(new HostSpec("host.yahoo.com", Collections.emptyList()))); - ProvisionInfo b = ProvisionInfo.withHosts(Collections.singleton(new HostSpec("host2.yahoo.com", Collections.emptyList()))); - provisionInfoMap.put(Version.fromIntValues(1, 2, 3), a); - provisionInfoMap.put(Version.fromIntValues(3, 2, 1), b); - return provisionInfoMap; + + AllocatedHosts deserialized = AllocatedHosts.fromJson(zk.getBytes(hostsPath.getAbsolute()), Optional.empty()); + assertEquals(hosts, deserialized.getHosts()); } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperDeployerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperDeployerTest.java index 9c7f12c2147..b256079d259 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperDeployerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperDeployerTest.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.config.server.deploy; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.application.provider.*; +import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.Version; import com.yahoo.io.IOUtils; import com.yahoo.path.Path; @@ -52,7 +53,7 @@ public class ZooKeeperDeployerTest { ZooKeeperClient client = new ZooKeeperClient(configCurator, logger, true, appPath); ZooKeeperDeployer deployer = new ZooKeeperDeployer(client); - deployer.deploy(applicationPackage, Collections.singletonMap(Version.fromIntValues(1, 0, 0), new MockFileRegistry()), Collections.emptyMap()); + deployer.deploy(applicationPackage, Collections.singletonMap(Version.fromIntValues(1, 0, 0), new MockFileRegistry()), AllocatedHosts.withHosts(Collections.emptySet())); assertTrue(configCurator.exists(appPath.getAbsolute())); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java index 5e10f364aeb..7fca78b087b 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java @@ -13,7 +13,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.InstanceName; -import com.yahoo.config.provision.ProvisionInfo; +import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; import com.yahoo.container.jdisc.HttpResponse; @@ -217,8 +217,8 @@ public class SessionActiveHandlerTest extends SessionHandlerTest { zkClient.writeStatus(status); ZooKeeperClient zkC = new ZooKeeperClient(configCurator, new BaseDeployLogger(), false, pathProvider.getSessionDirs().append(String.valueOf(sessionId))); VespaModelFactory modelFactory = new VespaModelFactory(new NullConfigModelRegistry()); - zkC.feedZKFileRegistries(Collections.singletonMap(modelFactory.getVersion(), new MockFileRegistry())); - zkC.feedProvisionInfos(Collections.singletonMap(modelFactory.getVersion(), ProvisionInfo.withHosts(Collections.emptySet()))); + zkC.write(Collections.singletonMap(modelFactory.getVersion(), new MockFileRegistry())); + zkC.write(AllocatedHosts.withHosts(Collections.emptySet())); TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder() .curator(curator) .configCurator(configCurator) @@ -318,7 +318,7 @@ public class SessionActiveHandlerTest extends SessionHandlerTest { ActivateRequest invoke(boolean createLocalSession) throws Exception { SessionZooKeeperClient zkClient = new MockSessionZKClient(curator, pathProvider.getSessionDirs().append(String.valueOf(sessionId)), - Optional.of(ProvisionInfo.withHosts(Collections.singleton(new HostSpec("bar", Collections.emptyList()))))); + Optional.of(AllocatedHosts.withHosts(Collections.singleton(new HostSpec("bar", Collections.emptyList()))))); session = createRemoteSession(sessionId, initialStatus, zkClient, clock); if (createLocalSession) { LocalSessionRepo repo = addLocalSession(sessionId, deployData, zkClient); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/provision/StaticProvisionerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/provision/StaticProvisionerTest.java index c54514eb097..badcdf53b77 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/provision/StaticProvisionerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/provision/StaticProvisionerTest.java @@ -20,7 +20,7 @@ import java.io.IOException; import static org.junit.Assert.assertEquals; /** - * @author lulf + * @author Ulf Lilleengen */ public class StaticProvisionerTest { @@ -30,7 +30,7 @@ public class StaticProvisionerTest { InMemoryProvisioner inMemoryHostProvisioner = new InMemoryProvisioner(false, "host1.yahoo.com", "host2.yahoo.com", "host3.yahoo.com", "host4.yahoo.com"); VespaModel firstModel = createModel(app, inMemoryHostProvisioner); - StaticProvisioner staticProvisioner = new StaticProvisioner(firstModel.getProvisionInfo().get()); + StaticProvisioner staticProvisioner = new StaticProvisioner(firstModel.allocatedHosts()); VespaModel secondModel = createModel(app, staticProvisioner); assertModelConfig(firstModel, secondModel); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java index f0086eabd26..be98f41c82a 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java @@ -120,19 +120,19 @@ public class LocalSessionTest { @Test(expected = IllegalStateException.class) public void require_that_no_provision_info_throws_exception() throws Exception { - createSession(TenantName.defaultName(), 3).getProvisionInfo(); + createSession(TenantName.defaultName(), 3).getAllocatedHosts(); } @Test public void require_that_provision_info_can_be_read() throws Exception { - ProvisionInfo input = ProvisionInfo.withHosts(Collections.singleton(new HostSpec("myhost", Collections.<String>emptyList()))); + AllocatedHosts input = AllocatedHosts.withHosts(Collections.singleton(new HostSpec("myhost", Collections.<String>emptyList()))); LocalSession session = createSession(TenantName.defaultName(), 3, new SessionTest.MockSessionPreparer(), Optional.of(input)); ApplicationId origId = new ApplicationId.Builder() .tenant("tenant") .applicationName("foo").instanceName("quux").build(); doPrepare(session, new PrepareParams.Builder().applicationId(origId).build(), Instant.now()); - ProvisionInfo info = session.getProvisionInfo(); + AllocatedHosts info = session.getAllocatedHosts(); assertNotNull(info); assertThat(info.getHosts().size(), is(1)); assertTrue(info.getHosts().contains(new HostSpec("myhost", Collections.emptyList()))); @@ -151,18 +151,18 @@ public class LocalSessionTest { } private LocalSession createSession(TenantName tenant, long sessionId, SessionTest.MockSessionPreparer preparer) throws Exception { - return createSession(tenant, sessionId, preparer, Optional.<ProvisionInfo>empty()); + return createSession(tenant, sessionId, preparer, Optional.<AllocatedHosts>empty()); } - private LocalSession createSession(TenantName tenant, long sessionId, SessionTest.MockSessionPreparer preparer, Optional<ProvisionInfo> provisionInfo) throws Exception { + private LocalSession createSession(TenantName tenant, long sessionId, SessionTest.MockSessionPreparer preparer, Optional<AllocatedHosts> allocatedHosts) throws Exception { Path appPath = Path.fromString("/" + sessionId); - SessionZooKeeperClient zkc = new MockSessionZKClient(curator, appPath, provisionInfo); + SessionZooKeeperClient zkc = new MockSessionZKClient(curator, appPath, allocatedHosts); zkc.createWriteStatusTransaction(Session.Status.NEW).commit(); ZooKeeperClient zkClient = new ZooKeeperClient(configCurator, new BaseDeployLogger(), false, appPath); - if (provisionInfo.isPresent()) { - zkClient.feedProvisionInfos(Collections.singletonMap(Version.fromIntValues(0, 0, 0), provisionInfo.get())); + if (allocatedHosts.isPresent()) { + zkClient.write(allocatedHosts.get()); } - zkClient.feedZKFileRegistries(Collections.singletonMap(Version.fromIntValues(0, 0, 0), new MockFileRegistry())); + zkClient.write(Collections.singletonMap(Version.fromIntValues(0, 0, 0), new MockFileRegistry())); File sessionDir = new File(tenantFileSystemDirs.path(), String.valueOf(sessionId)); sessionDir.createNewFile(); return new LocalSession(tenant, sessionId, preparer, new SessionContext(FilesApplicationPackage.fromFile(testApp), zkc, sessionDir, new MemoryTenantApplications(), new HostRegistry<>(), superModelGenerationCounter)); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/MockSessionZKClient.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/MockSessionZKClient.java index 9b658a807b9..412e7881a26 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/MockSessionZKClient.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/MockSessionZKClient.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.config.server.session; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.test.MockApplicationPackage; -import com.yahoo.config.provision.ProvisionInfo; +import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.transaction.Transaction; import com.yahoo.path.Path; import com.yahoo.vespa.curator.Curator; @@ -14,22 +14,21 @@ import java.util.Optional; /** * Overrides application package fetching, because this part is hard to do without feeding a full app. * - * @author lulf - * @since 5.1 + * @author Ulf Lilleengen */ public class MockSessionZKClient extends SessionZooKeeperClient { private ApplicationPackage app = null; - private Optional<ProvisionInfo> info = null; + private Optional<AllocatedHosts> info = null; private Session.Status sessionStatus; public MockSessionZKClient(Curator curator, Path rootPath) { this(curator, rootPath, (ApplicationPackage)null); } - public MockSessionZKClient(Curator curator, Path rootPath, Optional<ProvisionInfo> provisionInfo) { + public MockSessionZKClient(Curator curator, Path rootPath, Optional<AllocatedHosts> allocatedHosts) { this(curator, rootPath); - this.info = provisionInfo; + this.info = allocatedHosts; } public MockSessionZKClient(Curator curator, Path rootPath, ApplicationPackage application) { @@ -49,8 +48,8 @@ public class MockSessionZKClient extends SessionZooKeeperClient { } @Override - ProvisionInfo getProvisionInfo() { - return info.orElseThrow(() -> new IllegalStateException("Trying to read provision info, but no provision info exists")); + AllocatedHosts getAllocatedHosts() { + return info.orElseThrow(() -> new IllegalStateException("Could not find allocated hosts")); } @Override diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandlerTest.java index 65f546e149a..3b67597c43c 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandlerTest.java @@ -8,6 +8,7 @@ import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.config.model.application.provider.MockFileRegistry; import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.Version; import com.yahoo.io.IOUtils; import com.yahoo.path.Path; @@ -94,7 +95,7 @@ public class TenantRequestHandlerTest extends TestWithCurator { File app = tempFolder.newFolder(); IOUtils.copyDirectory(appDir, app); ZooKeeperDeployer deployer = zkc.createDeployer(new BaseDeployLogger()); - deployer.deploy(FilesApplicationPackage.fromFile(appDir), Collections.singletonMap(vespaVersion, new MockFileRegistry()), Collections.emptyMap()); + deployer.deploy(FilesApplicationPackage.fromFile(appDir), Collections.singletonMap(vespaVersion, new MockFileRegistry()), AllocatedHosts.withHosts(Collections.emptySet())); } private ApplicationSet reloadConfig(long id, Clock clock) { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java index abc77a91c51..adf26dbfa32 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java @@ -16,7 +16,7 @@ import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeFlavors; -import com.yahoo.config.provision.ProvisionInfo; +import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.Version; import com.yahoo.config.provisioning.FlavorsConfig; import com.yahoo.path.Path; @@ -33,7 +33,7 @@ public class ZKApplicationPackageTest extends TestWithCurator { private static final String APP = "src/test/apps/zkapp"; private static final String TEST_FLAVOR_NAME = "test-flavor"; private static final Optional<Flavor> TEST_FLAVOR = new MockNodeFlavors().getFlavor(TEST_FLAVOR_NAME); - private static final ProvisionInfo provisionInfo = ProvisionInfo.withHosts( + private static final AllocatedHosts ALLOCATED_HOSTS = AllocatedHosts.withHosts( Collections.singleton(new HostSpec("foo.yahoo.com", Collections.emptyList(), TEST_FLAVOR, Optional.empty()))); @Rule @@ -64,9 +64,8 @@ public class ZKApplicationPackageTest extends TestWithCurator { assertTrue(zkApp.getFileRegistryMap().containsKey(goodVersion)); assertFalse(zkApp.getFileRegistryMap().containsKey(Version.fromIntValues(0, 0, 0))); assertThat(zkApp.getFileRegistryMap().get(goodVersion).fileSourceHost(), is("dummyfiles")); - assertTrue(zkApp.getProvisionInfoMap().containsKey(goodVersion)); - ProvisionInfo readInfo = zkApp.getProvisionInfoMap().get(goodVersion); - assertThat(Utf8.toString(readInfo.toJson()), is(Utf8.toString(provisionInfo.toJson()))); + AllocatedHosts readInfo = zkApp.getAllocatedHosts().get(); + assertThat(Utf8.toString(readInfo.toJson()), is(Utf8.toString(ALLOCATED_HOSTS.toJson()))); assertThat(readInfo.getHosts().iterator().next().flavor(), is(TEST_FLAVOR)); assertTrue(zkApp.getDeployment().isPresent()); assertThat(DeploymentSpec.fromXml(zkApp.getDeployment().get()).globalServiceId().get(), is("mydisc")); @@ -78,7 +77,7 @@ public class ZKApplicationPackageTest extends TestWithCurator { String metaData = "{\"deploy\":{\"user\":\"foo\",\"from\":\"bar\",\"timestamp\":1},\"application\":{\"name\":\"foo\",\"checksum\":\"abc\",\"generation\":4,\"previousActiveGeneration\":3}}"; zk.putData("/0", ConfigCurator.META_ZK_PATH, metaData); zk.putData("/0/" + ZKApplicationPackage.fileRegistryNode + "/3.0.0", "dummyfiles"); - zk.putData("/0/" + ZKApplicationPackage.allocatedHostsNode + "/3.0.0", provisionInfo.toJson()); + zk.putData("/0/" + ZKApplicationPackage.allocatedHostsNode + "/3.0.0", ALLOCATED_HOSTS.toJson()); } private static class MockNodeFlavors extends NodeFlavors{ diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java index fa2ee8e89a9..e59f012856a 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java @@ -42,6 +42,7 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Level; import java.util.logging.Logger; @@ -79,7 +80,7 @@ public final class ConfiguredApplication implements Application { new ComponentRegistry<>(), new ComponentRegistry<>()); private final OsgiFramework restrictedOsgiFramework; - private volatile int applicationSerialNo = 0; + private final AtomicInteger applicationSerialNo = new AtomicInteger(0); private HandlersConfigurerDi configurer; private ScheduledThreadPoolExecutor shutdownDeadlineExecutor; private Thread reconfigurerThread; @@ -172,10 +173,12 @@ public final class ConfiguredApplication implements Application { startAndStopServers(); log.info("Switching to the latest deployed set of configurations and components. " + - "Application switch number: " + (applicationSerialNo++)); + "Application switch number: " + applicationSerialNo.getAndIncrement()); } private ContainerBuilder createBuilderWithGuiceBindings() { + log.info("Initializing new set of configurations and components. " + + "Application switch number: " + applicationSerialNo.get()); ContainerBuilder builder = activator.newContainerBuilder(); setupGuiceBindings(builder.guiceModules()); return builder; diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java index e0d18f2b571..06b50145fc1 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java @@ -322,10 +322,6 @@ public class FastHit extends Hit { needXmlEscape = ! (fieldType instanceof XMLField); this.contents = contents; } - public RawField(byte [] contents) { - needXmlEscape = true; - this.contents = contents; - } public byte [] getUtf8() { return contents; } public boolean needXmlEscape() { return needXmlEscape; } @@ -359,10 +355,6 @@ public class FastHit extends Hit { return queryPacketData; } - public void clearQueryPacketData() { - queryPacketData = null; - } - CacheKey getCacheKey() { return cacheKey; } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java index 3369eb64094..1a7e693caa7 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java @@ -396,6 +396,16 @@ public abstract class VespaBackEndSearcher extends PingableSearcher { s.append(" location=") .append(query.getRanking().getLocation().toString()); } + + if (query.getGroupingSessionCache()) { + s.append(" groupingSessionCache=true"); + } + if (query.getRanking().getQueryCache()) { + s.append(" ranking.queryCache=true"); + } + if (query.getGroupingSessionCache() || query.getRanking().getQueryCache()) { + s.append(" sessionId=" + query.getSessionId(true)); + } List<Grouping> grouping = GroupingExecutor.getGroupingList(query); s.append(" grouping=").append(grouping.size()).append(" : "); diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/NormalizingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/NormalizingSearcher.java index 3b9a03bb5da..02c8ecda60c 100644 --- a/container-search/src/main/java/com/yahoo/prelude/querytransform/NormalizingSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/querytransform/NormalizingSearcher.java @@ -52,10 +52,12 @@ public class NormalizingSearcher extends Searcher { } protected void normalize(Query query, IndexFacts.Session indexFacts) { - String oldQuery = (query.getTraceLevel() >= 2) ? query.getModel().getQueryTree().getRoot().toString() : ""; + String oldQuery = (query.getTraceLevel() >= 2) ? query.getModel().getQueryTree().getRoot().toString() : null; + normalizeBody(query, indexFacts); - if (query.getTraceLevel() >= 2) - if (!(oldQuery.equals(query.getModel().getQueryTree().getRoot().toString()))) query.trace(getFunctionName(), true, 2); + + if (query.getTraceLevel() >= 2 && ! query.getModel().getQueryTree().getRoot().toString().equals(oldQuery)) + query.trace(getFunctionName(), true, 2); } private Query normalizeBody(Query query, IndexFacts.Session indexFacts) { @@ -63,19 +65,18 @@ public class NormalizingSearcher extends Searcher { Language language = query.getModel().getParsingLanguage(); if (root instanceof BlockItem) { List<Item> rootItems = new ArrayList<>(1); - rootItems.add(root); ListIterator<Item> i = rootItems.listIterator(); - i.next(); normalizeBlocks(language, indexFacts, (BlockItem) root, i); - query.getModel().getQueryTree().setRoot(rootItems.get(0)); + if ( ! rootItems.isEmpty()) // give up normalizing if the root was removed + query.getModel().getQueryTree().setRoot(rootItems.get(0)); } else if (root instanceof CompositeItem) { query.getModel().getQueryTree().setRoot(normalizeComposite(language, indexFacts, (CompositeItem) root)); } return query; } - + private Item normalizeComposite(Language language, IndexFacts.Session indexFacts, CompositeItem item) { if (item instanceof PhraseItem) { return normalizePhrase(language, indexFacts, (PhraseItem) item); diff --git a/container-search/src/main/java/com/yahoo/search/Query.java b/container-search/src/main/java/com/yahoo/search/Query.java index 509a5f3d1de..53b39046f1d 100644 --- a/container-search/src/main/java/com/yahoo/search/Query.java +++ b/container-search/src/main/java/com/yahoo/search/Query.java @@ -153,7 +153,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { private QueryContext context = null; /** Used for downstream session caches */ - private final AtomicReference<UniqueRequestId> sessionId = new AtomicReference<>(); + private UniqueRequestId requestId = null; //--------------- Owned sub-objects containing query properties ---------------- @@ -936,6 +936,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { clone.setOffset(getOffset()); clone.setNoCache(getNoCache()); clone.setGroupingSessionCache(getGroupingSessionCache()); + clone.requestId = null; // Each clone should have their own requestId. } /** Returns the presentation to be used for this query, never null */ @@ -962,18 +963,13 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { * @return the session id of this query, or null if not set and create is false */ public SessionId getSessionId(boolean create) { - UniqueRequestId uniqId = sessionId.get(); - if (uniqId == null && ! create) return null; + if (requestId == null && ! create) return null; - if (uniqId == null && create) { - uniqId = UniqueRequestId.next(); - sessionId.compareAndSet(null, uniqId); - uniqId = sessionId.get(); + if (requestId == null && create) { + requestId = UniqueRequestId.next(); } - String rankProfile = getRanking().getProfile(); - - return new SessionId(uniqId, rankProfile); + return new SessionId(requestId, getRanking().getProfile()); } public boolean hasEncodableProperties() { diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java index 003ba9a5261..b03c0cb752f 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java @@ -21,7 +21,6 @@ import com.yahoo.slime.BinaryFormat; import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; import com.yahoo.data.access.Inspector; -import com.yahoo.text.Utf8String; import com.yahoo.vespa.config.search.DispatchConfig; import java.util.Iterator; diff --git a/container-search/src/main/java/com/yahoo/search/query/SessionId.java b/container-search/src/main/java/com/yahoo/search/query/SessionId.java index c6e34e7e430..b065bd9a0a9 100644 --- a/container-search/src/main/java/com/yahoo/search/query/SessionId.java +++ b/container-search/src/main/java/com/yahoo/search/query/SessionId.java @@ -22,4 +22,19 @@ public class SessionId { } public Utf8String asUtf8String() { return id; } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + SessionId sessionId = (SessionId) o; + + return id.equals(sessionId.id); + } + + @Override + public int hashCode() { + return id.hashCode(); + } } diff --git a/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java b/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java index ce9e3357c77..e59c03b33c3 100644 --- a/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java @@ -23,6 +23,7 @@ import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; import com.yahoo.search.query.QueryTree; +import com.yahoo.search.query.SessionId; import com.yahoo.search.query.profile.QueryProfile; import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.search.result.Hit; @@ -44,6 +45,7 @@ import java.util.stream.Collectors; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @@ -637,6 +639,54 @@ public class QueryTestCase { } @Test + public void testThatSessionIdIsUniquePerQuery() { + Query q = new Query(); + assertNull(q.getSessionId(false)); + assertNull(q.getSessionId(false)); + SessionId s1 = q.getSessionId(true); + assertNotNull(s1); + SessionId s2 = q.getSessionId(true); + assertNotSame(s1, s2); + assertEquals(s1, s2); + assertEquals(s1.toString(), s2.toString()); + + Query q2 = new Query(); + assertNotEquals(q.getSessionId(false), q2.getSessionId(true)); + assertNotEquals(q.getSessionId(false).toString(), q2.getSessionId(true).toString()); + } + @Test + public void testThatCloneGetANewSessionId() { + Query q = new Query(); + q.getSessionId(true); + Query clonedQ = q.clone(); + assertNull(clonedQ.getSessionId(false)); + assertNotEquals(q.getSessionId(false), clonedQ.getSessionId(true)); + } + + @Test + public void testThatSessionIdIsUniquePerRankProfilePerQuery() { + Query q = new Query(); + SessionId s1 = q.getSessionId(true); + q.getRanking().setProfile("my-profile"); + SessionId s2 = q.getSessionId(false); + assertNotEquals(s1, s2); + } + + @Test + public void testThatSessionIdIsNotSharedIfCreatedAfterClone() { + Query q = new Query(); + Query q2 = q.clone(); + assertNull(q.getSessionId(false)); + assertNull(q2.getSessionId(false)); + + assertNotNull(q.getSessionId(true)); + assertNull(q2.getSessionId(false)); + + assertNotNull(q2.getSessionId(true)); + assertNotEquals(q.getSessionId(false), q2.getSessionId(false)); + } + + @Test public void testPositiveTerms() { Query q = new Query(httpEncode("/?query=-a \"b c\" d e")); Item i = q.getModel().getQueryTree().getRoot(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 51bf530ed4a..27992220f1c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -17,12 +17,15 @@ import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus; import com.yahoo.vespa.hosted.controller.api.application.v4.model.GitRevision; import com.yahoo.vespa.hosted.controller.api.application.v4.model.ScrewdriverBuildJob; +import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.ConfigChangeActions; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname; import com.yahoo.vespa.hosted.controller.api.identifiers.RevisionId; import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerClient; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.Log; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NoInstanceException; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareResponse; import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService; import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordId; @@ -216,22 +219,32 @@ public class ApplicationController { /** Deploys an application. If the application does not exist it is created. */ // TODO: Get rid of the options arg - public ActivateResult deployApplication(ApplicationId applicationId, com.yahoo.config.provision.Zone zone, + public ActivateResult deployApplication(ApplicationId applicationId, Zone zone, ApplicationPackage applicationPackage, DeployOptions options) { try (Lock lock = lock(applicationId)) { // Determine what we are doing Application application = get(applicationId).orElse(new Application(applicationId)); - DeploymentJobs.JobType jobType = DeploymentJobs.JobType.from(controller.zoneRegistry().system(), zone); - Version version = decideVersion(application, zone, options); - ApplicationRevision revision = toApplicationPackageRevision(applicationPackage, options.screwdriverBuildJob); + + // Decide version to deploy, if applicable. + Version version; + if (options.deployCurrentVersion) + version = application.currentVersion(controller, zone); + else if (application.deploymentJobs().isSelfTriggering()) // legacy mode: let the client decide + version = options.vespaVersion.map(Version::new).orElse(controller.systemVersion()); + else if ( ! application.deploying().isPresent() && ! zone.environment().isManuallyDeployed()) + return unexpectedDeployment(applicationId, zone, applicationPackage); + else + version = application.currentDeployVersion(controller, zone); // Ensure that the deploying change is tested // FIXME: For now only for non-self-triggering applications - VESPA-8418 - if (!application.deploymentJobs().isSelfTriggering() && !zone.environment().isManuallyDeployed() && !application.deploymentJobs().isDeployableTo(zone.environment(), application.deploying())) { + if ( ! application.deploymentJobs().isSelfTriggering() && ! zone.environment().isManuallyDeployed() && ! application.deploymentJobs().isDeployableTo(zone.environment(), application.deploying())) throw new IllegalArgumentException("Rejecting deployment of " + application + " to " + zone + " as pending " + application.deploying().get() + " is untested"); - } + + DeploymentJobs.JobType jobType = DeploymentJobs.JobType.from(controller.zoneRegistry().system(), zone); + ApplicationRevision revision = toApplicationPackageRevision(applicationPackage, options.screwdriverBuildJob); // Don't update/store applicationpackage information when deploying previous application package (initial staging step) if(! options.deployCurrentVersion) { @@ -274,21 +287,17 @@ public class ApplicationController { return new ActivateResult(new RevisionId(applicationPackage.hash()), preparedApplication.messages(), preparedApplication.prepareResponse()); } } - - private Version decideVersion(Application application, Zone zone, DeployOptions options) { - if (options.deployCurrentVersion) - return application.currentVersion(controller, zone); - - if (application.deploymentJobs().isSelfTriggering()) // legacy mode: let the client decide - return options.vespaVersion.map(Version::new).orElse(controller.systemVersion()); - if ( ! application.deploying().isPresent() && ! zone.environment().isManuallyDeployed()) - throw new IllegalArgumentException("Rejecting deployment of " + application + " to " + zone + - " as a deployment is not currently expected"); - - return application.currentDeployVersion(controller, zone); + private ActivateResult unexpectedDeployment(ApplicationId applicationId, Zone zone, ApplicationPackage applicationPackage) { + Log logEntry = new Log(); + logEntry.level = "WARNING"; + logEntry.time = clock.instant().toEpochMilli(); + logEntry.message = "Ignoring deployment of " + get(applicationId) + " to " + zone + " as a deployment is not currently expected"; + PrepareResponse prepareResponse = new PrepareResponse(); + prepareResponse.configChangeActions = new ConfigChangeActions(Collections.emptyList(), Collections.emptyList()); + return new ActivateResult(new RevisionId(applicationPackage.hash()), Collections.singletonList(logEntry), prepareResponse); } - + private Application deleteRemovedDeployments(Application application) { List<Deployment> deploymentsToRemove = application.deployments().values().stream() .filter(deployment -> deployment.zone().environment() == Environment.prod) @@ -316,20 +325,14 @@ public class ApplicationController { private Application deleteUnreferencedDeploymentJobs(Application application) { for (DeploymentJobs.JobType job : application.deploymentJobs().jobStatus().keySet()) { - if (!job.isProduction()) { - continue; - } Optional<Zone> zone = job.zone(controller.system()); - if (!zone.isPresent()) { + if ( ! job.isProduction() || (zone.isPresent() && application.deploymentSpec().includes(zone.get().environment(), zone.map(Zone::region)))) continue; - } - if (!application.deploymentSpec().includes(zone.get().environment(), zone.map(Zone::region))) { - application = application.withoutDeploymentJob(job); - } + application = application.withoutDeploymentJob(job); } return application; } - + private boolean triggeredWith(ApplicationRevision revision, Application application, DeploymentJobs.JobType jobType) { if (jobType == null) return false; JobStatus status = application.deploymentJobs().jobStatus().get(jobType); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java index 3fcd285e0fc..fa7a48c85c2 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java @@ -80,21 +80,11 @@ public class ApplicationList { return listOf(list.stream().filter(application -> ! failingOn(version, application))); } - /** Returns the subset of applications which have one or more deployment jobs failing for the current change */ - public ApplicationList hasDeploymentFailures() { - return listOf(list.stream().filter(application -> application.deploying().isPresent() && application.deploymentJobs().failingOn(application.deploying().get()))); - } - /** Returns the subset of applications which have at least one deployment */ public ApplicationList hasDeployment() { return listOf(list.stream().filter(a -> !a.deployments().isEmpty())); } - /** Returns the subset of applications that are currently deploying a change */ - public ApplicationList isDeploying() { - return listOf(list.stream().filter(application -> application.deploying().isPresent())); - } - /** Returns the subset of applications which started failing after the given instant */ public ApplicationList startedFailingAfter(Instant instant) { return listOf(list.stream().filter(application -> application.deploymentJobs().failingSince().isAfter(instant))); @@ -140,18 +130,6 @@ public class ApplicationList { return listOf(list.stream().filter(a -> !hasRunningJob(a, change))); } - /** Returns the subset of applications which currently do not have any job in progress */ - public ApplicationList notRunningJob() { - return listOf(list.stream().filter(a -> !a.deploymentJobs().inProgress())); - } - - /** Returns the subset of applications which has a job that started running before the given instant */ - public ApplicationList jobRunningSince(Instant instant) { - return listOf(list.stream().filter(a -> a.deploymentJobs().runningSince() - .map(at -> at.isBefore(instant)) - .orElse(false))); - } - /** Returns the subset of applications which deploys to given environment and region */ public ApplicationList deploysTo(Environment environment, RegionName region) { return listOf(list.stream().filter(a -> a.deploymentSpec().includes(environment, Optional.of(region)))); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java index d9256f94086..61231404a94 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java @@ -14,7 +14,6 @@ import com.yahoo.vespa.hosted.controller.Controller; import java.time.Instant; import java.util.Collection; import java.util.Collections; -import java.util.Comparator; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; @@ -118,11 +117,6 @@ public class DeploymentJobs { return status.values().stream().anyMatch(JobStatus::inProgress); } - /** Returns whether any job is failing for the given change */ - public boolean failingOn(Change change) { - return status.values().stream().anyMatch(jobStatus -> !jobStatus.isSuccess() && jobStatus.lastCompletedFor(change)); - } - /** Returns whether change can be deployed to the given environment */ public boolean isDeployableTo(Environment environment, Optional<Change> change) { if (environment == null || !change.isPresent()) { @@ -147,15 +141,6 @@ public class DeploymentJobs { return failingSince; } - /** Returns the time at which the oldest running job started */ - public Optional<Instant> runningSince() { - return jobStatus().values().stream() - .filter(JobStatus::inProgress) - .sorted(Comparator.comparing(jobStatus -> jobStatus.lastTriggered().get().at())) - .map(jobStatus -> jobStatus.lastTriggered().get().at()) - .findFirst(); - } - /** * Returns the id of the Screwdriver project running these deployment jobs * - or empty when this is not known or does not exist. @@ -225,8 +210,8 @@ public class DeploymentJobs { } /** Returns the region of this job type, or null if it does not have a region */ - public RegionName region(SystemName system) { - return zone(system).map(Zone::region).orElse(null); + public Optional<RegionName> region(SystemName system) { + return zone(system).map(Zone::region); } public static JobType fromId(String id) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index 2bc219dde62..ce1fee4dd14 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -89,13 +89,13 @@ public class DeploymentTrigger { /** * Called periodically to cause triggering of jobs in the background */ - public void triggerFailing(ApplicationId applicationId) { + public void triggerFailing(ApplicationId applicationId, String cause) { try (Lock lock = applications().lock(applicationId)) { Application application = applications().require(applicationId); if (shouldRetryFromBeginning(application)) { // failed for a long time: Discard existing change and restart from the component job application = application.withDeploying(Optional.empty()); - application = trigger(JobType.component, application, "Retrying failing deployment from beginning", lock); + application = trigger(JobType.component, application, "Retrying failing deployment from beginning: " + cause, lock); applications().store(application, lock); } else { // retry the failed job (with backoff) @@ -103,7 +103,7 @@ public class DeploymentTrigger { JobStatus jobStatus = application.deploymentJobs().jobStatus().get(jobType); if (isFailing(jobStatus)) { if (shouldRetryNow(jobStatus)) { - application = trigger(jobType, application, "Retrying failing job", lock); + application = trigger(jobType, application, "Retrying failing job: " + cause, lock); applications().store(application, lock); } break; @@ -119,6 +119,9 @@ public class DeploymentTrigger { if ( ! application.deploying().isPresent() ) continue; if (application.deploymentJobs().hasFailures()) continue; if (application.deploymentJobs().inProgress()) continue; + if (application.deploymentSpec().steps().stream().noneMatch(step -> step instanceof DeploymentSpec.Delay)) { + continue; // Application does not have any delayed deployments + } Optional<JobStatus> lastSuccessfulJob = application.deploymentJobs().jobStatus().values() .stream() @@ -131,7 +134,7 @@ public class DeploymentTrigger { try (Lock lock = applications().lock(application.id())) { application = applications().require(application.id()); application = trigger(nextAfter(lastSuccessfulJob.get().type(), application), application, - "Delayed by deployment spec", lock); + "Resuming delayed deployment", lock); applications().store(application, lock); } } @@ -230,15 +233,9 @@ public class DeploymentTrigger { return application.deploymentSpec() .zones() .stream() - .filter(z -> { - if (jobType.isProduction()) { - return z.matches(jobType.environment(), - Optional.ofNullable(jobType.region(controller.system()))); - } else { - // Ignore region for test environments as it's not specified in deployment spec - return z.environment() == jobType.environment(); - } - }) + .filter(zone -> zone.deploysTo( + jobType.environment(), + jobType.isProduction() ? jobType.region(controller.system()) : Optional.empty())) .findFirst(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployer.java index 9e8f902a8db..c3424b8d9af 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployer.java @@ -3,12 +3,13 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.application.ApplicationList; +import com.yahoo.vespa.hosted.controller.application.JobStatus; import java.time.Duration; import java.time.Instant; -import java.util.ArrayList; +import java.util.Comparator; import java.util.List; +import java.util.Optional; /** * Attempts redeployment of failed jobs and deployments. @@ -16,6 +17,8 @@ import java.util.List; * @author bratseth */ public class FailureRedeployer extends Maintainer { + + private final static Duration jobTimeout = Duration.ofHours(12); public FailureRedeployer(Controller controller, Duration interval, JobControl jobControl) { super(controller, interval, jobControl); @@ -23,20 +26,61 @@ public class FailureRedeployer extends Maintainer { @Override public void maintain() { - ApplicationList applications = ApplicationList.from(controller().applications().asList()).isDeploying(); - List<Application> toTrigger = new ArrayList<>(); + List<Application> applications = controller().applications().asList(); + retryFailingJobs(applications); + retryStuckJobs(applications); + } + + private void retryFailingJobs(List<Application> applications) { + for (Application application : applications) { + if (!application.deploying().isPresent()) { + continue; + } + if (application.deploymentJobs().inProgress()) { + continue; + } + Optional<JobStatus> failingJob = jobFailingFor(application); + failingJob.ifPresent(job -> triggerFailing(application, "Job " + job.type().id() + + " has been failing since " + job.firstFailing().get())); + } + } - // Applications with deployment failures for current change and no running jobs - toTrigger.addAll(applications.hasDeploymentFailures() - .notRunningJob() - .asList()); + private void retryStuckJobs(List<Application> applications) { + Instant maxAge = controller().clock().instant().minus(jobTimeout); + for (Application application : applications) { + if (!application.deploying().isPresent()) { + continue; + } + Optional<JobStatus> job = oldestRunningJob(application); + if (!job.isPresent()) { + continue; + } + // Ignore job if it doesn't belong to a zone in this system + if (!job.get().type().zone(controller().system()).isPresent()) { + continue; + } + if (job.get().lastTriggered().get().at().isBefore(maxAge)) { + triggerFailing(application, "Job " + job.get().type().id() + + " has been running for more than " + jobTimeout); + } + } + } + + private Optional<JobStatus> jobFailingFor(Application application) { + return application.deploymentJobs().jobStatus().values().stream() + .filter(status -> !status.isSuccess() && status.lastCompletedFor(application.deploying().get())) + .findFirst(); + } - // Applications with jobs that have been in progress for more than 12 hours - Instant twelveHoursAgo = controller().clock().instant().minus(Duration.ofHours(12)); - toTrigger.addAll(applications.jobRunningSince(twelveHoursAgo).asList()); + private Optional<JobStatus> oldestRunningJob(Application application) { + return application.deploymentJobs().jobStatus().values().stream() + .filter(JobStatus::inProgress) + .sorted(Comparator.comparing(status -> status.lastTriggered().get().at())) + .findFirst(); + } - toTrigger.forEach(application -> controller().applications().deploymentTrigger() - .triggerFailing(application.id())); + private void triggerFailing(Application application, String cause) { + controller().applications().deploymentTrigger().triggerFailing(application.id(), cause); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java index 014c63a6779..04999f9e1c3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java @@ -241,14 +241,7 @@ public class ApplicationSerializer { private List<JobStatus> jobStatusListFromSlime(Inspector array) { List<JobStatus> jobStatusList = new ArrayList<>(); - array.traverse((ArrayTraverser) (int i, Inspector item) -> { - // TODO: This zone has been removed. Remove after Aug 2017 - String jobId = item.field(jobTypeField).asString(); - if ("production-ap-aue-1".equals(jobId)) { - return; - } - jobStatusList.add(jobStatusFromSlime(item)); - }); + array.traverse((ArrayTraverser) (int i, Inspector item) -> jobStatusList.add(jobStatusFromSlime(item))); return jobStatusList; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index e807762371a..3dddfaf58a1 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -11,6 +11,9 @@ import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.api.Tenant; import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus; @@ -31,6 +34,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.athens.NToken; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.ApplicationRevision; import com.yahoo.vespa.hosted.controller.application.Change; +import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobReport; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType; @@ -38,6 +42,7 @@ import com.yahoo.vespa.hosted.controller.application.JobStatus; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.BuildSystem; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; +import com.yahoo.vespa.hosted.controller.persistence.ApplicationSerializer; import com.yahoo.vespa.hosted.controller.versions.DeploymentStatistics; import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; @@ -46,6 +51,8 @@ import com.yahoo.vespa.hosted.controller.api.integration.athens.mock.NTokenMock; import org.junit.Test; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; import java.time.Duration; import java.time.Instant; import java.util.ArrayList; @@ -53,6 +60,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.function.Supplier; import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.component; import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.productionCorpUsEast1; @@ -212,13 +220,9 @@ public class ControllerTest { assertEquals(systemVersion, tester.configServerClientMock().lastPrepareVersion.get()); // Unexpected deployment - try { - tester.deploy(productionUsWest1, app1, applicationPackage); - fail("Expected exception as no change was to be deployed"); - } - catch (IllegalArgumentException expected) { - // success - } + tester.deploy(productionUsWest1, app1, applicationPackage); + // applications are immutable, so any change to one, including deployment changes, would give rise to a new instance. + assertEquals("Unexpected deployment is ignored", app1, applications.require(app1.id())); // Application change after a new system version, and a region added Version newSystemVersion = incrementSystemVersion(tester.controller()); @@ -598,4 +602,69 @@ public class ControllerTest { new DeployOptions(Optional.of(new ScrewdriverBuildJob(app1ScrewdriverId, app1RevisionId)), version, false, deployCurrentVersion)); } + + @Test + public void testCleanupOfStaleDeploymentData() throws IOException { + DeploymentTester tester = new DeploymentTester(); + tester.controllerTester().getZoneRegistryMock().setSystem(SystemName.cd); + + Supplier<Map<JobType, JobStatus>> statuses = () -> + tester.application(ApplicationId.from("vespa", "canary", "default")).deploymentJobs().jobStatus(); + + // Current system version, matches version in test data + Version version = Version.fromString("6.141.117"); + Version oldVersion = Version.fromString("6.98.12"); + tester.configServerClientMock().setDefaultConfigServerVersion(version); + tester.updateVersionStatus(version); + assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber()); + + // Load test data data + ApplicationSerializer serializer = new ApplicationSerializer(); + byte[] json = Files.readAllBytes(Paths.get("src/test/java/com/yahoo/vespa/hosted/controller/maintenance/testdata/canary-with-stale-data.json")); + Slime slime = SlimeUtils.jsonToSlime(json); + Application application = serializer.fromSlime(slime); + try (Lock lock = tester.controller().applications().lock(application.id())) { + tester.controller().applications().store(application, lock); + } + + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .upgradePolicy("canary") + .region("cd-us-central-1") + .build(); + + long cdJobsCount = statuses.get().keySet().stream() + .filter(type -> type.zone(SystemName.cd).isPresent()) + .count(); + + long mainJobsCount = statuses.get().keySet().stream() + .filter(type -> type.zone(SystemName.main).isPresent() && ! type.zone(SystemName.cd).isPresent()) + .count(); + + assertEquals("Irrelevant (main) data is present.", 8, mainJobsCount); + + // New version is released + version = Version.fromString("6.142.1"); + tester.configServerClientMock().setDefaultConfigServerVersion(version); + tester.updateVersionStatus(version); + assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber()); + tester.upgrader().maintain(); + + // Test environments pass + tester.deploy(DeploymentJobs.JobType.systemTest, application, applicationPackage); + tester.buildSystem().takeJobsToRun(); + tester.clock().advance(Duration.ofMinutes(10)); + tester.notifyJobCompletion(DeploymentJobs.JobType.systemTest, application, true); + + long newCdJobsCount = statuses.get().keySet().stream() + .filter(type -> type.zone(SystemName.cd).isPresent()) + .count(); + + long newMainJobsCount = statuses.get().keySet().stream() + .filter(type -> type.zone(SystemName.main).isPresent() && ! type.zone(SystemName.cd).isPresent()) + .count(); + + assertEquals("Irrelevant (main) job data is removed.", 0, newMainJobsCount); + assertEquals("Relevant (cd) data is not removed.", cdJobsCount, newCdJobsCount); + } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ZoneRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ZoneRegistryMock.java index 62b935842f7..bf21467bc8d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ZoneRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ZoneRegistryMock.java @@ -27,9 +27,11 @@ public class ZoneRegistryMock implements ZoneRegistry { deploymentTimeToLive.put(zone, duration); } + private SystemName system = SystemName.main; + @Override public SystemName system() { - return SystemName.main; + return system; } @Override @@ -71,4 +73,8 @@ public class ZoneRegistryMock implements ZoneRegistry { public URI getDashboardUri() { return URI.create("http://dashboard.test"); } + + public void setSystem(SystemName system) { + this.system = system; + } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java index 32d1714ea52..0e816be864d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java @@ -160,7 +160,7 @@ public class DeploymentTester { } public void deploy(JobType job, Application application, ApplicationPackage applicationPackage, boolean deployCurrentVersion) { - job.zone(SystemName.main).ifPresent(zone -> tester.deploy(application, zone, applicationPackage, deployCurrentVersion)); + job.zone(controller().system()).ifPresent(zone -> tester.deploy(application, zone, applicationPackage, deployCurrentVersion)); } public void deployAndNotify(JobType job, Application application, ApplicationPackage applicationPackage, boolean success) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java index ce06910240b..0f48afc0ca4 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java @@ -34,14 +34,14 @@ public class DeploymentTriggerTest { tester.buildSystem().takeJobsToRun(); assertEquals("Job removed", 0, tester.buildSystem().jobs().size()); tester.clock().advance(Duration.ofHours(2)); - tester.deploymentTrigger().triggerFailing(app1.id()); + tester.deploymentTrigger().triggerFailing(app1.id(), "unit test"); assertEquals("Retried job", 1, tester.buildSystem().jobs().size()); assertEquals(JobType.stagingTest.id(), tester.buildSystem().jobs().get(0).jobName()); tester.buildSystem().takeJobsToRun(); assertEquals("Job removed", 0, tester.buildSystem().jobs().size()); tester.clock().advance(Duration.ofHours(7)); - tester.deploymentTrigger().triggerFailing(app1.id()); + tester.deploymentTrigger().triggerFailing(app1.id(), "unit test"); assertEquals("Retried from the beginning", 1, tester.buildSystem().jobs().size()); assertEquals(JobType.component.id(), tester.buildSystem().jobs().get(0).jobName()); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java index cde511a9076..052ca87f791 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java @@ -3,13 +3,20 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.SystemName; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; +import com.yahoo.vespa.hosted.controller.persistence.ApplicationSerializer; import org.junit.Test; +import java.nio.file.Files; +import java.nio.file.Paths; import java.time.Duration; import static org.junit.Assert.assertEquals; @@ -166,4 +173,60 @@ public class FailureRedeployerTest { assertTrue("No jobs retried", tester.buildSystem().jobs().isEmpty()); } + @Test + public void retryIgnoresStaleJobData() throws Exception { + DeploymentTester tester = new DeploymentTester(); + tester.controllerTester().getZoneRegistryMock().setSystem(SystemName.cd); + + // Current system version, matches version in test data + Version version = Version.fromString("6.141.117"); + tester.configServerClientMock().setDefaultConfigServerVersion(version); + tester.updateVersionStatus(version); + assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber()); + + // Load test data data + ApplicationSerializer serializer = new ApplicationSerializer(); + byte[] json = Files.readAllBytes(Paths.get("src/test/java/com/yahoo/vespa/hosted/controller/maintenance/testdata/canary-with-stale-data.json")); + Slime slime = SlimeUtils.jsonToSlime(json); + Application application = serializer.fromSlime(slime); + try (Lock lock = tester.controller().applications().lock(application.id())) { + tester.controller().applications().store(application, lock); + } + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .upgradePolicy("canary") + .region("cd-us-central-1") + .build(); + + // New version is released + version = Version.fromString("6.142.1"); + tester.configServerClientMock().setDefaultConfigServerVersion(version); + tester.updateVersionStatus(version); + assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber()); + tester.upgrader().maintain(); + + // Test environments pass + tester.deploy(DeploymentJobs.JobType.systemTest, application, applicationPackage); + tester.buildSystem().takeJobsToRun(); + tester.clock().advance(Duration.ofMinutes(10)); + tester.notifyJobCompletion(DeploymentJobs.JobType.systemTest, application, true); + + tester.deploy(DeploymentJobs.JobType.stagingTest, application, applicationPackage); + tester.buildSystem().takeJobsToRun(); + tester.clock().advance(Duration.ofMinutes(10)); + tester.notifyJobCompletion(DeploymentJobs.JobType.stagingTest, application, true); + + // Production job starts, but does not complete + assertEquals(1, tester.buildSystem().jobs().size()); + assertEquals("Production job triggered", DeploymentJobs.JobType.productionCdUsCentral1.id(), tester.buildSystem().jobs().get(0).jobName()); + tester.buildSystem().takeJobsToRun(); + + // Failure re-deployer runs + tester.failureRedeployer().maintain(); + assertTrue("No jobs retried", tester.buildSystem().jobs().isEmpty()); + + // Deployment completes + tester.notifyJobCompletion(DeploymentJobs.JobType.productionCdUsCentral1, application, true); + assertFalse("Change deployed", tester.application(application.id()).deploying().isPresent()); + } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/testdata/canary-with-stale-data.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/testdata/canary-with-stale-data.json new file mode 100644 index 00000000000..323889c7c45 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/testdata/canary-with-stale-data.json @@ -0,0 +1,295 @@ +{ + "id": "vespa:canary:default", + "deploymentSpecField": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<deployment version=\"1.0\">\n <upgrade policy='canary'/>\n <test />\n <staging />\n <prod>\n <region active=\"true\">cd-us-central-1</region>\n </prod>\n</deployment>\n", + "validationOverrides": "<validation-overrides>\n <allow until=\"2017-04-27\">force-automatic-tenant-upgrade-test</allow>\n</validation-overrides>\n", + "deployments": [ + { + "zone": { + "environment": "prod", + "region": "cd-us-central-1" + }, + "version": "6.141.117", + "deployTime": 1503901783487, + "applicationPackageRevision": { + "applicationPackageHash": "72c3961314b96b1155a310f4785ae57ec74b1273", + "sourceRevision": { + "repositoryField": "git@git.test:vespa/canary-application.git", + "branchField": "origin/canary-cd", + "commitField": "566b7b30ee7886b845bb70958a0e1bdab2868633" + } + } + } + ], + "deploymentJobs": { + "projectId": 191186, + "jobStatus": [ + { + "jobType": "production-eu-west-1", + "lastTriggered": { + "version": "6.98.12", + "at": 1493034019032 + }, + "lastCompleted": { + "version": "6.98.12", + "at": 1493033995026 + }, + "lastSuccess": { + "version": "6.98.12", + "at": 1493033995026 + } + }, + { + "jobType": "production-cd-us-central-1", + "jobError": "unknown", + "lastTriggered": { + "version": "6.141.117", + "revision": { + "applicationPackageHash": "72c3961314b96b1155a310f4785ae57ec74b1273", + "sourceRevision": { + "repositoryField": "git@git.test:vespa/canary-application.git", + "branchField": "origin/canary-cd", + "commitField": "566b7b30ee7886b845bb70958a0e1bdab2868633" + } + }, + "at": 1503903384816 + }, + "lastCompleted": { + "version": "6.141.117", + "revision": { + "applicationPackageHash": "72c3961314b96b1155a310f4785ae57ec74b1273", + "sourceRevision": { + "repositoryField": "git@git.test:vespa/canary-application.git", + "branchField": "origin/canary-cd", + "commitField": "566b7b30ee7886b845bb70958a0e1bdab2868633" + } + }, + "at": 1503903659364 + }, + "firstFailing": { + "version": "6.141.117", + "revision": { + "applicationPackageHash": "72c3961314b96b1155a310f4785ae57ec74b1273", + "sourceRevision": { + "repositoryField": "git@git.test:vespa/canary-application.git", + "branchField": "origin/canary-cd", + "commitField": "566b7b30ee7886b845bb70958a0e1bdab2868633" + } + }, + "at": 1503903659364 + }, + "lastSuccess": { + "version": "6.141.117", + "revision": { + "applicationPackageHash": "72c3961314b96b1155a310f4785ae57ec74b1273", + "sourceRevision": { + "repositoryField": "git@git.test:vespa/canary-application.git", + "branchField": "origin/canary-cd", + "commitField": "566b7b30ee7886b845bb70958a0e1bdab2868633" + } + }, + "at": 1503903556127 + } + }, + { + "jobType": "component", + "lastTriggered": { + "version": "6.141.109", + "at": 1503867517105 + }, + "lastCompleted": { + "version": "6.141.109", + "at": 1503867704464 + }, + "lastSuccess": { + "version": "6.141.109", + "at": 1503867704464 + } + }, + { + "jobType": "production-corp-us-east-1", + "lastTriggered": { + "version": "6.98.12", + "at": 1493034428590 + }, + "lastCompleted": { + "version": "6.98.12", + "at": 1493034114538 + }, + "lastSuccess": { + "version": "6.98.12", + "at": 1493034114538 + } + }, + { + "jobType": "production-ap-southeast-1", + "lastTriggered": { + "version": "6.98.12", + "at": 1493034265146 + }, + "lastCompleted": { + "version": "6.98.12", + "at": 1493034097617 + }, + "lastSuccess": { + "version": "6.98.12", + "at": 1493034097617 + } + }, + { + "jobType": "production-us-central-1", + "lastTriggered": { + "version": "6.98.12", + "at": 1493033800484 + }, + "lastCompleted": { + "version": "6.98.12", + "at": 1493034273753 + }, + "lastSuccess": { + "version": "6.98.12", + "at": 1493034273753 + } + }, + { + "jobType": "staging-test", + "lastTriggered": { + "version": "6.141.117", + "revision": { + "applicationPackageHash": "72c3961314b96b1155a310f4785ae57ec74b1273", + "sourceRevision": { + "repositoryField": "git@git.test:vespa/canary-application.git", + "branchField": "origin/canary-cd", + "commitField": "566b7b30ee7886b845bb70958a0e1bdab2868633" + } + }, + "at": 1503900683154 + }, + "lastCompleted": { + "version": "6.141.117", + "revision": { + "applicationPackageHash": "72c3961314b96b1155a310f4785ae57ec74b1273", + "sourceRevision": { + "repositoryField": "git@git.test:vespa/canary-application.git", + "branchField": "origin/canary-cd", + "commitField": "566b7b30ee7886b845bb70958a0e1bdab2868633" + } + }, + "at": 1503901635745 + }, + "lastSuccess": { + "version": "6.141.117", + "revision": { + "applicationPackageHash": "72c3961314b96b1155a310f4785ae57ec74b1273", + "sourceRevision": { + "repositoryField": "git@git.test:vespa/canary-application.git", + "branchField": "origin/canary-cd", + "commitField": "566b7b30ee7886b845bb70958a0e1bdab2868633" + } + }, + "at": 1503901635745 + } + }, + { + "jobType": "system-test", + "lastTriggered": { + "version": "6.141.117", + "revision": { + "applicationPackageHash": "72c3961314b96b1155a310f4785ae57ec74b1273", + "sourceRevision": { + "repositoryField": "git@git.test:vespa/canary-application.git", + "branchField": "origin/canary-cd", + "commitField": "566b7b30ee7886b845bb70958a0e1bdab2868633" + } + }, + "at": 1503899621243 + }, + "lastCompleted": { + "version": "6.141.117", + "revision": { + "applicationPackageHash": "72c3961314b96b1155a310f4785ae57ec74b1273", + "sourceRevision": { + "repositoryField": "git@git.test:vespa/canary-application.git", + "branchField": "origin/canary-cd", + "commitField": "566b7b30ee7886b845bb70958a0e1bdab2868633" + } + }, + "at": 1503900025214 + }, + "lastSuccess": { + "version": "6.141.117", + "revision": { + "applicationPackageHash": "72c3961314b96b1155a310f4785ae57ec74b1273", + "sourceRevision": { + "repositoryField": "git@git.test:vespa/canary-application.git", + "branchField": "origin/canary-cd", + "commitField": "566b7b30ee7886b845bb70958a0e1bdab2868633" + } + }, + "at": 1503900025214 + } + }, + { + "jobType": "production-us-west-1", + "lastTriggered": { + "version": "6.98.12", + "at": 1493034273768 + }, + "lastCompleted": { + "version": "6.98.12", + "at": 1493034019015 + }, + "lastSuccess": { + "version": "6.98.12", + "at": 1493034019015 + } + }, + { + "jobType": "production-ap-northeast-1", + "lastTriggered": { + "version": "6.98.12", + "at": 1493033995045 + }, + "lastCompleted": { + "version": "6.98.12", + "at": 1493034257206 + }, + "lastSuccess": { + "version": "6.98.12", + "at": 1493034257206 + } + }, + { + "jobType": "production-ap-northeast-2", + "lastTriggered": { + "version": "6.98.12", + "at": 1493034257222 + }, + "lastCompleted": { + "version": "6.98.12", + "at": 1493034265048 + }, + "lastSuccess": { + "version": "6.98.12", + "at": 1493034265048 + } + }, + { + "jobType": "production-us-east-3", + "lastTriggered": { + "version": "6.98.12", + "at": 1493034114555 + }, + "lastCompleted": { + "version": "6.98.12", + "at": 1493033800469 + }, + "lastSuccess": { + "version": "6.98.12", + "at": 1493033800469 + } + } + ], + "selfTriggering": false + }, + "outstandingChangeField": false +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java index 645e38d0f2d..20e3aae9114 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java @@ -133,41 +133,6 @@ public class ApplicationSerializerTest { assertEquals(JobError.unknown, applicationWithFailingJob.deploymentJobs().jobStatus().get(DeploymentJobs.JobType.systemTest).jobError().get()); } - // TODO: Remove after Aug 2017 - @Test - public void serializeWithRemovedZone() throws Exception { - String json = "{\n" + - " \"id\": \"t1:a1:i1\",\n" + - " \"deploymentSpecField\": \"<deployment version='1.0'/>\",\n" + - " \"deploymentJobs\": {\n" + - " \"projectId\": 123,\n" + - " \"jobStatus\": [\n" + - " {\n" + - " \"jobType\": \"system-test\",\n" + - " \"version\": \"5.6.7\",\n" + - " \"completionTime\": 7,\n" + - " \"lastTriggered\": 8\n" + - " },\n" + - " {\n" + - " \"jobType\": \"production-ap-aue-1\",\n" + - " \"version\": \"5.6.7\",\n" + - " \"completionTime\": 7,\n" + - " \"lastTriggered\": 8\n" + - " },\n" + - " {\n" + - " \"jobType\": \"staging-test\",\n" + - " \"version\": \"5.6.7\",\n" + - " \"completionTime\": 7,\n" + - " \"lastTriggered\": 8\n" + - " }\n" + - " ],\n" + - " \"selfTriggering\": false\n" + - " }\n" + - "}\n"; - Application app = applicationSerializer.fromSlime(SlimeUtils.jsonToSlime(json.getBytes(StandardCharsets.UTF_8))); - assertEquals(2, app.deploymentJobs().jobStatus().size()); - } - private Slime applicationSlime(boolean error) { return SlimeUtils.jsonToSlime(applicationJson(error).getBytes(StandardCharsets.UTF_8)); } diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImpl.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImpl.java index f633c0a3971..e676a86d9fd 100644 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImpl.java +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImpl.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.hosted.dockerapi; import com.github.dockerjava.api.DockerClient; import com.github.dockerjava.api.command.CreateContainerCmd; -import com.github.dockerjava.api.exception.DockerException; import com.github.dockerjava.api.model.Bind; import com.github.dockerjava.api.model.Capability; import com.github.dockerjava.api.model.Ulimit; @@ -133,8 +132,8 @@ class CreateContainerCommandImpl implements Docker.CreateContainerCommand { public void create() { try { createCreateContainerCmd().exec(); - } catch (DockerException e) { - throw new RuntimeException("Failed to create container " + containerName.asString(), e); + } catch (RuntimeException e) { + throw new DockerException("Failed to create container " + containerName.asString(), e); } } diff --git a/document/CMakeLists.txt b/document/CMakeLists.txt index e9694390b4b..ca2ee029c87 100644 --- a/document/CMakeLists.txt +++ b/document/CMakeLists.txt @@ -8,10 +8,6 @@ vespa_define_module( config_cloudconfig vespaeval - EXTERNAL_DEPENDS - lz4 - zstd - LIBS src/vespa/document src/vespa/document/annotation diff --git a/document/src/tests/documenttestcase.cpp b/document/src/tests/documenttestcase.cpp index 44cff2c80de..c3f88ac3696 100644 --- a/document/src/tests/documenttestcase.cpp +++ b/document/src/tests/documenttestcase.cpp @@ -16,6 +16,7 @@ #include <fcntl.h> using vespalib::nbostream; +using vespalib::compression::CompressionConfig; using namespace document::config_builder; diff --git a/document/src/tests/repo/documenttyperepo_test.cpp b/document/src/tests/repo/documenttyperepo_test.cpp index 508e6f237cf..1e8fd9ec470 100644 --- a/document/src/tests/repo/documenttyperepo_test.cpp +++ b/document/src/tests/repo/documenttyperepo_test.cpp @@ -10,7 +10,6 @@ #include <vespa/document/datatype/weightedsetdatatype.h> #include <vespa/document/repo/configbuilder.h> #include <vespa/document/repo/documenttyperepo.h> -#include <stdlib.h> #include <vespa/vespalib/objects/identifiable.h> #include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/testkit/testapp.h> @@ -25,6 +24,7 @@ using std::vector; using vespalib::Identifiable; using vespalib::IllegalArgumentException; using vespalib::string; +using vespalib::compression::CompressionConfig; using namespace document::config_builder; using namespace document; diff --git a/document/src/tests/serialization/.gitignore b/document/src/tests/serialization/.gitignore index 9f5bc440533..7e94d2757ae 100644 --- a/document/src/tests/serialization/.gitignore +++ b/document/src/tests/serialization/.gitignore @@ -2,5 +2,4 @@ .depend Makefile document_annotationserializer_test_app -document_compression_test_app document_vespadocumentserializer_test_app diff --git a/document/src/tests/serialization/CMakeLists.txt b/document/src/tests/serialization/CMakeLists.txt index 5b23bffad26..e1ce43b12d4 100644 --- a/document/src/tests/serialization/CMakeLists.txt +++ b/document/src/tests/serialization/CMakeLists.txt @@ -17,12 +17,3 @@ vespa_add_executable(document_annotationserializer_test_app TEST document_documentconfig ) vespa_add_test(NAME document_annotationserializer_test_app COMMAND document_annotationserializer_test_app) -vespa_add_executable(document_compression_test_app TEST - SOURCES - compression_test.cpp - DEPENDS - document - AFTER - document_documentconfig -) -vespa_add_test(NAME document_compression_test_app COMMAND document_compression_test_app) diff --git a/document/src/tests/serialization/vespadocumentserializer_test.cpp b/document/src/tests/serialization/vespadocumentserializer_test.cpp index d09012a4e4b..9da20e5a84c 100644 --- a/document/src/tests/serialization/vespadocumentserializer_test.cpp +++ b/document/src/tests/serialization/vespadocumentserializer_test.cpp @@ -55,6 +55,7 @@ using vespalib::tensor::Tensor; using vespalib::tensor::TensorBuilder; using vespalib::tensor::TensorCells; using vespalib::tensor::TensorDimensions; +using vespalib::compression::CompressionConfig; using namespace document; using std::string; using std::vector; diff --git a/document/src/vespa/document/datatype/structdatatype.h b/document/src/vespa/document/datatype/structdatatype.h index f6c77f18f47..0f1c58316c9 100644 --- a/document/src/vespa/document/datatype/structdatatype.h +++ b/document/src/vespa/document/datatype/structdatatype.h @@ -11,15 +11,16 @@ #include <vespa/document/datatype/structureddatatype.h> #include <vespa/vespalib/stllike/hash_map.h> -#include <vespa/document/util/compressionconfig.h> +#include <vespa/vespalib/util/compressionconfig.h> #include <memory> namespace document { class StructDataType final : public StructuredDataType { public: - typedef std::unique_ptr<StructDataType> UP; - typedef std::shared_ptr<StructDataType> SP; + using UP = std::unique_ptr<StructDataType>; + using SP = std::shared_ptr<StructDataType>; + using CompressionConfig = vespalib::compression::CompressionConfig; StructDataType(); StructDataType(const vespalib::stringref &name); diff --git a/document/src/vespa/document/fieldvalue/serializablearray.cpp b/document/src/vespa/document/fieldvalue/serializablearray.cpp index 76216d75f1b..5dfd8eff891 100644 --- a/document/src/vespa/document/fieldvalue/serializablearray.cpp +++ b/document/src/vespa/document/fieldvalue/serializablearray.cpp @@ -2,7 +2,7 @@ #include "serializablearray.h" #include <vespa/document/util/serializableexceptions.h> #include <vespa/document/util/bytebuffer.h> -#include <vespa/document/util/compressor.h> +#include <vespa/vespalib/util/compressor.h> #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/vespalib/data/databuffer.h> #include <algorithm> @@ -181,7 +181,7 @@ SerializableArray::clear(int id) void SerializableArray::deCompress() // throw (DeserializeException) { - using document::compression::decompress; + using vespalib::compression::decompress; // will only do this once LOG_ASSERT(_compSerData); @@ -239,7 +239,7 @@ void SerializableArray::assign(EntryMap & entries, } } -CompressionInfo +vespalib::compression::CompressionInfo SerializableArray::getCompressionInfo() const { return CompressionInfo(_uncompressedLength, _compSerData->getRemaining()); } diff --git a/document/src/vespa/document/fieldvalue/serializablearray.h b/document/src/vespa/document/fieldvalue/serializablearray.h index 2d12c1191a6..2f7d65938aa 100644 --- a/document/src/vespa/document/fieldvalue/serializablearray.h +++ b/document/src/vespa/document/fieldvalue/serializablearray.h @@ -16,7 +16,7 @@ #pragma once -#include <vespa/document/util/compressionconfig.h> +#include <vespa/vespalib/util/compressionconfig.h> #include <vespa/vespalib/objects/cloneable.h> #include <vespa/vespalib/util/buffer.h> #include <vespa/vespalib/util/memory.h> @@ -107,6 +107,8 @@ public: using CP = vespalib::CloneablePtr<SerializableArray>; using UP = std::unique_ptr<SerializableArray>; using ByteBufferUP = std::unique_ptr<ByteBuffer>; + using CompressionConfig = vespalib::compression::CompressionConfig; + using CompressionInfo = vespalib::compression::CompressionInfo; SerializableArray(); virtual ~SerializableArray(); diff --git a/document/src/vespa/document/fieldvalue/structfieldvalue.cpp b/document/src/vespa/document/fieldvalue/structfieldvalue.cpp index c0ae342fd34..0a05ae60600 100644 --- a/document/src/vespa/document/fieldvalue/structfieldvalue.cpp +++ b/document/src/vespa/document/fieldvalue/structfieldvalue.cpp @@ -21,6 +21,7 @@ using std::vector; using vespalib::nbostream; using vespalib::nbostream_longlivedbuf; using vespalib::make_string; +using vespalib::compression::CompressionConfig; using namespace vespalib::xml; namespace document { diff --git a/document/src/vespa/document/fieldvalue/structfieldvalue.h b/document/src/vespa/document/fieldvalue/structfieldvalue.h index 6ab6f71ce56..bb6956cf012 100644 --- a/document/src/vespa/document/fieldvalue/structfieldvalue.h +++ b/document/src/vespa/document/fieldvalue/structfieldvalue.h @@ -52,7 +52,9 @@ private: mutable bool _hasChanged; public: - typedef std::unique_ptr<StructFieldValue> UP; + using UP = std::unique_ptr<StructFieldValue>; + using CompressionConfig = vespalib::compression::CompressionConfig; + StructFieldValue(const DataType &type); ~StructFieldValue(); void swap(StructFieldValue & rhs); diff --git a/document/src/vespa/document/repo/documenttyperepo.cpp b/document/src/vespa/document/repo/documenttyperepo.cpp index 870e88e5036..6bfae246c10 100644 --- a/document/src/vespa/document/repo/documenttyperepo.cpp +++ b/document/src/vespa/document/repo/documenttyperepo.cpp @@ -30,6 +30,7 @@ using vespalib::hash_map; using vespalib::make_string; using vespalib::string; using vespalib::stringref; +using vespalib::compression::CompressionConfig; namespace document { diff --git a/document/src/vespa/document/serialization/vespadocumentdeserializer.cpp b/document/src/vespa/document/serialization/vespadocumentdeserializer.cpp index bee1c258e4a..2b45e8a298c 100644 --- a/document/src/vespa/document/serialization/vespadocumentdeserializer.cpp +++ b/document/src/vespa/document/serialization/vespadocumentdeserializer.cpp @@ -38,6 +38,7 @@ using vespalib::asciistream; using vespalib::nbostream; using vespalib::Memory; using vespalib::stringref; +using vespalib::compression::CompressionConfig; namespace document { diff --git a/document/src/vespa/document/serialization/vespadocumentserializer.cpp b/document/src/vespa/document/serialization/vespadocumentserializer.cpp index 2534690b014..6c9e95a9dd6 100644 --- a/document/src/vespa/document/serialization/vespadocumentserializer.cpp +++ b/document/src/vespa/document/serialization/vespadocumentserializer.cpp @@ -27,7 +27,7 @@ #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/data/databuffer.h> #include <vespa/eval/tensor/serialization/typed_binary_format.h> -#include <vespa/document/util/compressor.h> +#include <vespa/vespalib/util/compressor.h> using std::make_pair; using std::pair; @@ -36,6 +36,7 @@ using vespalib::nbostream; using vespalib::stringref; using vespalib::string; using vespalib::slime::BinaryFormat; +using vespalib::compression::CompressionConfig; namespace document { @@ -275,7 +276,7 @@ vespalib::ConstBufferRef compressStream(const CompressionConfig &config, nbostream &stream, vespalib::DataBuffer & compressed_data) { - using compression::compress; + using vespalib::compression::compress; vespalib::ConstBufferRef buf(stream.c_str(), stream.size()); if (config.useCompression() && bigEnough(stream.size(), config)) { CompressionConfig::Type compressedType = compress(config, vespalib::ConstBufferRef(stream.c_str(), stream.size()), compressed_data, false); diff --git a/document/src/vespa/document/util/CMakeLists.txt b/document/src/vespa/document/util/CMakeLists.txt index 2179b1307d3..8cb148abe25 100644 --- a/document/src/vespa/document/util/CMakeLists.txt +++ b/document/src/vespa/document/util/CMakeLists.txt @@ -2,9 +2,6 @@ vespa_add_library(document_util OBJECT SOURCES bytebuffer.cpp - compressor.cpp - lz4compressor.cpp - zstdcompressor.cpp printable.cpp serializable.cpp stringutil.cpp diff --git a/documentapi/src/tests/policies/testframe.cpp b/documentapi/src/tests/policies/testframe.cpp index d231672da84..877e3164a8c 100644 --- a/documentapi/src/tests/policies/testframe.cpp +++ b/documentapi/src/tests/policies/testframe.cpp @@ -7,6 +7,7 @@ #include <vespa/messagebus/testlib/simplemessage.h> #include <vespa/messagebus/testlib/simpleprotocol.h> #include <vespa/messagebus/testlib/simplereply.h> +#include <vespa/messagebus/network/rpcnetworkparams.h> #include <vespa/log/log.h> LOG_SETUP(".testframe"); diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/putdocumentmessage.h b/documentapi/src/vespa/documentapi/messagebus/messages/putdocumentmessage.h index 9bc1c088dfa..f86e594dfa4 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/putdocumentmessage.h +++ b/documentapi/src/vespa/documentapi/messagebus/messages/putdocumentmessage.h @@ -38,6 +38,7 @@ public: * @return The document. */ const DocumentSP & getDocumentSP() const { return _document; } + DocumentSP stealDocument() { return std::move(_document); } const document::Document & getDocument() const { return *_document; } /** diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/updatedocumentmessage.h b/documentapi/src/vespa/documentapi/messagebus/messages/updatedocumentmessage.h index 3a320960515..55cb22e05e0 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/updatedocumentmessage.h +++ b/documentapi/src/vespa/documentapi/messagebus/messages/updatedocumentmessage.h @@ -21,8 +21,8 @@ public: /** * Convenience typedef. */ - typedef std::unique_ptr<UpdateDocumentMessage> UP; - typedef std::shared_ptr<UpdateDocumentMessage> SP; + using UP = std::unique_ptr<UpdateDocumentMessage>; + using SP = std::shared_ptr<UpdateDocumentMessage>; /** * Constructs a new document message for deserialization. @@ -42,7 +42,7 @@ public: * * @return The update. */ - const DocumentUpdateSP & getDocumentUpdateSP() const { return _documentUpdate; } + DocumentUpdateSP stealDocumentUpdate() const { return std::move(_documentUpdate); } const document::DocumentUpdate & getDocumentUpdate() const { return *_documentUpdate; } /** * Sets the document update to perform. diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ActiveContainerDeactivationWatchdog.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ActiveContainerDeactivationWatchdog.java index 2eb4cc0baa9..e3f6576d293 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ActiveContainerDeactivationWatchdog.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ActiveContainerDeactivationWatchdog.java @@ -32,7 +32,7 @@ import static java.util.stream.Collectors.toList; class ActiveContainerDeactivationWatchdog implements ActiveContainerMetrics, AutoCloseable { static final Duration WATCHDOG_FREQUENCY = Duration.ofMinutes(20); static final Duration ACTIVE_CONTAINER_GRACE_PERIOD = Duration.ofHours(4); - static final Duration GC_TRIGGER_FREQUENCY = ACTIVE_CONTAINER_GRACE_PERIOD.minusMinutes(5); + static final Duration GC_TRIGGER_FREQUENCY = Duration.ofHours(1); // Must be a fraction of ACTIVE_CONTAINER_GRACE_PERIOD static final Duration ENFORCE_DESTRUCTION_GCED_CONTAINERS_FREQUENCY = Duration.ofMinutes(5); private static final Logger log = Logger.getLogger(ActiveContainerDeactivationWatchdog.class.getName()); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java index 2e81ef19f5e..9d2198cedcc 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java @@ -180,7 +180,9 @@ public class NodeAgentImpl implements NodeAgent { @Override public void start(int intervalMillis) { - addDebugMessage("Starting with interval " + intervalMillis + "ms"); + String message = "Starting with interval " + intervalMillis + " ms"; + logger.info(message); + addDebugMessage(message); delaysBetweenEachConvergeMillis = intervalMillis; if (loopThread != null) { throw new RuntimeException("Can not restart a node agent."); @@ -214,6 +216,8 @@ public class NodeAgentImpl implements NodeAgent { } catch (InterruptedException e) { logger.error("Interrupted; Could not stop filebeatrestarter thread"); } + + logger.info("Stopped"); } private void runLocalResumeScriptIfNeeded() { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java index 1885b54e9c0..68bdffd75f9 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java @@ -25,6 +25,7 @@ import com.yahoo.vespa.hosted.provision.node.filter.StateFilter; import com.yahoo.vespa.hosted.provision.persistence.CuratorDatabaseClient; import com.yahoo.vespa.hosted.provision.persistence.DnsNameResolver; import com.yahoo.vespa.hosted.provision.persistence.NameResolver; +import com.yahoo.vespa.hosted.provision.restapi.v2.NotFoundException; import java.time.Clock; import java.time.Duration; @@ -468,24 +469,26 @@ public class NodeRepository extends AbstractComponent { /** * Removes a node. A node must be in a legal state before it can be removed. - * - * @return true if the node was removed, false if it was not found in one of the legal states */ - public boolean remove(String hostname) { - - Node.State[] legalStates = {Node.State.provisioned, Node.State.failed, Node.State.parked}; - Node.State[] legalDynamicStates = {Node.State.provisioned, Node.State.failed, Node.State.parked, Node.State.dirty}; - - Optional<Node> nodeToRemove = getNode(hostname, dynamicAllocationEnabled() ? legalDynamicStates : legalStates); - if ( ! nodeToRemove.isPresent()) return false; + public void remove(String hostname) { + Node nodeToRemove = getNode(hostname).orElseThrow(() -> new NotFoundException("No node with hostname \"" + hostname + '"')); + List<Node.State> legalStates = dynamicAllocationEnabled() ? + Arrays.asList(Node.State.provisioned, Node.State.failed, Node.State.parked, Node.State.dirty) : + Arrays.asList(Node.State.provisioned, Node.State.failed, Node.State.parked); + + if (! legalStates.contains(nodeToRemove.state())) { + throw new IllegalArgumentException("Can only remove node from following states: " + + legalStates.stream().map(Node.State::name).collect(Collectors.joining(", "))); + } - // Only docker nodes are allowed to be deleted in state dirty. - if ( nodeToRemove.get().state().equals(Node.State.dirty)) { - if (!(nodeToRemove.get().flavor().getType().equals(Flavor.Type.DOCKER_CONTAINER))) return false; + if (nodeToRemove.state().equals(Node.State.dirty)) { + if (!(nodeToRemove.flavor().getType().equals(Flavor.Type.DOCKER_CONTAINER))) { + throw new IllegalArgumentException("Only docker nodes can be deleted from state dirty"); + } } - try (Mutex lock = lock(nodeToRemove.get())) { - return db.removeNode(nodeToRemove.get().state(), hostname); + try (Mutex lock = lock(nodeToRemove)) { + db.removeNode(nodeToRemove.state(), hostname); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/History.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/History.java index 8f7317b28eb..3228e6ca17e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/History.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/History.java @@ -65,14 +65,15 @@ public class History { public History recordStateTransition(Node.State from, Node.State to, Agent agent, Instant at) { if (from == to) return this; switch (to) { - case ready: return this.withoutApplicationEvents().with(new Event(Event.Type.readied, agent, at)); - case active: return this.with(new Event(Event.Type.activated, agent, at)); - case inactive: return this.with(new Event(Event.Type.deactivated, agent, at)); - case reserved: return this.with(new Event(Event.Type.reserved, agent, at)); - case failed: return this.with(new Event(Event.Type.failed, agent, at)); - case dirty: return this.with(new Event(Event.Type.deallocated, agent, at)); - case parked: return this.with(new Event(Event.Type.parked, agent, at)); - default: return this; + case provisioned: return this.with(new Event(Event.Type.provisioned, agent, at)); + case ready: return this.withoutApplicationEvents().with(new Event(Event.Type.readied, agent, at)); + case active: return this.with(new Event(Event.Type.activated, agent, at)); + case inactive: return this.with(new Event(Event.Type.deactivated, agent, at)); + case reserved: return this.with(new Event(Event.Type.reserved, agent, at)); + case failed: return this.with(new Event(Event.Type.failed, agent, at)); + case dirty: return this.with(new Event(Event.Type.deallocated, agent, at)); + case parked: return this.with(new Event(Event.Type.parked, agent, at)); + default: return this; } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java index cd63599fed6..8f218d7e6fc 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java @@ -84,6 +84,8 @@ public class CuratorDatabaseClient { for (Node node : nodes) { if (node.state() != expectedState) throw new IllegalArgumentException(node + " is not in the " + node.state() + " state"); + + node = node.with(node.history().recordStateTransition(null, expectedState, Agent.system, clock.instant())); curatorTransaction.add(CuratorOperations.create(toPath(node).getAbsolute(), nodeSerializer.toJson(node))); } transaction.commit(); @@ -108,16 +110,14 @@ public class CuratorDatabaseClient { * * @param state the current state of the node * @param hostName the host name of the node to remove - * @return true if the node was removed, false if it was not found */ - public boolean removeNode(Node.State state, String hostName) { + public void removeNode(Node.State state, String hostName) { Path path = toPath(state, hostName); NestedTransaction transaction = new NestedTransaction(); CuratorTransaction curatorTransaction = curatorDatabase.newCuratorTransactionIn(transaction); curatorTransaction.add(CuratorOperations.delete(path.getAbsolute())); transaction.commit(); log.log(LogLevel.INFO, "Removed: " + state + " node " + hostName); - return true; } /** diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java index 914658302b6..e2ff17ba782 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java @@ -24,7 +24,6 @@ public class CapacityPolicies { this.flavors = flavors; } - /** provides capacity defaults for various environments */ public int decideSize(Capacity requestedCapacity) { int requestedNodes = requestedCapacity.nodeCount(); if (requestedCapacity.isRequired()) return requestedNodes; @@ -39,10 +38,10 @@ public class CapacityPolicies { } public Flavor decideFlavor(Capacity requestedCapacity, ClusterSpec cluster, Optional<String> defaultFlavorOverride) { - // for now, always use requested docker flavor when requested + // for now, always use the requested flavor if a docker flavor is requested Optional<String> requestedFlavor = requestedCapacity.flavor(); if (requestedFlavor.isPresent() && - flavors.getFlavorOrThrow(requestedFlavor.get()).getType() == Flavor.Type.DOCKER_CONTAINER) + flavors.getFlavorOrThrow(requestedFlavor.get()).getType() == Flavor.Type.DOCKER_CONTAINER) return flavors.getFlavorOrThrow(requestedFlavor.get()); String defaultFlavorName = defaultFlavorOverride.isPresent() ? diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java index db75894673e..4e7bb1f7d16 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java @@ -135,10 +135,8 @@ public class NodesApiHandler extends LoggingRequestHandler { */ String hostname = lastElement(path); if (nodeRepository.dynamicAllocationEnabled()) { - if (nodeRepository.remove(hostname)) - return new MessageResponse("Removed " + hostname); - else - throw new NotFoundException("No node in the provisioned, parked, dirty or failed state with hostname " + hostname); + nodeRepository.remove(hostname); + return new MessageResponse("Removed " + hostname); } else { nodeRepository.setReady(hostname); return new MessageResponse("Moved " + hostname + " to ready"); @@ -182,10 +180,8 @@ public class NodesApiHandler extends LoggingRequestHandler { String path = request.getUri().getPath(); if (path.startsWith("/nodes/v2/node/")) { String hostname = lastElement(path); - if (nodeRepository.remove(hostname)) - return new MessageResponse("Removed " + hostname); - else - throw new NotFoundException("No node in the provisioned, parked or failed state with hostname " + hostname); + nodeRepository.remove(hostname); + return new MessageResponse("Removed " + hostname); } else if (path.startsWith("/nodes/v2/maintenance/inactive/")) { return setActive(lastElement(path), true); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTest.java index 12c76638604..bcae9d293f6 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTest.java @@ -13,6 +13,7 @@ import org.junit.Test; import java.nio.charset.StandardCharsets; import java.util.Optional; +import static junit.framework.TestCase.fail; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -36,7 +37,7 @@ public class NodeRepositoryTest { assertEquals(3, tester.getNodes(NodeType.tenant).size()); tester.nodeRepository().park("host2", Agent.system, "Parking to unit test"); - assertTrue(tester.nodeRepository().remove("host2")); + tester.nodeRepository().remove("host2"); assertEquals(2, tester.getNodes(NodeType.tenant).size()); } @@ -71,17 +72,18 @@ public class NodeRepositoryTest { @Test public void only_allow_to_delete_dirty_nodes_when_dynamic_allocation_feature_enabled() { NodeRepositoryTester tester = new NodeRepositoryTester(); - try { - tester.addNode("id1", "host1", "default", NodeType.host); - tester.addNode("id2", "host2", "docker", NodeType.tenant); - tester.nodeRepository().setDirty("host2"); - - assertFalse(tester.nodeRepository().remove("host2")); + tester.addNode("id1", "host1", "default", NodeType.host); + tester.addNode("id2", "host2", "docker", NodeType.tenant); + tester.nodeRepository().setDirty("host2"); - tester.curator().set(Path.fromString("/provision/v1/dynamicDockerAllocation"), new byte[0]); - assertTrue(tester.nodeRepository().remove("host2")); - } finally { - tester.curator().delete(Path.fromString("/provision/v1/dynamicDockerAllocation")); + try { + tester.nodeRepository().remove("host2"); + fail("Should not be able to delete tenant node in state dirty"); + } catch (IllegalArgumentException ignored) { + // Expected } + + tester.curator().set(Path.fromString("/provision/v1/dynamicDockerAllocation"), new byte[0]); + tester.nodeRepository().remove("host2"); } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java index e27ca004862..704ded54479 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java @@ -20,10 +20,12 @@ import com.yahoo.vespa.hosted.provision.testutils.MockDeployer; import org.junit.Test; import java.time.Duration; +import java.util.Arrays; import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Optional; +import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -62,8 +64,9 @@ public class InactiveAndFailedExpirerTest { // One node is set back to ready Node ready = tester.nodeRepository().setReady(Collections.singletonList(dirty.get(0))).get(0); - assertEquals("Allocated history is removed on readying", 1, ready.history().events().size()); - assertEquals(History.Event.Type.readied, ready.history().events().iterator().next().type()); + assertEquals("Allocated history is removed on readying", + Arrays.asList(History.Event.Type.provisioned, History.Event.Type.readied), + ready.history().events().stream().map(History.Event::type).collect(Collectors.toList())); // Dirty times out for the other one tester.advanceTime(Duration.ofMinutes(14)); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ZooKeeperAccessMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ZooKeeperAccessMaintainerTest.java index ee0b8f55a4b..cb46d0a4624 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ZooKeeperAccessMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ZooKeeperAccessMaintainerTest.java @@ -49,7 +49,7 @@ public class ZooKeeperAccessMaintainerTest { assertEquals(asSet("host1,host2,host3,host4,host5,server1,server2"), ZooKeeperServer.getAllowedClientHostnames()); tester.nodeRepository().park("host2", Agent.system, "Parking to unit test"); - assertTrue(tester.nodeRepository().remove("host2")); + tester.nodeRepository().remove("host2"); maintainer.maintain(); assertEquals(2, tester.getNodes(NodeType.tenant).size()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java index deb3378679c..565dfe0457e 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java @@ -401,7 +401,7 @@ public class RestApiTest { // Attempt to DELETE a node which is not put in a deletable state first assertResponse(new Request("http://localhost:8080/nodes/v2/node/host2.yahoo.com", new byte[0], Request.Method.DELETE), - 404, "{\"error-code\":\"NOT_FOUND\",\"message\":\"No node in the provisioned, parked or failed state with hostname host2.yahoo.com\"}"); + 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Can only remove node from following states: provisioned, failed, parked, dirty\"}"); // PUT current restart generation with string instead of long assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node1.json index 075ce1693cb..56523f58164 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node1.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node1.json @@ -37,6 +37,11 @@ "wantToDeprovision": false, "history": [ { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { "event": "readied", "at": 123, "agent": "system" diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node10.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node10.json index 120d6286634..cb24df66cf8 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node10.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node10.json @@ -42,6 +42,11 @@ "wantToDeprovision": false, "history": [ { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { "event": "readied", "at": 123, "agent": "system" diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node11.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node11.json index 9b3feffad42..e0e899afbef 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node11.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node11.json @@ -1,26 +1,38 @@ { - "url":"http://localhost:8080/nodes/v2/node/host11.yahoo.com", - "id":"host11.yahoo.com", - "state":"provisioned", + "url": "http://localhost:8080/nodes/v2/node/host11.yahoo.com", + "id": "host11.yahoo.com", + "state": "provisioned", "type": "tenant", - "hostname":"host11.yahoo.com", - "parentHostname":"parent.host.yahoo.com", - "openStackId":"host11.yahoo.com", - "flavor":"docker", - "canonicalFlavor":"docker", - "minDiskAvailableGb":100.0, - "minMainMemoryAvailableGb":0.5, - "description":"Flavor-name-is-docker", - "minCpuCores":0.2, - "fastDisk":true, - "environment":"DOCKER_CONTAINER", - "rebootGeneration":0, - "currentRebootGeneration":0, - "failCount":0, - "hardwareFailure":false, - "wantToRetire":false, - "wantToDeprovision" : false, - "history":[], - "ipAddresses":["::1", "127.0.0.1"], - "additionalIpAddresses":["::10","::11"] + "hostname": "host11.yahoo.com", + "parentHostname": "parent.host.yahoo.com", + "openStackId": "host11.yahoo.com", + "flavor": "docker", + "canonicalFlavor": "docker", + "minDiskAvailableGb": 100.0, + "minMainMemoryAvailableGb": 0.5, + "description": "Flavor-name-is-docker", + "minCpuCores": 0.2, + "fastDisk": true, + "environment": "DOCKER_CONTAINER", + "rebootGeneration": 0, + "currentRebootGeneration": 0, + "failCount": 0, + "hardwareFailure": false, + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "provisioned", + "at": 123, + "agent": "system" + } + ], + "ipAddresses": [ + "::1", + "127.0.0.1" + ], + "additionalIpAddresses": [ + "::10", + "::11" + ] } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node2.json index 52864fc165c..30057dda1d7 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node2.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node2.json @@ -37,6 +37,11 @@ "wantToDeprovision": false, "history": [ { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { "event": "readied", "at": 123, "agent": "system" diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node3.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node3.json index 4b7af75ee3c..eb13d077d7f 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node3.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node3.json @@ -22,6 +22,11 @@ "wantToDeprovision": false, "history": [ { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { "event": "readied", "at": 123, "agent": "system" diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json index 4082db74ff4..1fa8feb4586 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json @@ -44,6 +44,11 @@ "wantToDeprovision" : true, "history": [ { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { "event": "readied", "at": 123, "agent": "system" diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4.json index 10b5689f8ce..1e9138283f7 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4.json @@ -42,6 +42,11 @@ "wantToDeprovision": false, "history": [ { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { "event": "readied", "at": 123, "agent": "system" diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5-after-changes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5-after-changes.json index bf81509b79a..1e1ea1a2445 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5-after-changes.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5-after-changes.json @@ -22,6 +22,11 @@ "wantToDeprovision": false, "history": [ { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { "event": "readied", "at": 123, "agent": "system" diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5.json index 1fc001fa224..f8b9fb72e5e 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5.json @@ -26,6 +26,11 @@ "wantToDeprovision": false, "history": [ { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { "event": "readied", "at": 123, "agent": "system" diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node55.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node55.json index 3c483fa3412..7d07037d076 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node55.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node55.json @@ -7,19 +7,33 @@ "openStackId": "node55", "flavor": "default", "canonicalFlavor": "default", - "minDiskAvailableGb":400.0, - "minMainMemoryAvailableGb":16.0, - "description":"Flavor-name-is-default", - "minCpuCores":2.0, - "fastDisk":true, - "environment":"BARE_METAL", + "minDiskAvailableGb": 400.0, + "minMainMemoryAvailableGb": 16.0, + "description": "Flavor-name-is-default", + "minCpuCores": 2.0, + "fastDisk": true, + "environment": "BARE_METAL", "rebootGeneration": 1, "currentRebootGeneration": 0, "failCount": 0, - "hardwareFailure" : false, - "wantToRetire" : false, - "wantToDeprovision" : false, - "history":[{"event":"deallocated","at":123,"agent":"system"}], - "ipAddresses":["::1", "127.0.0.1"], - "additionalIpAddresses":[] + "hardwareFailure": false, + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { + "event": "deallocated", + "at": 123, + "agent": "system" + } + ], + "ipAddresses": [ + "::1", + "127.0.0.1" + ], + "additionalIpAddresses": [] } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-after-changes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-after-changes.json index adb7ce18c80..be9f3d78663 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-after-changes.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-after-changes.json @@ -27,16 +27,40 @@ }, "restartGeneration": 0, "currentRestartGeneration": 0, - "wantedDockerImage":"docker-registry.domain.tld:8080/dist/vespa:6.42.0", - "wantedVespaVersion":"6.42.0", + "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", + "wantedVespaVersion": "6.42.0", "rebootGeneration": 1, "currentRebootGeneration": 0, "failCount": 0, - "hardwareFailure" : false, - "wantToRetire" : false, - "wantToDeprovision" : false, - "history":[{"event":"readied","at":123,"agent":"system"},{"event":"reserved","at":123,"agent":"application"},{"event":"activated","at":123,"agent":"application"}], - "ipAddresses":["::1", "127.0.0.1"], - "additionalIpAddresses":[], - "hardwareDivergence":"{\"actualCpuCores\":2}" + "hardwareFailure": false, + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { + "event": "readied", + "at": 123, + "agent": "system" + }, + { + "event": "reserved", + "at": 123, + "agent": "application" + }, + { + "event": "activated", + "at": 123, + "agent": "application" + } + ], + "ipAddresses": [ + "::1", + "127.0.0.1" + ], + "additionalIpAddresses": [], + "hardwareDivergence": "{\"actualCpuCores\":2}" } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6.json index 750ebbd695e..bd3b15f16ba 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6.json @@ -37,6 +37,11 @@ "wantToDeprovision": false, "history": [ { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { "event": "readied", "at": 123, "agent": "system" diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node7.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node7.json index 9b52598cc67..b1c654b4e00 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node7.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node7.json @@ -7,19 +7,28 @@ "openStackId": "node7", "flavor": "default", "canonicalFlavor": "default", - "minDiskAvailableGb":400.0, - "minMainMemoryAvailableGb":16.0, - "description":"Flavor-name-is-default", - "minCpuCores":2.0, - "fastDisk":true, - "environment":"BARE_METAL", + "minDiskAvailableGb": 400.0, + "minMainMemoryAvailableGb": 16.0, + "description": "Flavor-name-is-default", + "minCpuCores": 2.0, + "fastDisk": true, + "environment": "BARE_METAL", "rebootGeneration": 0, "currentRebootGeneration": 0, "failCount": 0, - "hardwareFailure" : false, - "wantToRetire" : false, - "wantToDeprovision" : false, - "history":[], - "ipAddresses":["::1", "127.0.0.1"], - "additionalIpAddresses":[] + "hardwareFailure": false, + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "provisioned", + "at": 123, + "agent": "system" + } + ], + "ipAddresses": [ + "::1", + "127.0.0.1" + ], + "additionalIpAddresses": [] } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node8.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node8.json index 34fb9d13d81..0fa0bf6631d 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node8.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node8.json @@ -7,19 +7,27 @@ "openStackId": "host8.yahoo.com", "flavor": "default", "canonicalFlavor": "default", - "minDiskAvailableGb":400.0, - "minMainMemoryAvailableGb":16.0, - "description":"Flavor-name-is-default", - "minCpuCores":2.0, - "fastDisk":true, - "environment":"BARE_METAL", + "minDiskAvailableGb": 400.0, + "minMainMemoryAvailableGb": 16.0, + "description": "Flavor-name-is-default", + "minCpuCores": 2.0, + "fastDisk": true, + "environment": "BARE_METAL", "rebootGeneration": 0, "currentRebootGeneration": 0, "failCount": 0, - "hardwareFailure" : false, - "wantToRetire" : false, - "wantToDeprovision" : false, - "history":[], - "ipAddresses":["127.0.0.1"], - "additionalIpAddresses":[] + "hardwareFailure": false, + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "provisioned", + "at": 123, + "agent": "system" + } + ], + "ipAddresses": [ + "127.0.0.1" + ], + "additionalIpAddresses": [] } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node9.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node9.json index f0695598264..43405f0aafc 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node9.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node9.json @@ -7,19 +7,27 @@ "openStackId": "host9.yahoo.com", "flavor": "large-variant", "canonicalFlavor": "large", - "minDiskAvailableGb":2000.0, - "minMainMemoryAvailableGb":128.0, - "description":"Flavor-name-is-large-variant", - "minCpuCores":64.0, - "fastDisk":true, - "environment":"BARE_METAL", + "minDiskAvailableGb": 2000.0, + "minMainMemoryAvailableGb": 128.0, + "description": "Flavor-name-is-large-variant", + "minCpuCores": 64.0, + "fastDisk": true, + "environment": "BARE_METAL", "rebootGeneration": 0, "currentRebootGeneration": 0, "failCount": 0, - "hardwareFailure" : false, - "wantToRetire" : false, - "wantToDeprovision" : false, - "history":[], - "ipAddresses":["::1", "127.0.0.1"], - "additionalIpAddresses":[] + "hardwareFailure": false, + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "provisioned", + "at": 123, + "agent": "system" + } + ], + "ipAddresses": [ + "::1", + "127.0.0.1"], + "additionalIpAddresses": [] } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent1.json index da6742275a4..190dbc41f34 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent1.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent1.json @@ -17,15 +17,23 @@ "currentRebootGeneration": 0, "failCount": 0, "hardwareFailure": false, - "wantToRetire" : false, - "wantToDeprovision" : false, + "wantToRetire": false, + "wantToDeprovision": false, "history": [ { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { "event": "readied", "at": 123, "agent": "system" } ], - "ipAddresses":["::1", "127.0.0.1"], - "additionalIpAddresses":[] + "ipAddresses": [ + "::1", + "127.0.0.1" + ], + "additionalIpAddresses": [] } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent2.json index 65735ecb37c..c7f6344d974 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent2.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent2.json @@ -17,9 +17,18 @@ "currentRebootGeneration": 0, "failCount": 0, "hardwareFailure": false, - "wantToRetire" : false, - "wantToDeprovision" : false, - "history": [], - "ipAddresses":["::1", "127.0.0.1"], - "additionalIpAddresses":[] + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "provisioned", + "at": 123, + "agent": "system" + } + ], + "ipAddresses": [ + "::1", + "127.0.0.1" + ], + "additionalIpAddresses": [] } diff --git a/searchcore/src/apps/vespa-dump-feed/vespa-dump-feed.cpp b/searchcore/src/apps/vespa-dump-feed/vespa-dump-feed.cpp index d942049192a..26a745df148 100644 --- a/searchcore/src/apps/vespa-dump-feed/vespa-dump-feed.cpp +++ b/searchcore/src/apps/vespa-dump-feed/vespa-dump-feed.cpp @@ -9,6 +9,7 @@ #include <vespa/messagebus/destinationsession.h> #include <vespa/messagebus/protocolset.h> #include <vespa/messagebus/rpcmessagebus.h> +#include <vespa/messagebus/network/rpcnetworkparams.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/util/signalhandler.h> #include <vespa/vespalib/util/slaveproc.h> diff --git a/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp b/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp index c9761ee7e6e..a389e702c69 100644 --- a/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp +++ b/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp @@ -2017,6 +2017,53 @@ TEST("require that document sizes are saved") TEST_DO(assertSize(dms4, 3, 1)); } +namespace { + +void +assertLidGidFound(uint32_t lid, DocumentMetaStore &dms) +{ + GlobalId gid = createGid(lid); + EXPECT_TRUE(assertLid(lid, gid, dms)); + EXPECT_TRUE(assertGid(gid, lid, dms)); + EXPECT_TRUE(dms.validLid(lid)); +} + +void +assertLidGidNotFound(uint32_t lid, DocumentMetaStore &dms) +{ + GlobalId gid = createGid(lid); + uint32_t resultLid; + GlobalId resultGid; + EXPECT_FALSE(dms.getLid(gid, resultLid)); + EXPECT_FALSE(dms.getGid(lid, resultGid)); + EXPECT_FALSE(dms.validLid(lid)); +} + +} + +TEST("require that multiple lids can be removed with removeBatch()") +{ + DocumentMetaStore dms(createBucketDB()); + dms.constructFreeList(); + TEST_DO(addLid(dms, 1)); + TEST_DO(addLid(dms, 2)); + TEST_DO(addLid(dms, 3)); + TEST_DO(addLid(dms, 4)); + + TEST_DO(assertLidGidFound(1, dms)); + TEST_DO(assertLidGidFound(2, dms)); + TEST_DO(assertLidGidFound(3, dms)); + TEST_DO(assertLidGidFound(4, dms)); + + dms.removeBatch({1, 3}, 5); + dms.removeBatchComplete({1, 3}); + + TEST_DO(assertLidGidNotFound(1, dms)); + TEST_DO(assertLidGidFound(2, dms)); + TEST_DO(assertLidGidNotFound(3, dms)); + TEST_DO(assertLidGidFound(4, dms)); +} + } TEST_MAIN() diff --git a/searchcore/src/tests/proton/summaryengine/summaryengine.cpp b/searchcore/src/tests/proton/summaryengine/summaryengine.cpp index a17c19804c5..82c7b22c8c7 100644 --- a/searchcore/src/tests/proton/summaryengine/summaryengine.cpp +++ b/searchcore/src/tests/proton/summaryengine/summaryengine.cpp @@ -6,7 +6,7 @@ #include <vespa/searchlib/util/rawbuf.h> #include <vespa/searchlib/util/slime_output_raw_buf_adapter.h> #include <vespa/vespalib/data/databuffer.h> -#include <vespa/document/util/compressor.h> +#include <vespa/vespalib/util/compressor.h> #include <vespa/searchlib/common/transport.h> #include <vespa/fnet/frt/rpcrequest.h> #include <vespa/log/log.h> @@ -20,6 +20,8 @@ using vespalib::stringref; using vespalib::ConstBufferRef; using vespalib::DataBuffer; using vespalib::Memory; +using vespalib::compression::CompressionConfig; + namespace proton { @@ -377,7 +379,7 @@ TEST("requireThatSlimeInterfaceWorksFine") { } void -verifyReply(size_t count, document::CompressionConfig::Type encoding, size_t orgSize, size_t compressedSize, +verifyReply(size_t count, CompressionConfig::Type encoding, size_t orgSize, size_t compressedSize, FRT_RPCRequest *request) { FRT_Values &ret = *request->GetReturn(); EXPECT_EQUAL(encoding, ret[0]._intval8); @@ -386,7 +388,8 @@ verifyReply(size_t count, document::CompressionConfig::Type encoding, size_t org DataBuffer uncompressed; ConstBufferRef blob(ret[2]._data._buf, ret[2]._data._len); - compression::decompress(CompressionConfig::toType(ret[0]._intval8), ret[1]._intval32, blob, uncompressed, false); + vespalib::compression::decompress(CompressionConfig::toType(ret[0]._intval8), ret[1]._intval32, + blob, uncompressed, false); EXPECT_EQUAL(orgSize, uncompressed.getDataLen()); vespalib::Slime summaries; @@ -396,8 +399,8 @@ verifyReply(size_t count, document::CompressionConfig::Type encoding, size_t org void verifyRPC(size_t count, - document::CompressionConfig::Type requestCompression, size_t requestSize, size_t requestBlobSize, - document::CompressionConfig::Type replyCompression, size_t replySize, size_t replyBlobSize) { + CompressionConfig::Type requestCompression, size_t requestSize, size_t requestBlobSize, + CompressionConfig::Type replyCompression, size_t replySize, size_t replyBlobSize) { Server server; vespalib::Slime slimeRequest = createSlimeRequestLarger(count); vespalib::SimpleBuffer buf; @@ -406,8 +409,9 @@ verifyRPC(size_t count, CompressionConfig config(requestCompression, 9, 100); DataBuffer compressed(const_cast<char *>(buf.get().data), buf.get().size); - CompressionConfig::Type type = compression::compress(config, ConstBufferRef(buf.get().data, buf.get().size), - compressed, true); + CompressionConfig::Type type = vespalib::compression::compress(config, + ConstBufferRef(buf.get().data, buf.get().size), + compressed, true); EXPECT_EQUAL(type, requestCompression); FRT_RPCRequest *request = new FRT_RPCRequest(); @@ -424,9 +428,9 @@ verifyRPC(size_t count, } TEST("requireThatRPCInterfaceWorks") { - verifyRPC(1, document::CompressionConfig::NONE, 55, 55, document::CompressionConfig::NONE, 38, 38); - verifyRPC(100, document::CompressionConfig::NONE, 2631, 2631, document::CompressionConfig::LZ4, 1426, 46); - verifyRPC(100, document::CompressionConfig::LZ4, 2631, 69, document::CompressionConfig::LZ4, 1426, 46); + verifyRPC(1, CompressionConfig::NONE, 55, 55, CompressionConfig::NONE, 38, 38); + verifyRPC(100, CompressionConfig::NONE, 2631, 2631, CompressionConfig::LZ4, 1426, 46); + verifyRPC(100, CompressionConfig::LZ4, 2631, 69, CompressionConfig::LZ4, 1426, 46); } } diff --git a/searchcore/src/vespa/searchcore/fdispatch/program/fdispatch.cpp b/searchcore/src/vespa/searchcore/fdispatch/program/fdispatch.cpp index fc0a57ff519..b6a1133d01d 100644 --- a/searchcore/src/vespa/searchcore/fdispatch/program/fdispatch.cpp +++ b/searchcore/src/vespa/searchcore/fdispatch/program/fdispatch.cpp @@ -19,7 +19,7 @@ LOG_SETUP(".fdispatch"); using search::fs4transport::FS4PersistentPacketStreamer; using vespa::config::search::core::FdispatchrcConfig; using vespa::config::search::core::internal::InternalFdispatchrcType; -using document::CompressionConfig; +using vespalib::compression::CompressionConfig; char FastS_VersionTag[] = V_TAG; diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp index e65209bf526..79d82108ee8 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp @@ -13,7 +13,6 @@ #include <vespa/searchsummary/docsummary/docsumconfig.h> #include <vespa/vespalib/util/exceptions.h> #include <sstream> -#include <future> #include <vespa/log/log.h> LOG_SETUP(".proton.docsummary.summarymanager"); @@ -26,6 +25,8 @@ using namespace vespa::config::search::summary; using namespace vespa::config::search; using vespalib::make_string; using vespalib::IllegalArgumentException; +using vespalib::compression::CompressionConfig; + using search::DocumentStore; using search::IDocumentStore; using search::LogDocumentStore; @@ -160,13 +161,13 @@ SummaryManager::createSummarySetup(const SummaryConfig & summaryCfg, namespace { template<typename T> -document::CompressionConfig +CompressionConfig deriveCompression(const T & config) { - document::CompressionConfig compression; + CompressionConfig compression; if (config.type == T::LZ4) { - compression.type = document::CompressionConfig::LZ4; + compression.type = CompressionConfig::LZ4; } else if (config.type == T::ZSTD) { - compression.type = document::CompressionConfig::ZSTD; + compression.type = CompressionConfig::ZSTD; } compression.compressionLevel = config.level; return compression; diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp index c47d8592b69..2fc395d4438 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp @@ -591,7 +591,7 @@ DocumentMetaStore::updateMetaData(DocId lid, } bool -DocumentMetaStore::remove(DocId lid) +DocumentMetaStore::remove(DocId lid, BucketDBOwner::Guard &bucketGuard) { if (!validLid(lid)) { return false; @@ -606,15 +606,23 @@ DocumentMetaStore::remove(DocId lid) lid, gid.toString().c_str())); } _lidAlloc.unregisterLid(lid); - incGeneration(); RawDocumentMetaData &oldMetaData = _metaDataStore[lid]; - _bucketDB->takeGuard()->remove(oldMetaData.getGid(), - oldMetaData.getBucketId().stripUnused(), - oldMetaData.getTimestamp(), oldMetaData.getDocSize(), - _subDbType); + bucketGuard->remove(oldMetaData.getGid(), + oldMetaData.getBucketId().stripUnused(), + oldMetaData.getTimestamp(), oldMetaData.getDocSize(), + _subDbType); return true; } +bool +DocumentMetaStore::remove(DocId lid) +{ + BucketDBOwner::Guard bucketGuard = _bucketDB->takeGuard(); + bool result = remove(lid, bucketGuard); + incGeneration(); + return result; +} + void DocumentMetaStore::removeComplete(DocId lid) { @@ -649,14 +657,16 @@ DocumentMetaStore::move(DocId fromLid, DocId toLid) void DocumentMetaStore::removeBatch(const std::vector<DocId> &lidsToRemove, const uint32_t docIdLimit) { + BucketDBOwner::Guard bucketGuard = _bucketDB->takeGuard(); for (const auto &lid : lidsToRemove) { assert(lid > 0 && lid < docIdLimit); (void) docIdLimit; - bool removed = remove(lid); + bool removed = remove(lid, bucketGuard); assert(removed); (void) removed; } + incGeneration(); } void diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.h index 613952f5c4f..180fb438808 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.h +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.h @@ -136,6 +136,8 @@ private: VESPA_DLL_LOCAL DocId readNextDoc(documentmetastore::Reader & reader, TreeType::Builder & treeBuilder); + bool remove(DocId lid, BucketDBOwner::Guard &bucketGuard); + public: typedef TreeType::Iterator Iterator; typedef TreeType::ConstIterator ConstIterator; diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp index ed350453da2..d95b0fd44d1 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp @@ -48,7 +48,7 @@ using search::index::SchemaBuilder; using search::transactionlog::DomainStats; using vespa::config::search::core::ProtonConfig; using vespa::config::search::core::internal::InternalProtonType; -using document::CompressionConfig; +using vespalib::compression::CompressionConfig; namespace proton { diff --git a/searchcore/src/vespa/searchcore/proton/server/rpc_hooks.cpp b/searchcore/src/vespa/searchcore/proton/server/rpc_hooks.cpp index 2e8c985fc0a..0595cb5d983 100644 --- a/searchcore/src/vespa/searchcore/proton/server/rpc_hooks.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/rpc_hooks.cpp @@ -9,7 +9,7 @@ LOG_SETUP(".proton.server.rtchooks"); using namespace vespalib; -using document::CompressionConfig; +using vespalib::compression::CompressionConfig; namespace { diff --git a/searchcore/src/vespa/searchcore/proton/summaryengine/docsum_by_slime.cpp b/searchcore/src/vespa/searchcore/proton/summaryengine/docsum_by_slime.cpp index 9a37e0ae495..79928f96d7a 100644 --- a/searchcore/src/vespa/searchcore/proton/summaryengine/docsum_by_slime.cpp +++ b/searchcore/src/vespa/searchcore/proton/summaryengine/docsum_by_slime.cpp @@ -1,6 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "docsum_by_slime.h" -#include <vespa/document/util/compressor.h> +#include <vespa/vespalib/util/compressor.h> #include <vespa/searchlib/util/slime_output_raw_buf_adapter.h> #include <vespa/searchlib/common/packets.h> #include <vespa/fnet/frt/rpcrequest.h> @@ -23,7 +23,7 @@ using vespalib::slime::ArrayTraverser; using vespalib::SimpleBuffer; using vespalib::DataBuffer; using vespalib::ConstBufferRef; -using document::CompressionConfig; +using vespalib::compression::CompressionConfig; namespace { @@ -99,8 +99,8 @@ DocsumByRPC::DocsumByRPC(DocsumBySlime & slimeDocsumServer) : void DocsumByRPC::getDocsums(FRT_RPCRequest & req) { - using document::compression::decompress; - using document::compression::compress; + using vespalib::compression::decompress; + using vespalib::compression::compress; FRT_Values &arg = *req.GetParams(); uint8_t encoding = arg[0]._intval8; uint32_t uncompressedSize = arg[1]._intval32; diff --git a/searchlib/src/apps/docstore/create-idx-from-dat.cpp b/searchlib/src/apps/docstore/create-idx-from-dat.cpp index ac3b34a3b41..6124b36de8d 100644 --- a/searchlib/src/apps/docstore/create-idx-from-dat.cpp +++ b/searchlib/src/apps/docstore/create-idx-from-dat.cpp @@ -39,7 +39,7 @@ bool tryDecode(size_t chunks, size_t offset, const char * p, size_t sz, size_t n } bool validUncompressed(const char * n, size_t offset) { - return (n[1] == document::CompressionConfig::NONE) && + return (n[1] == vespalib::compression::CompressionConfig::NONE) && (n[2] == 0) && (n[3] == 0) && (n[4] == 0) && diff --git a/searchlib/src/tests/common/packets/packets_test.cpp b/searchlib/src/tests/common/packets/packets_test.cpp index 7504e1c1570..cf0e858a014 100644 --- a/searchlib/src/tests/common/packets/packets_test.cpp +++ b/searchlib/src/tests/common/packets/packets_test.cpp @@ -8,6 +8,8 @@ #include <vespa/fnet/controlpacket.h> using namespace search::fs4transport; +using vespalib::compression::CompressionConfig; + // ---------------------------------------------------------------------------- // @@ -524,7 +526,7 @@ TEST("test pre serializing packets with compression") { EXPECT_EQUAL(src->GetLength(), decoded->GetLength()); FS4PersistentPacketStreamer::Instance.SetCompressionLimit(100); FS4Packet_PreSerialized serialized(*src); - EXPECT_EQUAL(218u | (document::CompressionConfig::LZ4 << 24), serialized.GetPCODE()); + EXPECT_EQUAL(218u | (CompressionConfig::LZ4 << 24), serialized.GetPCODE()); EXPECT_GREATER_EQUAL(321u, serialized.GetLength()); FNET_Packet::UP decoded2(testEncodeDecode(serialized)); EXPECT_EQUAL(500u, decoded2->GetLength()); diff --git a/searchlib/src/tests/docstore/chunk/chunk_test.cpp b/searchlib/src/tests/docstore/chunk/chunk_test.cpp index 4687f45acde..84ac877c54d 100644 --- a/searchlib/src/tests/docstore/chunk/chunk_test.cpp +++ b/searchlib/src/tests/docstore/chunk/chunk_test.cpp @@ -12,7 +12,7 @@ LOG_SETUP("chunk_test"); using namespace search; -using document::CompressionConfig; +using vespalib::compression::CompressionConfig; TEST("require that Chunk obey limits") { diff --git a/searchlib/src/tests/docstore/document_store/document_store_test.cpp b/searchlib/src/tests/docstore/document_store/document_store_test.cpp index e3e4a1432d1..e8c2173a87f 100644 --- a/searchlib/src/tests/docstore/document_store/document_store_test.cpp +++ b/searchlib/src/tests/docstore/document_store/document_store_test.cpp @@ -5,6 +5,7 @@ #include <vespa/document/repo/documenttyperepo.h> using namespace search; +using CompressionConfig = vespalib::compression::CompressionConfig; document::DocumentTypeRepo repo; @@ -43,7 +44,7 @@ struct NullDataStore : IDataStore { }; TEST_FFF("require that uncache docstore lookups are counted", - DocumentStore::Config(document::CompressionConfig::NONE, 0, 0), + DocumentStore::Config(CompressionConfig::NONE, 0, 0), NullDataStore(), DocumentStore(f1, f2)) { EXPECT_EQUAL(0u, f3.getCacheStats().misses); @@ -52,7 +53,7 @@ TEST_FFF("require that uncache docstore lookups are counted", } TEST_FFF("require that cached docstore lookups are counted", - DocumentStore::Config(document::CompressionConfig::NONE, 100000, 100), + DocumentStore::Config(CompressionConfig::NONE, 100000, 100), NullDataStore(), DocumentStore(f1, f2)) { EXPECT_EQUAL(0u, f3.getCacheStats().misses); diff --git a/searchlib/src/tests/docstore/document_store/visitcache_test.cpp b/searchlib/src/tests/docstore/document_store/visitcache_test.cpp index 70e0c7f01fb..d5d95097c66 100644 --- a/searchlib/src/tests/docstore/document_store/visitcache_test.cpp +++ b/searchlib/src/tests/docstore/document_store/visitcache_test.cpp @@ -56,11 +56,12 @@ void verifyAB(const BlobSet & a) { using B=vespalib::ConstBufferRef; TEST("require that BlobSet can be built") { + using CompressionConfig = vespalib::compression::CompressionConfig; BlobSet a; a.append(7, B("aaaaaa",6)); a.append(9, B("bbbbb",5)); verifyAB(a); - document::CompressionConfig cfg(document::CompressionConfig::LZ4); + CompressionConfig cfg(CompressionConfig::LZ4); CompressedBlobSet ca(cfg, a); BlobSet b = ca.getBlobSet(); verifyAB(b); diff --git a/searchlib/src/tests/docstore/document_store_visitor/document_store_visitor_test.cpp b/searchlib/src/tests/docstore/document_store_visitor/document_store_visitor_test.cpp index 944d0a543f5..1c7053500c7 100644 --- a/searchlib/src/tests/docstore/document_store_visitor/document_store_visitor_test.cpp +++ b/searchlib/src/tests/docstore/document_store_visitor/document_store_visitor_test.cpp @@ -24,7 +24,7 @@ using document::Document; using document::DocumentId; using document::DocumentType; using document::DocumentTypeRepo; -using document::CompressionConfig; +using vespalib::compression::CompressionConfig; using vespalib::asciistream; using index::DummyFileHeaderContext; diff --git a/searchlib/src/tests/docstore/file_chunk/file_chunk_test.cpp b/searchlib/src/tests/docstore/file_chunk/file_chunk_test.cpp index 6d75c6365f9..598913a3222 100644 --- a/searchlib/src/tests/docstore/file_chunk/file_chunk_test.cpp +++ b/searchlib/src/tests/docstore/file_chunk/file_chunk_test.cpp @@ -113,6 +113,7 @@ struct ReadFixture : public FixtureBase { struct WriteFixture : public FixtureBase { WriteableFileChunk chunk; + using CompressionConfig = vespalib::compression::CompressionConfig; WriteFixture(const vespalib::string &baseName, uint32_t docIdLimit, @@ -124,7 +125,7 @@ struct WriteFixture : public FixtureBase { baseName, serialNum, docIdLimit, - WriteableFileChunk::Config(document::CompressionConfig(), 0x1000), + WriteableFileChunk::Config(CompressionConfig(), 0x1000), tuneFile, fileHeaderCtx, &bucketizer, diff --git a/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp b/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp index 82a158fd53e..ed6afb06681 100644 --- a/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp +++ b/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp @@ -15,7 +15,6 @@ #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/threadstackexecutor.h> #include <iomanip> -#include <iostream> using document::BucketId; using namespace search::docstore; @@ -36,7 +35,7 @@ public: using namespace search; using namespace search::docstore; using search::index::DummyFileHeaderContext; -using document::CompressionConfig; +using vespalib::compression::CompressionConfig; namespace { diff --git a/searchlib/src/tests/docstore/store_by_bucket/store_by_bucket_test.cpp b/searchlib/src/tests/docstore/store_by_bucket/store_by_bucket_test.cpp index 3faf9395ec5..e9514c1d385 100644 --- a/searchlib/src/tests/docstore/store_by_bucket/store_by_bucket_test.cpp +++ b/searchlib/src/tests/docstore/store_by_bucket/store_by_bucket_test.cpp @@ -4,17 +4,17 @@ #include <vespa/document/bucket/bucketid.h> #include <vespa/document/base/documentid.h> -#include <vespa/log/log.h> #include <vespa/searchlib/docstore/storebybucket.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/stllike/hash_set.h> #include <vespa/vespalib/util/threadstackexecutor.h> +#include <vespa/log/log.h> LOG_SETUP("store_by_bucket_test"); using namespace search::docstore; using document::BucketId; -using document::CompressionConfig; +using vespalib::compression::CompressionConfig; vespalib::string createPayload(BucketId b) { diff --git a/searchlib/src/vespa/searchlib/attribute/attribute_header.cpp b/searchlib/src/vespa/searchlib/attribute/attribute_header.cpp index 0848d9996c3..4a69e0a827d 100644 --- a/searchlib/src/vespa/searchlib/attribute/attribute_header.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attribute_header.cpp @@ -2,9 +2,9 @@ #include "attribute_header.h" #include <vespa/vespalib/data/fileheader.h> +#include <vespa/vespalib/data/databuffer.h> -namespace search { -namespace attribute { +namespace search::attribute { namespace { @@ -168,5 +168,4 @@ AttributeHeader::hasWeightedSetType() const return _collectionType.isWeightedSet(); } -} // namespace search::attribute -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/attribute/imported_search_context.cpp b/searchlib/src/vespa/searchlib/attribute/imported_search_context.cpp index 9d351eadd62..0834df280fd 100644 --- a/searchlib/src/vespa/searchlib/attribute/imported_search_context.cpp +++ b/searchlib/src/vespa/searchlib/attribute/imported_search_context.cpp @@ -33,6 +33,7 @@ ImportedSearchContext::ImportedSearchContext( _target_attribute(*_imported_attribute.getTargetAttribute()), _target_search_context(_target_attribute.getSearch(std::move(term), params)), _referencedLids(_reference_attribute.getReferencedLids()), + _referencedLidLimit(_target_attribute.getCommittedDocIdLimit()), _merger(_reference_attribute.getCommittedDocIdLimit()), _fetchPostingsDone(false) { @@ -93,12 +94,16 @@ struct TargetResult { TargetResult getTargetResult(ReverseMappingRefs reverseMappingRefs, const ReverseMapping &reverseMapping, - SearchContext &target_search_context) + SearchContext &target_search_context, + uint32_t committedDocIdLimit) { TargetResult targetResult; fef::TermFieldMatchData matchData; auto targetItr = target_search_context.createIterator(&matchData, true); uint32_t docIdLimit = reverseMappingRefs.size(); + if (docIdLimit > committedDocIdLimit) { + docIdLimit = committedDocIdLimit; + } uint32_t lid = 1; targetItr->initRange(1, docIdLimit); while (lid < docIdLimit) { @@ -147,9 +152,12 @@ public: void ImportedSearchContext::makeMergedPostings() { + uint32_t committedTargetDocIdLimit = _target_attribute.getCommittedDocIdLimit(); + std::atomic_thread_fence(std::memory_order_acquire); TargetResult targetResult(getTargetResult(_reference_attribute.getReverseMappingRefs(), _reference_attribute.getReverseMapping(), - *_target_search_context)); + *_target_search_context, + committedTargetDocIdLimit)); _merger.reserveArray(targetResult.weightedRefs.size(), targetResult.sizeSum); const auto &reverseMapping = _reference_attribute.getReverseMapping(); for (const auto &weightedRef : targetResult.weightedRefs) { @@ -183,13 +191,5 @@ const vespalib::string& ImportedSearchContext::attributeName() const { return _imported_attribute.getName(); } -bool ImportedSearchContext::cmp(DocId docId, int32_t& weight) const { - return _target_search_context->cmp(_referencedLids[docId], weight); -} - -bool ImportedSearchContext::cmp(DocId docId) const { - return _target_search_context->cmp(_referencedLids[docId]); -} - } // attribute } // search diff --git a/searchlib/src/vespa/searchlib/attribute/imported_search_context.h b/searchlib/src/vespa/searchlib/attribute/imported_search_context.h index 9be4578fac0..c142754e302 100644 --- a/searchlib/src/vespa/searchlib/attribute/imported_search_context.h +++ b/searchlib/src/vespa/searchlib/attribute/imported_search_context.h @@ -35,9 +35,15 @@ class ImportedSearchContext : public ISearchContext { const AttributeVector& _target_attribute; std::unique_ptr<AttributeVector::SearchContext> _target_search_context; ReferencedLids _referencedLids; + uint32_t _referencedLidLimit; PostingListMerger<int32_t> _merger; bool _fetchPostingsDone; + uint32_t getReferencedLid(uint32_t lid) const { + uint32_t referencedLid = _referencedLids[lid]; + return ((referencedLid >= _referencedLidLimit) ? 0u : referencedLid); + } + void makeMergedPostings(); public: ImportedSearchContext(std::unique_ptr<QueryTermSimple> term, @@ -62,8 +68,13 @@ public: using DocId = IAttributeVector::DocId; - bool cmp(DocId docId, int32_t& weight) const; - bool cmp(DocId docId) const; + bool cmp(DocId docId, int32_t& weight) const { + return _target_search_context->cmp(getReferencedLid(docId), weight); + } + + bool cmp(DocId docId) const { + return _target_search_context->cmp(getReferencedLid(docId)); + } const ReferenceAttribute& attribute() const noexcept { return _reference_attribute; } @@ -74,6 +85,3 @@ public: } // attribute } // search - - - diff --git a/searchlib/src/vespa/searchlib/attribute/readerbase.h b/searchlib/src/vespa/searchlib/attribute/readerbase.h index 88081906f71..358038f9ba2 100644 --- a/searchlib/src/vespa/searchlib/attribute/readerbase.h +++ b/searchlib/src/vespa/searchlib/attribute/readerbase.h @@ -3,6 +3,7 @@ #pragma once #include <vespa/searchlib/util/fileutil.h> +#include <cassert> namespace search { diff --git a/searchlib/src/vespa/searchlib/attribute/reference_mappings.cpp b/searchlib/src/vespa/searchlib/attribute/reference_mappings.cpp index 03acf6f2167..4edd9d45e60 100644 --- a/searchlib/src/vespa/searchlib/attribute/reference_mappings.cpp +++ b/searchlib/src/vespa/searchlib/attribute/reference_mappings.cpp @@ -9,6 +9,7 @@ namespace search::attribute { ReferenceMappings::ReferenceMappings(GenerationHolder &genHolder) : _reverseMappingIndices(genHolder), + _referencedLidLimit(0), _reverseMapping(), _referencedLids(genHolder) { @@ -38,7 +39,6 @@ ReferenceMappings::syncForwardMapping(const Reference &entry) { referencedLids[lid] = referencedLid; }); } - void ReferenceMappings::syncReverseMappingIndices(const Reference &entry) { @@ -46,6 +46,10 @@ ReferenceMappings::syncReverseMappingIndices(const Reference &entry) if (referencedLid != 0u) { _reverseMappingIndices.ensure_size(referencedLid + 1); _reverseMappingIndices[referencedLid] = entry.revMapIdx(); + if (referencedLid >= _referencedLidLimit) { + std::atomic_thread_fence(std::memory_order_release); + _referencedLidLimit = referencedLid + 1; + } } } diff --git a/searchlib/src/vespa/searchlib/attribute/reference_mappings.h b/searchlib/src/vespa/searchlib/attribute/reference_mappings.h index 97fdb77e78d..3190e1b5a83 100644 --- a/searchlib/src/vespa/searchlib/attribute/reference_mappings.h +++ b/searchlib/src/vespa/searchlib/attribute/reference_mappings.h @@ -4,6 +4,7 @@ #include <vespa/searchlib/btree/btreestore.h> #include <vespa/searchlib/common/rcuvector.h> +#include <atomic> namespace search::attribute { @@ -27,6 +28,8 @@ class ReferenceMappings // Vector containing references to trees of lids referencing given // referenced lid. ReverseMappingIndices _reverseMappingIndices; + // limit for referenced lid when accessing _reverseMappingIndices + uint32_t _referencedLidLimit; // Store of B-Trees, used to map from gid or referenced lid to // referencing lids. ReverseMapping _reverseMapping; @@ -77,7 +80,11 @@ public: ReferencedLids getReferencedLids() const { return ReferencedLids(&_referencedLids[0], _referencedLids.size()); } uint32_t getReferencedLid(uint32_t doc) const { return _referencedLids[doc]; } - ReverseMappingRefs getReverseMappingRefs() const { return ReverseMappingRefs(&_reverseMappingIndices[0], _reverseMappingIndices.size()); } + ReverseMappingRefs getReverseMappingRefs() const { + uint32_t referencedLidLimit = _referencedLidLimit; + std::atomic_thread_fence(std::memory_order_acquire); + return ReverseMappingRefs(&_reverseMappingIndices[0], referencedLidLimit); + } const ReverseMapping &getReverseMapping() const { return _reverseMapping; } }; diff --git a/searchlib/src/vespa/searchlib/bitcompression/compression.cpp b/searchlib/src/vespa/searchlib/bitcompression/compression.cpp index e50ebec105b..aa39b798b7a 100644 --- a/searchlib/src/vespa/searchlib/bitcompression/compression.cpp +++ b/searchlib/src/vespa/searchlib/bitcompression/compression.cpp @@ -5,10 +5,9 @@ #include <vespa/searchlib/fef/termfieldmatchdata.h> #include <vespa/searchlib/fef/termfieldmatchdataarray.h> #include <vespa/vespalib/data/fileheader.h> +#include <vespa/vespalib/data/databuffer.h> -namespace search { - -namespace bitcompression { +namespace search::bitcompression { using vespalib::nbostream; @@ -157,17 +156,12 @@ DecodeContext64Base::checkPointRead(nbostream &in) (void) in; } -} // namespace bitcompression - - namespace { vespalib::string noFeatures = "NoFeatures"; } -namespace bitcompression { - template <bool bigEndian> void FeatureDecodeContext<bigEndian>:: @@ -486,6 +480,4 @@ template class FeatureEncodeContext<true>; template class FeatureEncodeContext<false>; -} // namespace bitcompression - -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/common/packets.cpp b/searchlib/src/vespa/searchlib/common/packets.cpp index 667334bf64e..a0b64e2ef94 100644 --- a/searchlib/src/vespa/searchlib/common/packets.cpp +++ b/searchlib/src/vespa/searchlib/common/packets.cpp @@ -4,7 +4,7 @@ #include "packets.h" #include "sortdata.h" #include <vespa/searchlib/util/rawbuf.h> -#include <vespa/document/util/compressor.h> +#include <vespa/vespalib/util/compressor.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/data/databuffer.h> @@ -12,7 +12,6 @@ #include <vespa/log/log.h> LOG_SETUP(".searchlib.common.fs4packets"); -using document::CompressionConfig; using vespalib::ConstBufferRef; using vespalib::make_string; using vespalib::stringref; @@ -151,7 +150,7 @@ FS4PersistentPacketStreamer::Decode(FNET_DataBuffer *src, uint32_t plen, uint32_ uint32_t uncompressed_size = src->ReadInt32(); ConstBufferRef org(src->GetData(), plen - sizeof(uint32_t)); vespalib::DataBuffer uncompressed(uncompressed_size); - document::compression::decompress(compressionType, uncompressed_size, org, uncompressed, false); + vespalib::compression::decompress(compressionType, uncompressed_size, org, uncompressed, false); FNET_DataBuffer buf(uncompressed.getData(), uncompressed.getDataLen()); decodePacket(packet, buf, uncompressed_size, pcode); src->DataToDead(plen - sizeof(uint32_t)); @@ -192,7 +191,7 @@ FS4PersistentPacketStreamer::Encode(FNET_Packet *packet, uint32_t chid, FNET_Dat CompressionConfig config(_compressionType, _compressionLevel, 90); ConstBufferRef org(dst->GetData() + packet_start + header_len, body_len); vespalib::DataBuffer compressed(org.size()); - CompressionConfig::Type r = document::compression::compress(config, org, compressed, false); + CompressionConfig::Type r = vespalib::compression::compress(config, org, compressed, false); if (r != CompressionConfig::NONE) { dst->DataToFree(body_len + header_len); // sizeof(data + header + uncompressed_size) - sizeof(uint32_t) @@ -455,7 +454,7 @@ FS4Packet_PreSerialized::FS4Packet_PreSerialized(FNET_Packet & packet) 90); ConstBufferRef org(tmp.GetData(), tmp.GetDataLen()); vespalib::DataBuffer compressed(org.size()); - _compressionType = document::compression::compress(config, org, compressed, false); + _compressionType = vespalib::compression::compress(config, org, compressed, false); if (_compressionType != CompressionConfig::NONE) { _data.WriteInt32Fast(body_len); _data.WriteBytes(compressed.getData(), compressed.getDataLen()); @@ -1285,18 +1284,18 @@ FS4Packet_QUERYX::Encode(FNET_DataBuffer *dst) void FS4Packet::throwPropertieDecodeError(size_t i) { - throw vespalib::IllegalArgumentException(vespalib::make_string("Failed decoding properties[%ld]", i)); + throw vespalib::IllegalArgumentException(make_string("Failed decoding properties[%ld]", i)); } void FS4Packet::throwUnsupportedFeatures(uint32_t features, uint32_t set) { - throw vespalib::UnderflowException(vespalib::make_string("Unsupported features(%x), supported set(%x)", features, set)); + throw vespalib::UnderflowException(make_string("Unsupported features(%x), supported set(%x)", features, set)); } void FS4Packet::throwNotEnoughData(FNET_DataBuffer & buf, uint32_t left, uint32_t needed, const char * text) { (void) buf; - throw vespalib::UnderflowException(vespalib::make_string("Failed decoding packet of type %d. Only %d bytes left, needed %d from '%s'", GetPCODE(), left, needed, text)); + throw vespalib::UnderflowException(make_string("Failed decoding packet of type %d. Only %d bytes left, needed %d from '%s'", GetPCODE(), left, needed, text)); } #define VERIFY_LEN(needed, text) \ @@ -1436,7 +1435,7 @@ FS4Packet_QUERYX::toString(uint32_t indent) const } s += make_string("%*s sortspec : %s\n", indent, "", _sortSpec.c_str()); s += make_string("%*s groupspec : (%d bytes)\n", indent, "", (int)_groupSpec.size()); - s += make_string("%*s sessionId : (%d bytes)\n", indent, "", (int)_sessionId.size()); + s += make_string("%*s sessionId : (%d bytes) %s\n", indent, "", (int)_sessionId.size(), _sessionId.c_str()); s += make_string("%*s location : %s\n", indent, "", _location.c_str()); s += make_string("%*s timeout : %d\n", indent, "", _timeout); s += make_string("%*s stackitems : %d\n", indent, "", _numStackItems); diff --git a/searchlib/src/vespa/searchlib/common/packets.h b/searchlib/src/vespa/searchlib/common/packets.h index d9bc4d50462..52130c57374 100644 --- a/searchlib/src/vespa/searchlib/common/packets.h +++ b/searchlib/src/vespa/searchlib/common/packets.h @@ -9,7 +9,7 @@ #include <vespa/fnet/packet.h> #include <vespa/fnet/databuffer.h> #include <vespa/document/base/globalid.h> -#include <vespa/document/util/compressionconfig.h> +#include <vespa/vespalib/util/compressionconfig.h> #include <vespa/vespalib/util/memory.h> #include <vespa/fastos/timestamp.h> #include <vector> @@ -110,10 +110,11 @@ public: class FS4PersistentPacketStreamer : public FNET_IPacketStreamer { FS4PersistentPacketStreamer(const FS4PersistentPacketStreamer &); FS4PersistentPacketStreamer& operator=(const FS4PersistentPacketStreamer &); + using CompressionConfig = vespalib::compression::CompressionConfig; unsigned int _compressionLimit; unsigned int _compressionLevel; - document::CompressionConfig::Type _compressionType; + CompressionConfig::Type _compressionType; protected: bool _conservative; // Set to true if out of sync should mark the // stream as broken. @@ -139,8 +140,8 @@ public: void SetConservativeMode(bool cons) { _conservative = cons; } void SetCompressionLimit(unsigned int limit) { _compressionLimit = limit; } void SetCompressionLevel(unsigned int level) { _compressionLevel = level; } - void SetCompressionType(document::CompressionConfig::Type compressionType) { _compressionType = compressionType; } - document::CompressionConfig::Type getCompressionType() const { return _compressionType; } + void SetCompressionType(CompressionConfig::Type compressionType) { _compressionType = compressionType; } + CompressionConfig::Type getCompressionType() const { return _compressionType; } uint32_t getCompressionLimit() const { return _compressionLimit; } uint32_t getCompressionLevel() const { return _compressionLevel; } }; @@ -243,9 +244,10 @@ public: bool Decode(FNET_DataBuffer *src, uint32_t len) override; vespalib::string toString(uint32_t indent) const override; private: - uint32_t _pcode; - document::CompressionConfig::Type _compressionType; - FNET_DataBuffer _data; + using CompressionConfig = vespalib::compression::CompressionConfig; + uint32_t _pcode; + CompressionConfig::Type _compressionType; + FNET_DataBuffer _data; }; class FS4Packet_Shared : public FS4Packet diff --git a/searchlib/src/vespa/searchlib/diskindex/checkpointfile.cpp b/searchlib/src/vespa/searchlib/diskindex/checkpointfile.cpp index a55ec9494dc..0324f00f63c 100644 --- a/searchlib/src/vespa/searchlib/diskindex/checkpointfile.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/checkpointfile.cpp @@ -1,18 +1,16 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - #include "checkpointfile.h" #include <vespa/vespalib/data/fileheader.h> #include <vespa/searchlib/common/fileheadercontext.h> +#include <cassert> #include <vespa/log/log.h> LOG_SETUP(".diskindex.checkpointfile"); using vespalib::getLastErrorString; -namespace search { - -namespace diskindex { +namespace search::diskindex { using common::FileHeaderContext; @@ -182,6 +180,4 @@ CheckPointFile::readHeader() } -} // namespace diskindex - -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/docstore/chunk.cpp b/searchlib/src/vespa/searchlib/docstore/chunk.cpp index 179fda8689b..ba467501fba 100644 --- a/searchlib/src/vespa/searchlib/docstore/chunk.cpp +++ b/searchlib/src/vespa/searchlib/docstore/chunk.cpp @@ -43,12 +43,12 @@ Chunk::hasRoom(size_t len) const } size_t -Chunk::getMaxPackSize(const document::CompressionConfig & compression) const { +Chunk::getMaxPackSize(const CompressionConfig & compression) const { return _format->getMaxPackSize(compression); } void -Chunk::pack(uint64_t lastSerial, vespalib::DataBuffer & compressed, const document::CompressionConfig & compression) +Chunk::pack(uint64_t lastSerial, vespalib::DataBuffer & compressed, const CompressionConfig & compression) { _lastSerial = lastSerial; _format->pack(_lastSerial, compressed, compression); diff --git a/searchlib/src/vespa/searchlib/docstore/chunk.h b/searchlib/src/vespa/searchlib/docstore/chunk.h index bdf0d9793b2..cd551de1a9e 100644 --- a/searchlib/src/vespa/searchlib/docstore/chunk.h +++ b/searchlib/src/vespa/searchlib/docstore/chunk.h @@ -4,7 +4,7 @@ #include <vespa/searchlib/util/memoryusage.h> #include <vespa/vespalib/util/buffer.h> -#include <vespa/document/util/compressionconfig.h> +#include <vespa/vespalib/util/compressionconfig.h> #include <memory> #include <vector> @@ -60,7 +60,8 @@ private: class Chunk { public: - typedef std::unique_ptr<Chunk> UP; + using UP = std::unique_ptr<Chunk>; + using CompressionConfig = vespalib::compression::CompressionConfig; class Config { public: Config(size_t maxBytes) : _maxBytes(maxBytes) { } @@ -93,8 +94,8 @@ public: size_t size() const; const LidList & getLids() const { return _lids; } LidList getUniqueLids() const; - size_t getMaxPackSize(const document::CompressionConfig & compression) const; - void pack(uint64_t lastSerial, vespalib::DataBuffer & buffer, const document::CompressionConfig & compression); + size_t getMaxPackSize(const CompressionConfig & compression) const; + void pack(uint64_t lastSerial, vespalib::DataBuffer & buffer, const CompressionConfig & compression); uint64_t getLastSerial() const { return _lastSerial; } uint32_t getId() const { return _id; } bool validSerial() const { return getLastSerial() != static_cast<uint64_t>(-1l); } diff --git a/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp b/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp index 204c7b07acb..37381cfa3f6 100644 --- a/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp +++ b/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp @@ -1,17 +1,17 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "chunkformats.h" -#include <vespa/document/util/compressor.h> +#include <vespa/vespalib/util/compressor.h> #include <vespa/vespalib/util/stringfmt.h> namespace search { using vespalib::make_string; using vespalib::Exception; -using document::compression::compress; -using document::compression::decompress; -using document::compression::computeMaxCompressedsize; -using document::CompressionConfig; +using vespalib::compression::compress; +using vespalib::compression::decompress; +using vespalib::compression::computeMaxCompressedsize; +using vespalib::compression::CompressionConfig; ChunkException::ChunkException(const vespalib::stringref & msg, const vespalib::stringref & location) : Exception(make_string("Illegal chunk: %s", msg.c_str()), location) diff --git a/searchlib/src/vespa/searchlib/docstore/chunkformat.h b/searchlib/src/vespa/searchlib/docstore/chunkformat.h index 5f1e3d852a9..9f9580f1f1d 100644 --- a/searchlib/src/vespa/searchlib/docstore/chunkformat.h +++ b/searchlib/src/vespa/searchlib/docstore/chunkformat.h @@ -2,7 +2,7 @@ #pragma once -#include <vespa/document/util/compressionconfig.h> +#include <vespa/vespalib/util/compressionconfig.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/data/databuffer.h> #include <vespa/vespalib/util/exception.h> @@ -20,7 +20,8 @@ class ChunkFormat { public: virtual ~ChunkFormat(); - typedef std::unique_ptr<ChunkFormat> UP; + using UP = std::unique_ptr<ChunkFormat>; + using CompressionConfig = vespalib::compression::CompressionConfig; vespalib::nbostream & getBuffer() { return _dataBuf; } const vespalib::nbostream & getBuffer() const { return _dataBuf; } @@ -30,7 +31,7 @@ public: * @param compressed The buffer where the serialized data shall be placed. * @param compression What kind of compression shall be employed. */ - void pack(uint64_t lastSerial, vespalib::DataBuffer & compressed, const document::CompressionConfig & compression); + void pack(uint64_t lastSerial, vespalib::DataBuffer & compressed, const CompressionConfig & compression); /** * Will deserialize and create a representation of the uncompressed data. * param buffer Pointer to the serialized data @@ -44,7 +45,7 @@ public: * @param compression Compression config to be used. * @return maximum number of bytes a packet can take in serialized form. */ - size_t getMaxPackSize(const document::CompressionConfig & compression) const; + size_t getMaxPackSize(const CompressionConfig & compression) const; protected: /** * Constructor used when deserializing diff --git a/searchlib/src/vespa/searchlib/docstore/compacter.h b/searchlib/src/vespa/searchlib/docstore/compacter.h index 264cce8d3aa..362896487aa 100644 --- a/searchlib/src/vespa/searchlib/docstore/compacter.h +++ b/searchlib/src/vespa/searchlib/docstore/compacter.h @@ -33,7 +33,7 @@ private: **/ class BucketCompacter : public IWriteData, public StoreByBucket::IWrite { - using CompressionConfig = document::CompressionConfig; + using CompressionConfig = vespalib::compression::CompressionConfig; using ThreadExecutor = vespalib::ThreadExecutor; public: using FileId = FileChunk::FileId; diff --git a/searchlib/src/vespa/searchlib/docstore/documentstore.cpp b/searchlib/src/vespa/searchlib/docstore/documentstore.cpp index a13fffdaa24..bf59614a297 100644 --- a/searchlib/src/vespa/searchlib/docstore/documentstore.cpp +++ b/searchlib/src/vespa/searchlib/docstore/documentstore.cpp @@ -6,12 +6,12 @@ #include "ibucketizer.h" #include <vespa/vespalib/stllike/cache.hpp> #include <vespa/vespalib/data/databuffer.h> -#include <vespa/document/util/compressor.h> +#include <vespa/vespalib/util/compressor.h> using document::DocumentTypeRepo; -using document::CompressionConfig; -using document::compression::compress; -using document::compression::decompress; +using vespalib::compression::CompressionConfig; +using vespalib::compression::compress; +using vespalib::compression::decompress; namespace search { diff --git a/searchlib/src/vespa/searchlib/docstore/documentstore.h b/searchlib/src/vespa/searchlib/docstore/documentstore.h index 45dfcb35e37..4ba5c27cd07 100644 --- a/searchlib/src/vespa/searchlib/docstore/documentstore.h +++ b/searchlib/src/vespa/searchlib/docstore/documentstore.h @@ -26,25 +26,26 @@ class DocumentStore : public IDocumentStore public: class Config { public: + using CompressionConfig = vespalib::compression::CompressionConfig; Config() : - _compression(document::CompressionConfig::LZ4, 9, 70), + _compression(CompressionConfig::LZ4, 9, 70), _maxCacheBytes(1000000000), _initialCacheEntries(0), _allowVisitCaching(false) { } - Config(const document::CompressionConfig & compression, size_t maxCacheBytes, size_t initialCacheEntries) : - _compression((maxCacheBytes != 0) ? compression : document::CompressionConfig::NONE), + Config(const CompressionConfig & compression, size_t maxCacheBytes, size_t initialCacheEntries) : + _compression((maxCacheBytes != 0) ? compression : CompressionConfig::NONE), _maxCacheBytes(maxCacheBytes), _initialCacheEntries(initialCacheEntries), _allowVisitCaching(false) { } - const document::CompressionConfig & getCompression() const { return _compression; } + const CompressionConfig & getCompression() const { return _compression; } size_t getMaxCacheBytes() const { return _maxCacheBytes; } size_t getInitialCacheEntries() const { return _initialCacheEntries; } bool allowVisitCaching() const { return _allowVisitCaching; } Config & allowVisitCaching(bool allow) { _allowVisitCaching = allow; return *this; } private: - document::CompressionConfig _compression; + CompressionConfig _compression; size_t _maxCacheBytes; size_t _initialCacheEntries; bool _allowVisitCaching; diff --git a/searchlib/src/vespa/searchlib/docstore/filechunk.cpp b/searchlib/src/vespa/searchlib/docstore/filechunk.cpp index f271e6f320b..2a748a302c6 100644 --- a/searchlib/src/vespa/searchlib/docstore/filechunk.cpp +++ b/searchlib/src/vespa/searchlib/docstore/filechunk.cpp @@ -5,6 +5,7 @@ #include "summaryexceptions.h" #include "randreaders.h" #include <vespa/vespalib/data/fileheader.h> +#include <vespa/vespalib/data/databuffer.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/util/array.hpp> #include <vespa/vespalib/stllike/hash_map.hpp> diff --git a/searchlib/src/vespa/searchlib/docstore/logdatastore.h b/searchlib/src/vespa/searchlib/docstore/logdatastore.h index 0ca658ba803..080e6f80503 100644 --- a/searchlib/src/vespa/searchlib/docstore/logdatastore.h +++ b/searchlib/src/vespa/searchlib/docstore/logdatastore.h @@ -5,7 +5,7 @@ #include "idatastore.h" #include "lid_info.h" #include "writeablefilechunk.h" -#include <vespa/document/util/compressionconfig.h> +#include <vespa/vespalib/util/compressionconfig.h> #include <vespa/searchlib/common/rcuvector.h> #include <vespa/searchlib/common/tunefileinfo.h> #include <vespa/searchlib/transactionlog/syncproxy.h> @@ -33,7 +33,7 @@ private: public: using NameIdSet = std::set<NameId>; using LockGuard = vespalib::LockGuard; - using CompressionConfig = document::CompressionConfig; + using CompressionConfig = vespalib::compression::CompressionConfig; class Config { public: Config() diff --git a/searchlib/src/vespa/searchlib/docstore/storebybucket.h b/searchlib/src/vespa/searchlib/docstore/storebybucket.h index f93826027f5..ac1f6fbe007 100644 --- a/searchlib/src/vespa/searchlib/docstore/storebybucket.h +++ b/searchlib/src/vespa/searchlib/docstore/storebybucket.h @@ -23,9 +23,9 @@ class StoreByBucket using MemoryDataStore = vespalib::MemoryDataStore; using ThreadExecutor = vespalib::ThreadExecutor; using ConstBufferRef = vespalib::ConstBufferRef; - using CompressionConfig = document::CompressionConfig; + using CompressionConfig = vespalib::compression::CompressionConfig; public: - StoreByBucket(vespalib::MemoryDataStore & backingMemory, const document::CompressionConfig & compression); + StoreByBucket(vespalib::MemoryDataStore & backingMemory, const CompressionConfig & compression); StoreByBucket(MemoryDataStore & backingMemory, ThreadExecutor & executor, const CompressionConfig & compression); StoreByBucket(StoreByBucket &&) = default; ~StoreByBucket(); diff --git a/searchlib/src/vespa/searchlib/docstore/visitcache.cpp b/searchlib/src/vespa/searchlib/docstore/visitcache.cpp index 22a16947b67..b3fd236d73d 100644 --- a/searchlib/src/vespa/searchlib/docstore/visitcache.cpp +++ b/searchlib/src/vespa/searchlib/docstore/visitcache.cpp @@ -5,7 +5,7 @@ #include <vespa/vespalib/stllike/cache.hpp> #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/vespalib/data/databuffer.h> -#include <vespa/document/util/compressor.h> +#include <vespa/vespalib/util/compressor.h> #include <algorithm> namespace search::docstore { @@ -74,7 +74,7 @@ BlobSet::get(uint32_t lid) const } CompressedBlobSet::CompressedBlobSet() : - _compression(document::CompressionConfig::Type::LZ4), + _compression(CompressionConfig::Type::LZ4), _positions(), _buffer() { @@ -83,7 +83,7 @@ CompressedBlobSet::CompressedBlobSet() : CompressedBlobSet::~CompressedBlobSet() { } -CompressedBlobSet::CompressedBlobSet(const document::CompressionConfig &compression, const BlobSet & uncompressed) : +CompressedBlobSet::CompressedBlobSet(const CompressionConfig &compression, const BlobSet & uncompressed) : _compression(compression.type), _positions(uncompressed.getPositions()), _buffer() @@ -91,7 +91,7 @@ CompressedBlobSet::CompressedBlobSet(const document::CompressionConfig &compress if ( ! _positions.empty() ) { DataBuffer compressed; ConstBufferRef org = uncompressed.getBuffer(); - _compression = document::compression::compress(compression, org, compressed, false); + _compression = vespalib::compression::compress(compression, org, compressed, false); _buffer.resize(compressed.getDataLen()); memcpy(_buffer, compressed.getData(), compressed.getDataLen()); } @@ -100,7 +100,7 @@ CompressedBlobSet::CompressedBlobSet(const document::CompressionConfig &compress BlobSet CompressedBlobSet::getBlobSet() const { - using document::compression::decompress; + using vespalib::compression::decompress; // These are frequent lage allocations that are to expensive to mmap. DataBuffer uncompressed(0, 1, Alloc::alloc(0, 16 * MemoryAllocator::HUGEPAGE_SIZE)); if ( ! _positions.empty() ) { @@ -145,7 +145,7 @@ VisitCache::BackingStore::read(const KeySet &key, CompressedBlobSet &blobs) cons return ! blobs.empty(); } -VisitCache::VisitCache(IDataStore &store, size_t cacheSize, const document::CompressionConfig &compression) : +VisitCache::VisitCache(IDataStore &store, size_t cacheSize, const CompressionConfig &compression) : _store(store, compression), _cache(std::make_unique<Cache>(_store, cacheSize)) { diff --git a/searchlib/src/vespa/searchlib/docstore/visitcache.h b/searchlib/src/vespa/searchlib/docstore/visitcache.h index 44ee5542a72..a89620b7bde 100644 --- a/searchlib/src/vespa/searchlib/docstore/visitcache.h +++ b/searchlib/src/vespa/searchlib/docstore/visitcache.h @@ -72,8 +72,9 @@ private: **/ class CompressedBlobSet { public: + using CompressionConfig = vespalib::compression::CompressionConfig; CompressedBlobSet(); - CompressedBlobSet(const document::CompressionConfig &compression, const BlobSet & uncompressed); + CompressedBlobSet(const CompressionConfig &compression, const BlobSet & uncompressed); CompressedBlobSet(CompressedBlobSet && rhs) = default; CompressedBlobSet & operator=(CompressedBlobSet && rhs) = default; CompressedBlobSet(const CompressedBlobSet & rhs) = default; @@ -83,7 +84,7 @@ public: bool empty() const { return _positions.empty(); } BlobSet getBlobSet() const; private: - document::CompressionConfig::Type _compression; + CompressionConfig::Type _compression; BlobSet::Positions _positions; vespalib::MallocPtr _buffer; }; @@ -95,7 +96,8 @@ private: **/ class VisitCache { public: - VisitCache(IDataStore &store, size_t cacheSize, const document::CompressionConfig &compression); + using CompressionConfig = vespalib::compression::CompressionConfig; + VisitCache(IDataStore &store, size_t cacheSize, const CompressionConfig &compression); CompressedBlobSet read(const IDocumentStore::LidVector & keys) const; void remove(uint32_t key); @@ -111,17 +113,17 @@ private: */ class BackingStore { public: - BackingStore(IDataStore &store, const document::CompressionConfig &compression) : + BackingStore(IDataStore &store, const CompressionConfig &compression) : _backingStore(store), _compression(compression) { } bool read(const KeySet &key, CompressedBlobSet &blobs) const; void write(const KeySet &, const CompressedBlobSet &) { } void erase(const KeySet &) { } - const document::CompressionConfig &getCompression() const { return _compression; } + const CompressionConfig &getCompression() const { return _compression; } private: IDataStore &_backingStore; - const document::CompressionConfig _compression; + const CompressionConfig _compression; }; using CacheParams = vespalib::CacheParam< diff --git a/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp b/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp index c6b93a59ae8..5cf5c646148 100644 --- a/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp +++ b/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp @@ -6,6 +6,7 @@ #include <vespa/vespalib/util/closuretask.h> #include <vespa/vespalib/util/array.hpp> #include <vespa/vespalib/data/fileheader.h> +#include <vespa/vespalib/data/databuffer.h> #include <vespa/searchlib/common/fileheadercontext.h> #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/vespalib/objects/nbostream.h> diff --git a/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.h b/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.h index 86415e0cb40..2b21f12a314 100644 --- a/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.h +++ b/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.h @@ -22,20 +22,21 @@ public: class Config { public: + using CompressionConfig = vespalib::compression::CompressionConfig; Config() - : _compression(document::CompressionConfig::LZ4, 9, 60), + : _compression(CompressionConfig::LZ4, 9, 60), _maxChunkBytes(0x10000) { } - Config(const document::CompressionConfig &compression, size_t maxChunkBytes) + Config(const CompressionConfig &compression, size_t maxChunkBytes) : _compression(compression), _maxChunkBytes(maxChunkBytes) { } - const document::CompressionConfig & getCompression() const { return _compression; } + const CompressionConfig & getCompression() const { return _compression; } size_t getMaxChunkBytes() const { return _maxChunkBytes; } private: - document::CompressionConfig _compression; + CompressionConfig _compression; size_t _maxChunkBytes; }; diff --git a/searchlib/src/vespa/searchlib/grouping/hyperloglog.h b/searchlib/src/vespa/searchlib/grouping/hyperloglog.h index 753b81d92f2..931b832c76d 100644 --- a/searchlib/src/vespa/searchlib/grouping/hyperloglog.h +++ b/searchlib/src/vespa/searchlib/grouping/hyperloglog.h @@ -3,8 +3,8 @@ #pragma once #include "sketch.h" -#include <vespa/document/util/compressionconfig.h> -#include <vespa/document/util/compressor.h> +#include <vespa/vespalib/util/compressionconfig.h> +#include <vespa/vespalib/util/compressor.h> #include <vespa/vespalib/data/databuffer.h> #include <vespa/vespalib/objects/deserializer.h> #include <vespa/vespalib/objects/serializer.h> diff --git a/searchlib/src/vespa/searchlib/grouping/sketch.h b/searchlib/src/vespa/searchlib/grouping/sketch.h index bb7db02d81c..317c1bfef9d 100644 --- a/searchlib/src/vespa/searchlib/grouping/sketch.h +++ b/searchlib/src/vespa/searchlib/grouping/sketch.h @@ -2,8 +2,8 @@ #pragma once -#include <vespa/document/util/compressionconfig.h> -#include <vespa/document/util/compressor.h> +#include <vespa/vespalib/util/compressionconfig.h> +#include <vespa/vespalib/util/compressor.h> #include <lz4.h> #include <vespa/searchlib/common/identifiable.h> #include <vespa/vespalib/data/databuffer.h> @@ -205,13 +205,13 @@ deserialize(vespalib::Deserializer &is) { template <int BucketBits, typename HashT> uint32_t NormalSketch<BucketBits, HashT>:: compress_buckets_into(char *buffer, uint32_t size) const { - document::CompressionConfig config(document::CompressionConfig::LZ4, 9, 9); + vespalib::compression::CompressionConfig config(vespalib::compression::CompressionConfig::LZ4, 9, 9); vespalib::ConstBufferRef org(&bucket[0], BUCKET_COUNT); vespalib::DataBuffer compress_buffer(buffer, size); - document::CompressionConfig::Type r = - document::compression::compress(config, org, compress_buffer, false); + vespalib::compression::CompressionConfig::Type r = + vespalib::compression::compress(config, org, compress_buffer, false); assert(compress_buffer.getDead() == buffer); - if (r == document::CompressionConfig::LZ4) { + if (r == vespalib::compression::CompressionConfig::LZ4) { assert(compress_buffer.getDataLen() < BUCKET_COUNT); return compress_buffer.getDataLen(); } else { @@ -228,7 +228,8 @@ decompress_buckets_from(char *buffer, uint32_t size) { } else { vespalib::ConstBufferRef compressed(buffer, size); vespalib::DataBuffer uncompressed(reinterpret_cast<char *>(&bucket[0]), BUCKET_COUNT); - document::compression::decompress(document::CompressionConfig::LZ4, BUCKET_COUNT, compressed, uncompressed, false); + vespalib::compression::decompress(vespalib::compression::CompressionConfig::LZ4, BUCKET_COUNT, + compressed, uncompressed, false); } } template <int BucketBits, typename HashT> diff --git a/searchlib/src/vespa/searchlib/index/dummyfileheadercontext.cpp b/searchlib/src/vespa/searchlib/index/dummyfileheadercontext.cpp index 6c4155b035e..e5e582b673b 100644 --- a/searchlib/src/vespa/searchlib/index/dummyfileheadercontext.cpp +++ b/searchlib/src/vespa/searchlib/index/dummyfileheadercontext.cpp @@ -4,6 +4,7 @@ #include <vespa/vespalib/data/fileheader.h> #include <vespa/searchlib/util/fileheadertk.h> #include <vespa/vespalib/util/host_name.h> +#include <cassert> #include <unistd.h> namespace search::index { diff --git a/searchlib/src/vespa/searchlib/util/fileutil.cpp b/searchlib/src/vespa/searchlib/util/fileutil.cpp index c1d34f66b0b..674608bdda1 100644 --- a/searchlib/src/vespa/searchlib/util/fileutil.cpp +++ b/searchlib/src/vespa/searchlib/util/fileutil.cpp @@ -4,6 +4,7 @@ #include "filesizecalculator.h" #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/guard.h> +#include <cassert> #include <fcntl.h> #include <sys/mman.h> #include <sys/stat.h> @@ -18,7 +19,6 @@ using vespalib::FileDescriptor; using vespalib::getLastErrorString; namespace search { - namespace fileutil { LoadedMmap::LoadedMmap(const vespalib::string &fileName) @@ -177,4 +177,5 @@ FileWriterBase::write(const void *buf, size_t sz) { } return numWritten; } + } diff --git a/slobrok/src/apps/slobrok/CMakeLists.txt b/slobrok/src/apps/slobrok/CMakeLists.txt index 09d6a3ce9eb..5e1bcbf56ed 100644 --- a/slobrok/src/apps/slobrok/CMakeLists.txt +++ b/slobrok/src/apps/slobrok/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(slobrok_app SOURCES slobrok.cpp OUTPUT_NAME vespa-slobrok - INSTALL bin + INSTALL sbin DEPENDS slobrok_slobrokserver ) diff --git a/staging_vespalib/CMakeLists.txt b/staging_vespalib/CMakeLists.txt index 5a169416775..d59529df828 100644 --- a/staging_vespalib/CMakeLists.txt +++ b/staging_vespalib/CMakeLists.txt @@ -13,7 +13,6 @@ vespa_define_module( src/tests/bits src/tests/clock src/tests/crc - src/tests/databuffer src/tests/directio src/tests/encoding/base64 src/tests/fileheader diff --git a/staging_vespalib/src/tests/databuffer/.gitignore b/staging_vespalib/src/tests/databuffer/.gitignore deleted file mode 100644 index e7b0e69c372..00000000000 --- a/staging_vespalib/src/tests/databuffer/.gitignore +++ /dev/null @@ -1 +0,0 @@ -staging_vespalib_databuffer_test_app diff --git a/staging_vespalib/src/tests/databuffer/CMakeLists.txt b/staging_vespalib/src/tests/databuffer/CMakeLists.txt deleted file mode 100644 index 0c96a8cdf3e..00000000000 --- a/staging_vespalib/src/tests/databuffer/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(staging_vespalib_databuffer_test_app TEST - SOURCES - databuffer_test.cpp - DEPENDS - staging_vespalib -) -vespa_add_test(NAME staging_vespalib_databuffer_test_app COMMAND staging_vespalib_databuffer_test_app) diff --git a/staging_vespalib/src/tests/fileheader/fileheader_test.cpp b/staging_vespalib/src/tests/fileheader/fileheader_test.cpp index b895b92db4c..f6f9c19b48c 100644 --- a/staging_vespalib/src/tests/fileheader/fileheader_test.cpp +++ b/staging_vespalib/src/tests/fileheader/fileheader_test.cpp @@ -2,6 +2,7 @@ #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/data/fileheader.h> +#include <vespa/vespalib/data/databuffer.h> #include <vespa/fastos/file.h> #include <iostream> diff --git a/staging_vespalib/src/vespa/vespalib/data/CMakeLists.txt b/staging_vespalib/src/vespa/vespalib/data/CMakeLists.txt index 3fbece6211d..8ca70d2a63d 100644 --- a/staging_vespalib/src/vespa/vespalib/data/CMakeLists.txt +++ b/staging_vespalib/src/vespa/vespalib/data/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_library(staging_vespalib_vespalib_data OBJECT SOURCES - databuffer.cpp fileheader.cpp DEPENDS ) diff --git a/staging_vespalib/src/vespa/vespalib/data/fileheader.cpp b/staging_vespalib/src/vespa/vespalib/data/fileheader.cpp index 3cd8190d894..a9e131b0c63 100644 --- a/staging_vespalib/src/vespa/vespalib/data/fileheader.cpp +++ b/staging_vespalib/src/vespa/vespalib/data/fileheader.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "fileheader.h" #include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/vespalib/data/databuffer.h> #include <vespa/fastos/file.h> #include <vespa/log/log.h> diff --git a/staging_vespalib/src/vespa/vespalib/data/fileheader.h b/staging_vespalib/src/vespa/vespalib/data/fileheader.h index 157703ef0a8..e4449a0c36a 100644 --- a/staging_vespalib/src/vespa/vespalib/data/fileheader.h +++ b/staging_vespalib/src/vespa/vespalib/data/fileheader.h @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include "databuffer.h" #include <vespa/vespalib/util/exception.h> #include <map> @@ -9,6 +8,7 @@ class FastOS_FileInterface; namespace vespalib { +class DataBuffer; class asciistream; /** diff --git a/storage/src/tests/distributor/statecheckerstest.cpp b/storage/src/tests/distributor/statecheckerstest.cpp index fbaa8a7662d..29c922248e7 100644 --- a/storage/src/tests/distributor/statecheckerstest.cpp +++ b/storage/src/tests/distributor/statecheckerstest.cpp @@ -83,6 +83,7 @@ struct StateCheckersTest : public CppUnit::TestFixture, void inhibitBucketDeactivationIfDisabledInConfig(); void retiredNodesOutOfSyncAreMerged(); void testGarbageCollection(); + void gc_ops_are_prioritized_with_low_priority_category(); void gcInhibitedWhenIdealNodeInMaintenance(); void testNoRemoveWhenIdealNodeInMaintenance(); void testStepwiseJoinForSmallBucketsWithoutSiblings(); @@ -186,7 +187,8 @@ struct StateCheckersTest : public CppUnit::TestFixture, uint32_t nowTimestamp, uint32_t checkInterval, uint32_t lastChangeTime = 0, - bool includePriority = false); + bool includePriority = false, + bool includeSchedulingPri = false); std::string testSplit(uint32_t splitCount, uint32_t splitSize, @@ -321,6 +323,7 @@ struct StateCheckersTest : public CppUnit::TestFixture, CPPUNIT_TEST(inhibitBucketActivationIfDisabledInConfig); CPPUNIT_TEST(inhibitBucketDeactivationIfDisabledInConfig); CPPUNIT_TEST(testGarbageCollection); + CPPUNIT_TEST(gc_ops_are_prioritized_with_low_priority_category); CPPUNIT_TEST(gcInhibitedWhenIdealNodeInMaintenance); CPPUNIT_TEST(testNoRemoveWhenIdealNodeInMaintenance); CPPUNIT_TEST(testStepwiseJoinForSmallBucketsWithoutSiblings); @@ -1458,7 +1461,7 @@ StateCheckersTest::inhibitBucketDeactivationIfDisabledInConfig() std::string StateCheckersTest::testGarbageCollection( uint32_t prevTimestamp, uint32_t nowTimestamp, uint32_t checkInterval, uint32_t lastChangeTime, - bool includePriority) + bool includePriority, bool includeSchedulingPri) { BucketDatabase::Entry e(document::BucketId(17, 0)); e.getBucketInfo().addNode(BucketCopy(prevTimestamp, 0, @@ -1475,7 +1478,7 @@ std::string StateCheckersTest::testGarbageCollection( e.getBucketId()); getClock().setAbsoluteTimeInSeconds(nowTimestamp); return testStateChecker(checker, c, false, PendingMessage(), - includePriority); + includePriority, includeSchedulingPri); } void @@ -1527,6 +1530,13 @@ StateCheckersTest::testGarbageCollection() testGarbageCollection(3850, 4000, 300, 1)); } +void StateCheckersTest::gc_ops_are_prioritized_with_low_priority_category() { + CPPUNIT_ASSERT_EQUAL( + std::string("[Needs garbage collection: Last check at 3, current time 4000, " + "configured interval 300] (scheduling pri LOW)"), + testGarbageCollection(3, 4000, 300, 1, false, true)); +} + /** * When a node is in maintenance, we want to do our best to avoid any unneeded * changes to the bucket replicas' states, as this will require re-syncing of diff --git a/storage/src/vespa/storage/config/stor-communicationmanager.def b/storage/src/vespa/storage/config/stor-communicationmanager.def index 23c713d41f9..2c00dbe2f37 100644 --- a/storage/src/vespa/storage/config/stor-communicationmanager.def +++ b/storage/src/vespa/storage/config/stor-communicationmanager.def @@ -17,3 +17,11 @@ mbus_content_node_max_pending_count int default=0 mbus_distributor_node_max_pending_size int default=0 mbus_content_node_max_pending_size int default=0 +# Minimum size of packets to compress (0 means no compression) +mbus.compress.limit int default=1024 + +## Compression level for packets +mbus.compress.level int default=3 + +## Compression type for packets. +mbus.compress.type enum {NONE, LZ4, ZSTD} default=LZ4 diff --git a/storage/src/vespa/storage/distributor/statecheckers.cpp b/storage/src/vespa/storage/distributor/statecheckers.cpp index f81b6d2bcb6..701a147a56f 100644 --- a/storage/src/vespa/storage/distributor/statecheckers.cpp +++ b/storage/src/vespa/storage/distributor/statecheckers.cpp @@ -1140,7 +1140,7 @@ GarbageCollectionStateChecker::check(Context& c) op->setPriority(c.distributorConfig.getMaintenancePriorities() .garbageCollection); op->setDetailedReason(reason.c_str()); - return Result::createStoredResult(std::move(op), MaintenancePriority::MEDIUM); + return Result::createStoredResult(std::move(op), MaintenancePriority::LOW); } else { return Result::noMaintenanceNeeded(); } diff --git a/storage/src/vespa/storage/storageserver/communicationmanager.cpp b/storage/src/vespa/storage/storageserver/communicationmanager.cpp index aac52550fa0..ae8f3290fef 100644 --- a/storage/src/vespa/storage/storageserver/communicationmanager.cpp +++ b/storage/src/vespa/storage/storageserver/communicationmanager.cpp @@ -8,6 +8,7 @@ #include <vespa/documentapi/messagebus/messages/wrongdistributionreply.h> #include <vespa/storageapi/message/state.h> #include <vespa/messagebus/rpcmessagebus.h> +#include <vespa/messagebus/network/rpcnetworkparams.h> #include <vespa/messagebus/emptyreply.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/util/stringfmt.h> @@ -362,8 +363,7 @@ void CommunicationManager::onClose() } void -CommunicationManager::configureMessageBusLimits( - const CommunicationManagerConfig& cfg) +CommunicationManager::configureMessageBusLimits(const CommunicationManagerConfig& cfg) { const bool isDist(_component.getNodeType() == lib::NodeType::DISTRIBUTOR); auto& mbus(_mbus->getMessageBus()); diff --git a/storage/src/vespa/storage/storageserver/documentapiconverter.cpp b/storage/src/vespa/storage/storageserver/documentapiconverter.cpp index 9df177a32dd..59f3a1f6a27 100644 --- a/storage/src/vespa/storage/storageserver/documentapiconverter.cpp +++ b/storage/src/vespa/storage/storageserver/documentapiconverter.cpp @@ -36,7 +36,7 @@ DocumentApiConverter::toStorageAPI(documentapi::DocumentMessage& fromMsg, case DocumentProtocol::MESSAGE_PUTDOCUMENT: { documentapi::PutDocumentMessage& from(static_cast<documentapi::PutDocumentMessage&>(fromMsg)); - api::PutCommand::UP to(new api::PutCommand(document::BucketId(0), from.getDocumentSP(), from.getTimestamp())); + auto to = std::make_unique<api::PutCommand>(document::BucketId(0), from.stealDocument(), from.getTimestamp()); to->setCondition(from.getCondition()); toMsg = std::move(to); break; @@ -44,8 +44,8 @@ DocumentApiConverter::toStorageAPI(documentapi::DocumentMessage& fromMsg, case DocumentProtocol::MESSAGE_UPDATEDOCUMENT: { documentapi::UpdateDocumentMessage& from(static_cast<documentapi::UpdateDocumentMessage&>(fromMsg)); - api::UpdateCommand::UP to(new api::UpdateCommand(document::BucketId(0), from.getDocumentUpdateSP(), - from.getNewTimestamp())); + auto to = std::make_unique<api::UpdateCommand>(document::BucketId(0), from.stealDocumentUpdate(), + from.getNewTimestamp()); to->setOldTimestamp(from.getOldTimestamp()); to->setCondition(from.getCondition()); toMsg = std::move(to); @@ -54,7 +54,7 @@ DocumentApiConverter::toStorageAPI(documentapi::DocumentMessage& fromMsg, case DocumentProtocol::MESSAGE_REMOVEDOCUMENT: { documentapi::RemoveDocumentMessage& from(static_cast<documentapi::RemoveDocumentMessage&>(fromMsg)); - api::RemoveCommand::UP to(new api::RemoveCommand(document::BucketId(0), from.getDocumentId(), 0)); + auto to = std::make_unique<api::RemoveCommand>(document::BucketId(0), from.getDocumentId(), 0); to->setCondition(from.getCondition()); toMsg = std::move(to); break; @@ -62,15 +62,15 @@ DocumentApiConverter::toStorageAPI(documentapi::DocumentMessage& fromMsg, case DocumentProtocol::MESSAGE_GETDOCUMENT: { documentapi::GetDocumentMessage& from(static_cast<documentapi::GetDocumentMessage&>(fromMsg)); - api::GetCommand::UP to(new api::GetCommand(document::BucketId(0), from.getDocumentId(), from.getFieldSet())); + auto to = std::make_unique<api::GetCommand>(document::BucketId(0), from.getDocumentId(), from.getFieldSet()); toMsg.reset(to.release()); break; } case DocumentProtocol::MESSAGE_CREATEVISITOR: { documentapi::CreateVisitorMessage& from(static_cast<documentapi::CreateVisitorMessage&>(fromMsg)); - api::CreateVisitorCommand::UP to(new api::CreateVisitorCommand(from.getLibraryName(), from.getInstanceId(), - from.getDocumentSelection())); + auto to = std::make_unique<api::CreateVisitorCommand>(from.getLibraryName(), from.getInstanceId(), + from.getDocumentSelection()); to->setControlDestination(from.getControlDestination()); to->setDataDestination(from.getDataDestination()); diff --git a/storageserver/src/tests/storageservertest.cpp b/storageserver/src/tests/storageservertest.cpp index f3595afe1e7..21b190d9e76 100644 --- a/storageserver/src/tests/storageservertest.cpp +++ b/storageserver/src/tests/storageservertest.cpp @@ -6,6 +6,7 @@ #include <vespa/document/base/testdocman.h> #include <vespa/documentapi/documentapi.h> #include <vespa/messagebus/rpcmessagebus.h> +#include <vespa/messagebus/network/rpcnetworkparams.h> #include <vespa/memfilepersistence/spi/memfilepersistenceprovider.h> #include <vespa/messagebus/staticthrottlepolicy.h> #include <vespa/messagebus/testlib/slobrok.h> @@ -20,8 +21,8 @@ #include <vespa/storageserver/app/distributorprocess.h> #include <vespa/storageserver/app/memfileservicelayerprocess.h> #include <vespa/vespalib/util/exceptions.h> +#include <vespa/fnet/frt/supervisor.h> #include <sys/time.h> -#include <fstream> #include <vespa/log/log.h> LOG_SETUP(".storageservertest"); diff --git a/travis/travis-build-cpp.sh b/travis/travis-build-cpp.sh index 42dbf0e6467..825da67bf54 100755 --- a/travis/travis-build-cpp.sh +++ b/travis/travis-build-cpp.sh @@ -7,11 +7,13 @@ BUILD_DIR=~/build mkdir "${BUILD_DIR}" -export CCACHE_SIZE="1G" +export CCACHE_MAXSIZE="1250M" export CCACHE_COMPRESS=1 NUM_THREADS=4 +ccache --print-config cd ${BUILD_DIR} bash ${SOURCE_DIR}/bootstrap-cpp.sh ${SOURCE_DIR} ${BUILD_DIR} make -j ${NUM_THREADS} ctest3 --output-on-failure -j ${NUM_THREADS} +ccache --show-stats diff --git a/vagrant/.gitignore b/vagrant/.gitignore new file mode 100644 index 00000000000..a977916f658 --- /dev/null +++ b/vagrant/.gitignore @@ -0,0 +1 @@ +.vagrant/ diff --git a/vagrant/README.md b/vagrant/README.md index 7c69f8f22b0..02d11900558 100644 --- a/vagrant/README.md +++ b/vagrant/README.md @@ -25,4 +25,4 @@ This is needed in order to compile and run tests fast on the local file system i ## Build C++ modules -Please follow the instructions described [here](../README.md). +Please follow the instructions described [here](../README.md#build-c-modules). diff --git a/vdslib/src/vespa/vdslib/container/mutabledocumentlist.cpp b/vdslib/src/vespa/vdslib/container/mutabledocumentlist.cpp index ce55c3a87f8..940d142db2f 100644 --- a/vdslib/src/vespa/vdslib/container/mutabledocumentlist.cpp +++ b/vdslib/src/vespa/vdslib/container/mutabledocumentlist.cpp @@ -5,17 +5,17 @@ #include <vespa/document/datatype/documenttype.h> #include <vespa/vespalib/objects/nbostream.h> +using vespalib::compression::CompressionConfig; using vespalib::nbostream; namespace vdslib { -MutableDocumentList::MutableDocumentList(const document::DocumentTypeRepo::SP & repo, char* buffer, uint32_t bufferSize, bool keepexisting) +MutableDocumentList::MutableDocumentList(const document::DocumentTypeRepo::SP & repo, char* buffer, + uint32_t bufferSize, bool keepexisting) : DocumentList(repo, buffer, bufferSize, keepexisting) { } -MutableDocumentList::MutableDocumentList(const DocumentList& source, - char* buffer, - uint32_t bufferSize) +MutableDocumentList::MutableDocumentList(const DocumentList& source, char* buffer, uint32_t bufferSize) : DocumentList(source, buffer, bufferSize) { } @@ -41,8 +41,7 @@ MutableDocumentList::addOperationList(const OperationList& opl) } bool -MutableDocumentList::addPut(const document::Document& doc, Timestamp ts, - bool addBody) +MutableDocumentList::addPut(const document::Document& doc, Timestamp ts, bool addBody) { uint32_t freePos = _freePtr - _buffer; @@ -69,9 +68,7 @@ MutableDocumentList::addPut(const document::Document& doc, Timestamp ts, entry.bodyLen = bodySize; entry.flags = 0; - if (doc.getType().getFieldsType().getCompressionConfig().type - != document::CompressionConfig::NONE) - { + if (doc.getType().getFieldsType().getCompressionConfig().type != CompressionConfig::NONE) { entry.flags |= MetaEntry::COMPRESSED; } @@ -91,8 +88,7 @@ MutableDocumentList::addPut(const document::Document& doc, Timestamp ts, } bool -MutableDocumentList::addUpdate(const document::DocumentUpdate& update, - Timestamp ts) +MutableDocumentList::addUpdate(const document::DocumentUpdate& update, Timestamp ts) { vespalib::nbostream os; update.serialize42(os); diff --git a/vespa-hadoop/pom.xml b/vespa-hadoop/pom.xml index ea461e9783f..b9fbd589525 100644 --- a/vespa-hadoop/pom.xml +++ b/vespa-hadoop/pom.xml @@ -16,7 +16,7 @@ <properties> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> - <hadoop.version>2.7.3</hadoop.version> + <hadoop.version>2.8.0</hadoop.version> <pig.version>0.14.0</pig.version> </properties> diff --git a/vespaclient/src/vespa/vespaclient/vesparoute/mynetwork.cpp b/vespaclient/src/vespa/vespaclient/vesparoute/mynetwork.cpp index ec7c1c623c9..d9d8d0c4056 100644 --- a/vespaclient/src/vespa/vespaclient/vesparoute/mynetwork.cpp +++ b/vespaclient/src/vespa/vespaclient/vesparoute/mynetwork.cpp @@ -3,6 +3,8 @@ #include "mynetwork.h" #include <vespa/messagebus/emptyreply.h> #include <vespa/messagebus/sendproxy.h> +#include <vespa/messagebus/network/oosmanager.h> + class MyServiceAddress : public mbus::IServiceAddress { private: diff --git a/vespajlib/src/main/java/com/yahoo/lang/SettableOptional.java b/vespajlib/src/main/java/com/yahoo/lang/SettableOptional.java new file mode 100644 index 00000000000..00ff06b8f01 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/lang/SettableOptional.java @@ -0,0 +1,41 @@ +package com.yahoo.lang; + +import java.util.NoSuchElementException; +import java.util.Optional; + +/** + * An optional which contains a settable value + * + * @author bratseth + */ +public final class SettableOptional<T> { + + private T value = null; + + /** Creates a new empty settable optional */ + public SettableOptional() {} + + /** Creates a new settable optional with the given value */ + public SettableOptional(T value) { this.value = value; } + + public boolean isPresent() { + return value != null; + } + + public T get() { + if (value == null) + throw new NoSuchElementException("No value present"); + return value; + } + + public void set(T value) { + this.value = value; + } + + public Optional<T> asOptional() { + if (value == null) return Optional.empty(); + return Optional.of(value); + } + +} + diff --git a/vespalib/CMakeLists.txt b/vespalib/CMakeLists.txt index 112f67f3a70..3ac7b388ec4 100644 --- a/vespalib/CMakeLists.txt +++ b/vespalib/CMakeLists.txt @@ -6,6 +6,7 @@ vespa_define_module( EXTERNAL_DEPENDS lz4 + zstd APPS src/apps/make_fixture_macros @@ -22,6 +23,8 @@ vespa_define_module( src/tests/closure src/tests/component src/tests/compress + src/tests/compression + src/tests/data/databuffer src/tests/data/input_reader src/tests/data/lz4_encode_decode src/tests/data/memory_input diff --git a/vespalib/src/tests/compression/.gitignore b/vespalib/src/tests/compression/.gitignore new file mode 100644 index 00000000000..60ffd040a47 --- /dev/null +++ b/vespalib/src/tests/compression/.gitignore @@ -0,0 +1,4 @@ +*_test +.depend +Makefile +vespalib_compression_test_app diff --git a/vespalib/src/tests/compression/CMakeLists.txt b/vespalib/src/tests/compression/CMakeLists.txt new file mode 100644 index 00000000000..c5738733218 --- /dev/null +++ b/vespalib/src/tests/compression/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_compression_test_app TEST + SOURCES + compression_test.cpp + DEPENDS + vespalib +) +vespa_add_test(NAME vespalib_compression_test_app COMMAND vespalib_compression_test_app) diff --git a/document/src/tests/serialization/compression_test.cpp b/vespalib/src/tests/compression/compression_test.cpp index 6574b4ac34c..01cfe0af223 100644 --- a/document/src/tests/serialization/compression_test.cpp +++ b/vespalib/src/tests/compression/compression_test.cpp @@ -2,15 +2,14 @@ #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/stllike/string.h> -#include <vespa/document/util/compressor.h> +#include <vespa/vespalib/util/compressor.h> #include <vespa/vespalib/data/databuffer.h> #include <vespa/log/log.h> LOG_SETUP("compression_test"); -using namespace document; -using namespace document::compression; using namespace vespalib; +using namespace vespalib::compression; static vespalib::string _G_compressableText("AAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEEEEE" "AAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEEEEE" diff --git a/vespalib/src/tests/data/databuffer/.gitignore b/vespalib/src/tests/data/databuffer/.gitignore new file mode 100644 index 00000000000..f144796c66a --- /dev/null +++ b/vespalib/src/tests/data/databuffer/.gitignore @@ -0,0 +1 @@ +vespalib_data_databuffer_test_app diff --git a/vespalib/src/tests/data/databuffer/CMakeLists.txt b/vespalib/src/tests/data/databuffer/CMakeLists.txt new file mode 100644 index 00000000000..f1c6c7c1862 --- /dev/null +++ b/vespalib/src/tests/data/databuffer/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_data_databuffer_test_app TEST + SOURCES + databuffer_test.cpp + DEPENDS + vespalib +) +vespa_add_test(NAME vespalib_data_databuffer_test_app COMMAND vespalib_data_databuffer_test_app) diff --git a/staging_vespalib/src/tests/databuffer/databuffer_test.cpp b/vespalib/src/tests/data/databuffer/databuffer_test.cpp index f440ca1e15c..f440ca1e15c 100644 --- a/staging_vespalib/src/tests/databuffer/databuffer_test.cpp +++ b/vespalib/src/tests/data/databuffer/databuffer_test.cpp diff --git a/vespalib/src/vespa/vespalib/component/version.cpp b/vespalib/src/vespa/vespalib/component/version.cpp index 3fab6d6f130..af38a675de8 100644 --- a/vespalib/src/vespa/vespalib/component/version.cpp +++ b/vespalib/src/vespa/vespalib/component/version.cpp @@ -7,7 +7,6 @@ namespace vespalib { - Version::Version(int major, int minor, int micro, const string & qualifier) : _major(major), _minor(minor), @@ -89,7 +88,7 @@ Version::Version(const string & versionString) _qualifier(), _stringValue(versionString) { - if (versionString != "") { + if ( ! versionString.empty()) { stringref r(versionString.c_str(), versionString.size()); stringref::size_type dot(r.find('.')); stringref majorS(r.substr(0, dot)); diff --git a/vespalib/src/vespa/vespalib/data/CMakeLists.txt b/vespalib/src/vespa/vespalib/data/CMakeLists.txt index 29c8055a0c0..3a94e00ae33 100644 --- a/vespalib/src/vespa/vespalib/data/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/data/CMakeLists.txt @@ -1,6 +1,7 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_library(vespalib_vespalib_data OBJECT SOURCES + databuffer.cpp input.cpp input_reader.cpp lz4_input_decoder.cpp diff --git a/staging_vespalib/src/vespa/vespalib/data/databuffer.cpp b/vespalib/src/vespa/vespalib/data/databuffer.cpp index 529e475e987..5558a371836 100644 --- a/staging_vespalib/src/vespa/vespalib/data/databuffer.cpp +++ b/vespalib/src/vespa/vespalib/data/databuffer.cpp @@ -33,6 +33,7 @@ DataBuffer::DataBuffer(size_t len, size_t alignment, const Alloc & initial) } } +DataBuffer::~DataBuffer() = default; void DataBuffer::moveFreeToData(size_t len) diff --git a/staging_vespalib/src/vespa/vespalib/data/databuffer.h b/vespalib/src/vespa/vespalib/data/databuffer.h index f7707a3ea56..28524f373b2 100644 --- a/staging_vespalib/src/vespa/vespalib/data/databuffer.h +++ b/vespalib/src/vespa/vespalib/data/databuffer.h @@ -81,6 +81,8 @@ public: _buffer(Alloc::alloc(0)) { } + ~DataBuffer(); + /** * @return a pointer to the dead part of this buffer. **/ diff --git a/vespalib/src/vespa/vespalib/data/memory.h b/vespalib/src/vespa/vespalib/data/memory.h index 74024549a0b..eae7c8d2f23 100644 --- a/vespalib/src/vespa/vespalib/data/memory.h +++ b/vespalib/src/vespa/vespalib/data/memory.h @@ -25,6 +25,7 @@ struct Memory Memory(const vespalib::stringref &str_ref) : data(str_ref.data()), size(str_ref.size()) {} vespalib::string make_string() const; + vespalib::stringref make_stringref() const { return stringref(data, size); } bool operator == (const Memory &rhs) const { return ((size == rhs.size) && ((data == rhs.data) || diff --git a/vespalib/src/vespa/vespalib/util/CMakeLists.txt b/vespalib/src/vespa/vespalib/util/CMakeLists.txt index 310e1dde68d..6d08c3b1126 100644 --- a/vespalib/src/vespa/vespalib/util/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/util/CMakeLists.txt @@ -14,6 +14,7 @@ vespa_add_library(vespalib_vespalib_util OBJECT classname.cpp closuretask.cpp compress.cpp + compressor.cpp dual_merge_director.cpp error.cpp exception.cpp @@ -25,6 +26,7 @@ vespa_add_library(vespalib_vespalib_util OBJECT host_name.cpp joinable.cpp left_right_heap.cpp + lz4compressor.cpp md5.c printable.cpp priority_queue.cpp @@ -47,5 +49,6 @@ vespa_add_library(vespalib_vespalib_util OBJECT threadstackexecutorbase.cpp time_tracker.cpp valgrind.cpp + zstdcompressor.cpp DEPENDS ) diff --git a/vespalib/src/vespa/vespalib/util/compress.cpp b/vespalib/src/vespa/vespalib/util/compress.cpp index bf32fcfe593..617b632e0bf 100644 --- a/vespalib/src/vespa/vespalib/util/compress.cpp +++ b/vespalib/src/vespa/vespalib/util/compress.cpp @@ -4,8 +4,7 @@ #include "stringfmt.h" #include "exceptions.h" -namespace vespalib { -namespace compress { +namespace vespalib::compress { size_t Integer::compressedPositiveLength(uint64_t n) { @@ -84,4 +83,3 @@ size_t Integer::compress(int64_t n, void *destination) } } -} diff --git a/document/src/vespa/document/util/compressionconfig.h b/vespalib/src/vespa/vespalib/util/compressionconfig.h index 413bdc1fb3b..bb54a74ea41 100644 --- a/document/src/vespa/document/util/compressionconfig.h +++ b/vespalib/src/vespa/vespalib/util/compressionconfig.h @@ -4,8 +4,9 @@ #include <cmath> #include <cstdint> #include <cstddef> +#include <cstring> -namespace document { +namespace vespalib::compression { struct CompressionConfig { enum Type { @@ -51,6 +52,14 @@ struct CompressionConfig { default: return NONE; } } + static Type toType(const char * val) { + if (strncasecmp(val, "lz4", 3) == 0) { + return LZ4; + } if (strncasecmp(val, "zstd", 4) == 0) { + return ZSTD; + } + return NONE; + } static bool isCompressed(Type type) { return (type != CompressionConfig::NONE && type != CompressionConfig::UNCOMPRESSABLE); diff --git a/document/src/vespa/document/util/compressor.cpp b/vespalib/src/vespa/vespalib/util/compressor.cpp index cd45017dd69..6853c8375fd 100644 --- a/document/src/vespa/document/util/compressor.cpp +++ b/vespalib/src/vespa/vespalib/util/compressor.cpp @@ -7,11 +7,8 @@ #include <vespa/vespalib/data/databuffer.h> using vespalib::alloc::Alloc; -using vespalib::ConstBufferRef; -using vespalib::DataBuffer; -using vespalib::make_string; -namespace document::compression { +namespace vespalib::compression { CompressionConfig::Type compress(ICompressor & compressor, const CompressionConfig & compression, const ConstBufferRef & org, DataBuffer & dest) @@ -129,10 +126,10 @@ decompress(const CompressionConfig::Type & type, size_t uncompressedLen, const C size_t computeMaxCompressedsize(CompressionConfig::Type type, size_t payloadSize) { if (type == CompressionConfig::LZ4) { - document::LZ4Compressor lz4; + LZ4Compressor lz4; return lz4.adjustProcessLen(0, payloadSize); } else if (type == CompressionConfig::ZSTD) { - document::ZStdCompressor zstd; + ZStdCompressor zstd; return zstd.adjustProcessLen(0, payloadSize); } return payloadSize; diff --git a/document/src/vespa/document/util/compressor.h b/vespalib/src/vespa/vespalib/util/compressor.h index a8d4803e038..8f319a6735d 100644 --- a/document/src/vespa/document/util/compressor.h +++ b/vespalib/src/vespa/vespalib/util/compressor.h @@ -2,11 +2,11 @@ #pragma once #include "compressionconfig.h" -#include <vespa/vespalib/util/buffer.h> +#include "buffer.h" namespace vespalib { class DataBuffer; } -namespace document { +namespace vespalib::compression { class ICompressor { @@ -17,8 +17,6 @@ public: virtual size_t adjustProcessLen(uint16_t options, size_t len) const = 0; }; -namespace compression { - /** * Will try to compress a buffer according to the config. If the criteria can not * be met it will return NONE and dest will get the input buffer. @@ -43,9 +41,6 @@ CompressionConfig::Type compress(const CompressionConfig & compression, const ve */ void decompress(const CompressionConfig::Type & compression, size_t uncompressedLen, const vespalib::ConstBufferRef & org, vespalib::DataBuffer & dest, bool allowSwap); - size_t computeMaxCompressedsize(CompressionConfig::Type type, size_t uncompressedSize); } - -} diff --git a/document/src/vespa/document/util/lz4compressor.cpp b/vespalib/src/vespa/vespalib/util/lz4compressor.cpp index 04d68f394a1..366609ce7bf 100644 --- a/document/src/vespa/document/util/lz4compressor.cpp +++ b/vespalib/src/vespa/vespalib/util/lz4compressor.cpp @@ -8,7 +8,7 @@ using vespalib::alloc::Alloc; -namespace document { +namespace vespalib::compression { size_t LZ4Compressor::adjustProcessLen(uint16_t, size_t len) const { return LZ4_compressBound(len); } diff --git a/document/src/vespa/document/util/lz4compressor.h b/vespalib/src/vespa/vespalib/util/lz4compressor.h index 6dd8f8fd1cb..558088a914c 100644 --- a/document/src/vespa/document/util/lz4compressor.h +++ b/vespalib/src/vespa/vespalib/util/lz4compressor.h @@ -3,7 +3,7 @@ #include "compressor.h" -namespace document { +namespace vespalib::compression { class LZ4Compressor : public ICompressor { diff --git a/document/src/vespa/document/util/zstdcompressor.cpp b/vespalib/src/vespa/vespalib/util/zstdcompressor.cpp index 74a17212a2e..2f09abf4846 100644 --- a/document/src/vespa/document/util/zstdcompressor.cpp +++ b/vespalib/src/vespa/vespalib/util/zstdcompressor.cpp @@ -9,7 +9,7 @@ using vespalib::alloc::Alloc; -namespace document { +namespace vespalib::compression { namespace { diff --git a/document/src/vespa/document/util/zstdcompressor.h b/vespalib/src/vespa/vespalib/util/zstdcompressor.h index ba12ed2594c..0d84b8e7ca4 100644 --- a/document/src/vespa/document/util/zstdcompressor.h +++ b/vespalib/src/vespa/vespalib/util/zstdcompressor.h @@ -3,7 +3,7 @@ #include "compressor.h" -namespace document { +namespace vespalib::compression { class ZStdCompressor : public ICompressor { |