diff options
155 files changed, 3062 insertions, 981 deletions
diff --git a/bootstrap.sh b/bootstrap.sh index b71af73c529..29e18226ca7 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -26,7 +26,7 @@ else fi mvn_install() { - mvn --quiet --batch-mode --threads 1.5C --no-snapshot-updates install -Dmaven.test.skip=true -Dmaven.javadoc.skip=true "$@" + mvn -e -X --quiet --batch-mode --threads 1.5C --no-snapshot-updates install -Dmaven.test.skip=true -Dmaven.javadoc.skip=true "$@" } # Generate vtag map diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationFile.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationFile.java index 60524fbca8d..a8e1256e032 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationFile.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationFile.java @@ -111,8 +111,8 @@ public class FilesApplicationFile extends ApplicationFile { file.getParentFile().mkdirs(); } try { - String data = com.yahoo.io.IOUtils.readAll(input); String status = file.exists() ? ApplicationFile.ContentStatusChanged : ApplicationFile.ContentStatusNew; + String data = com.yahoo.io.IOUtils.readAll(input); IOUtils.writeFile(file, data, false); writeMetaFile(data, status); } catch (IOException e) { @@ -122,6 +122,21 @@ public class FilesApplicationFile extends ApplicationFile { } @Override + public ApplicationFile appendFile(String value) { + if (file.getParentFile() != null) { + file.getParentFile().mkdirs(); + } + try { + String status = file.exists() ? ApplicationFile.ContentStatusChanged : ApplicationFile.ContentStatusNew; + IOUtils.writeFile(file, value, true); + writeMetaFile(value, status); + } catch (IOException e) { + throw new RuntimeException(e); + } + return this; + } + + @Override public List<ApplicationFile> listFiles(final PathFilter filter) { List<ApplicationFile> files = new ArrayList<>(); if (!file.isDirectory()) { diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationFile.java b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationFile.java index 0384a5c7a1c..33b7807aac5 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationFile.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationFile.java @@ -75,6 +75,13 @@ public abstract class ApplicationFile implements Comparable<ApplicationFile> { public abstract ApplicationFile writeFile(Reader input); /** + * Appends the given string to this text file. + * + * @return this + */ + public abstract ApplicationFile appendFile(String value); + + /** * List the files under this directory. If this is file, an empty list is returned. * Only immediate files/subdirectories are returned. * diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java index 43d3fafdb78..eb0c6067fca 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java @@ -47,8 +47,7 @@ import java.util.Set; /** * Contains various state during deploy that should be available in all builders of a {@link com.yahoo.config.model.ConfigModel} * - * @author lulf - * @since 5.8 + * @author Ulf Lilleengen */ public class DeployState implements ConfigDefinitionStore { @@ -211,9 +210,9 @@ public class DeployState implements ConfigDefinitionStore { public QueryProfiles getQueryProfiles() { return queryProfiles; } public SemanticRules getSemanticRules() { return semanticRules; } - + public Version getWantedNodeVespaVersion() { return wantedNodeVespaVersion; } - + public Instant now() { return now; } public boolean disableFiledistributor() { return disableFiledistributor; } @@ -288,7 +287,7 @@ public class DeployState implements ConfigDefinitionStore { this.now = now; return this; } - + public Builder wantedNodeVespaVersion(Version version) { this.wantedNodeVespaVersion = version; return this; @@ -309,10 +308,12 @@ public class DeployState implements ConfigDefinitionStore { zone, queryProfiles, semanticRules, now, wantedNodeVespaVersion, disableFiledistributor); } - private SearchDocumentModel createSearchDocumentModel(RankProfileRegistry rankProfileRegistry, DeployLogger logger, QueryProfiles queryProfiles) { + private SearchDocumentModel createSearchDocumentModel(RankProfileRegistry rankProfileRegistry, + DeployLogger logger, + QueryProfiles queryProfiles) { Collection<NamedReader> readers = applicationPackage.getSearchDefinitions(); Map<String, String> names = new LinkedHashMap<>(); - SearchBuilder builder = new SearchBuilder(applicationPackage, rankProfileRegistry); + SearchBuilder builder = new SearchBuilder(applicationPackage, rankProfileRegistry, queryProfiles.getRegistry()); for (NamedReader reader : readers) { try { String readerName = reader.getName(); @@ -321,7 +322,8 @@ public class DeployState implements ConfigDefinitionStore { names.put(searchName, sdName); if (!sdName.equals(searchName)) { throw new IllegalArgumentException("Search definition file name ('" + sdName + "') and name of " + - "search element ('" + searchName + "') are not equal for file '" + readerName + "'"); + "search element ('" + searchName + + "') are not equal for file '" + readerName + "'"); } } catch (ParseException e) { throw new IllegalArgumentException("Could not parse search definition file '" + getSearchDefinitionRelativePath(reader.getName()) + "': " + e.getMessage(), e); diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java index 271ec6958ec..6a8d754af1c 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java @@ -8,6 +8,8 @@ import com.yahoo.config.provision.Version; import com.yahoo.io.IOUtils; import com.yahoo.path.Path; import com.yahoo.io.reader.NamedReader; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.search.query.profile.config.QueryProfileXMLReader; import com.yahoo.searchdefinition.*; import com.yahoo.searchdefinition.parser.ParseException; import com.yahoo.vespa.config.ConfigDefinitionKey; @@ -36,9 +38,11 @@ public class MockApplicationPackage implements ApplicationPackage { private final Optional<String> deploymentSpec; private final Optional<String> validationOverrides; private final boolean failOnValidateXml; + private final QueryProfileRegistry queryProfileRegistry; protected MockApplicationPackage(String hosts, String services, List<String> searchDefinitions, String searchDefinitionDir, - String deploymentSpec, String validationOverrides, boolean failOnValidateXml) { + String deploymentSpec, String validationOverrides, boolean failOnValidateXml, + String queryProfile, String queryProfileType) { this.hostsS = hosts; this.servicesS = services; this.searchDefinitions = searchDefinitions; @@ -46,6 +50,8 @@ public class MockApplicationPackage implements ApplicationPackage { this.deploymentSpec = Optional.ofNullable(deploymentSpec); this.validationOverrides = Optional.ofNullable(validationOverrides); this.failOnValidateXml = failOnValidateXml; + queryProfileRegistry = new QueryProfileXMLReader().read(asNamedReaderList(queryProfileType), + asNamedReaderList(queryProfile)); } @Override @@ -67,7 +73,9 @@ public class MockApplicationPackage implements ApplicationPackage { @Override public List<NamedReader> getSearchDefinitions() { ArrayList<NamedReader> readers = new ArrayList<>(); - SearchBuilder searchBuilder = new SearchBuilder(this, new RankProfileRegistry()); + SearchBuilder searchBuilder = new SearchBuilder(this, + new RankProfileRegistry(), + queryProfileRegistry); for (String sd : searchDefinitions) { try { String name = searchBuilder.importString(sd); @@ -123,6 +131,8 @@ public class MockApplicationPackage implements ApplicationPackage { return Collections.emptyList(); } + public QueryProfileRegistry getQueryProfiles() { return queryProfileRegistry; } + @Override public Reader getRankingExpression(String name) { File expressionFile = new File(searchDefinitionDir, name); @@ -147,6 +157,7 @@ public class MockApplicationPackage implements ApplicationPackage { } public static class Builder { + private String hosts = null; private String services = null; private List<String> searchDefinitions = Collections.emptyList(); @@ -154,6 +165,8 @@ public class MockApplicationPackage implements ApplicationPackage { private String deploymentSpec = null; private String validationOverrides = null; private boolean failOnValidateXml = false; + private String queryProfile = null; + private String queryProfileType = null; public Builder() { } @@ -206,9 +219,20 @@ public class MockApplicationPackage implements ApplicationPackage { return this; } + public Builder queryProfile(String queryProfile) { + this.queryProfile = queryProfile; + return this; + } + + public Builder queryProfileType(String queryProfileType) { + this.queryProfileType = queryProfileType; + return this; + } + public ApplicationPackage build() { return new MockApplicationPackage(hosts, services, searchDefinitions, searchDefinitionDir, - deploymentSpec, validationOverrides, failOnValidateXml); + deploymentSpec, validationOverrides, failOnValidateXml, + queryProfile, queryProfileType); } } @@ -242,4 +266,16 @@ public class MockApplicationPackage implements ApplicationPackage { } } + private List<NamedReader> asNamedReaderList(String value) { + if (value == null) return Collections.emptyList(); + return Collections.singletonList(new NamedReader(extractId(value) + ".xml", new StringReader(value))); + } + + private String extractId(String xmlStringWithIdAttribute) { + int idStart = xmlStringWithIdAttribute.indexOf("id="); + int idEnd = Math.min(xmlStringWithIdAttribute.indexOf(" ", idStart), + xmlStringWithIdAttribute.indexOf(">", idStart)); + return xmlStringWithIdAttribute.substring(idStart + 4, idEnd - 1); + } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java index 90ea5a11486..b4d72f3a456 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java @@ -43,20 +43,18 @@ import static java.util.stream.Collectors.toSet; */ public class DocumentModelBuilder { - public static class RetryLaterException extends IllegalArgumentException { - public RetryLaterException(String message) { - super(message); - } - } private DocumentModel model; private final Map<NewDocumentType, List<SDDocumentType>> scratchInheritsMap = new HashMap<>(); + public DocumentModelBuilder(DocumentModel model) { this.model = model; model.getDocumentManager().add(VespaDocumentType.INSTANCE); } + public boolean valid() { return scratchInheritsMap.isEmpty(); } + public void addToModel(Collection<Search> searchList) { List<SDDocumentType> docList = new LinkedList<>(); for (Search search : searchList) { @@ -65,7 +63,8 @@ public class DocumentModelBuilder { docList = sortDocumentTypes(docList); addDocumentTypes(docList); for (Collection<Search> toAdd = tryAdd(searchList); - !toAdd.isEmpty() && (toAdd.size() < searchList.size()); toAdd = tryAdd(searchList)) { + ! toAdd.isEmpty() && (toAdd.size() < searchList.size()); + toAdd = tryAdd(searchList)) { searchList = toAdd; } } @@ -126,6 +125,7 @@ public class DocumentModelBuilder { } return left; } + public void addToModel(Search search) { // Then we add the search specific stuff SearchDef searchDef = new SearchDef(search.getName()); @@ -133,9 +133,9 @@ public class DocumentModelBuilder { for (Field f : search.getDocument().fieldSet()) { addSearchField((SDField) f, searchDef); } - for(SDField field : search.allConcreteFields()) { - for(Attribute attribute : field.getAttributes().values()) { - if (!searchDef.getFields().containsKey(attribute.getName())) { + for (SDField field : search.allConcreteFields()) { + for (Attribute attribute : field.getAttributes().values()) { + if ( ! searchDef.getFields().containsKey(attribute.getName())) { searchDef.add(new SearchField(new Field(attribute.getName(), field), !field.getIndices().isEmpty(), true)); } } @@ -146,14 +146,15 @@ public class DocumentModelBuilder { } model.getSearchManager().add(searchDef); } + private static void addSearchFields(Collection<SDField> fields, SearchDef searchDef) { for (SDField field : fields) { addSearchField(field, searchDef); } } - private static void addSearchField(SDField field, SearchDef searchDef) { - SearchField searchField = + private static void addSearchField(SDField field, SearchDef searchDef) { + SearchField searchField = new SearchField(field, field.getIndices().containsKey(field.getName()) && field.getIndices().get(field.getName()).getType().equals(Index.Type.VESPA), field.getAttributes().containsKey(field.getName())); @@ -453,4 +454,11 @@ public class DocumentModelBuilder { } return false; } + + public static class RetryLaterException extends IllegalArgumentException { + public RetryLaterException(String message) { + super(message); + } + } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/FeatureNames.java b/config-model/src/main/java/com/yahoo/searchdefinition/FeatureNames.java new file mode 100644 index 00000000000..dd03cb8b2a7 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/FeatureNames.java @@ -0,0 +1,120 @@ +/* + * // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + * + * + */ +package com.yahoo.searchdefinition; + +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +/** + * Utility methods for query, document and constant rank feature names + * + * @author bratseth + */ +public class FeatureNames { + + private static final Pattern identifierRegexp = Pattern.compile("[A-Za-z0-9_][A-Za-z0-9_-]*"); + + /** + * <p>Returns the given query, document or constant feature in canonical form. + * A feature name consists of a feature type name (query, attribute or constant), + * followed by one argument enclosed in quotes. + * The argument may be an identifier or any string single or double quoted.</p> + * + * <p>Argument string values may not contain comma, single quote nor double quote characters.</p> + * + * <p><i>The canonical form use no quotes for arguments which are identifiers, and double quotes otherwise.</i></p> + * + * <p>Note that the above definition is not true for features in general, which accept any ranking expression + * as argument.</p> + * + * @throws IllegalArgumentException if the feature name is not valid + */ + // Note that this implementation is more general than what is described above: + // It accepts any number of arguments and an optional output + public static String canonicalize(String feature) { + return canonicalizeIfValid(feature).orElseThrow(() -> + new IllegalArgumentException("A feature name must be on the form query(name), attribute(name) or " + + "constant(name), but was '" + feature + "'" + )); + } + + /** + * Canonicalizes the given argument as in canonicalize, but returns empty instead of throwing an exception if + * the argument is not a valid feature + */ + public static Optional<String> canonicalizeIfValid(String feature) { + int startParenthesis = feature.indexOf('('); + if (startParenthesis < 0) + return Optional.empty(); + int endParenthesis = feature.lastIndexOf(')'); + String featureType = feature.substring(0, startParenthesis); + if ( ! ( featureType.equals("query") || featureType.equals("attribute") || featureType.equals("constant"))) + return Optional.empty(); + if (startParenthesis < 1) return Optional.of(feature); // No arguments + if (endParenthesis < startParenthesis) + return Optional.empty(); + String argumentString = feature.substring(startParenthesis + 1, endParenthesis); + List<String> canonicalizedArguments = + Arrays.stream(argumentString.split(",")) + .map(FeatureNames::canonicalizeArgument) + .collect(Collectors.toList()); + return Optional.of(featureType + "(" + + canonicalizedArguments.stream().collect(Collectors.joining(",")) + + feature.substring(endParenthesis)); + } + + /** Canomicalizes a single argument */ + private static String canonicalizeArgument(String argument) { + if (argument.startsWith("'")) { + if ( ! argument.endsWith("'")) + throw new IllegalArgumentException("Feature arguments starting by a single quote " + + "must end by a single quote, but was \"" + argument + "\""); + argument = argument.substring(1, argument.length() - 1); + } + if (argument.startsWith("\"")) { + if ( ! argument.endsWith("\"")) + throw new IllegalArgumentException("Feature arguments starting by a double quote " + + "must end by a double quote, but was '" + argument + "'"); + argument = argument.substring(1, argument.length() - 1); + } + if (identifierRegexp.matcher(argument).matches()) + return argument; + else + return "\"" + argument + "\""; + } + + public static String asConstantFeature(String constantName) { + return canonicalize("constant(\"" + constantName + "\")"); + } + + public static String asAttributeFeature(String attributeName) { + return canonicalize("attribute(\"" + attributeName + "\")"); + } + + public static String asQueryFeature(String propertyName) { + return canonicalize("query(\"" + propertyName + "\")"); + } + + /** + * Returns the single argument of the given feature name, without any quotes, + * or empty if it is not a valid query, attribute or constant feature name + */ + public static Optional<String> argumentOf(String feature) { + return canonicalizeIfValid(feature).map(f -> { + int startParenthesis = f.indexOf("("); + int endParenthesis = f.indexOf(")"); + String possiblyQuotedArgument = f.substring(startParenthesis + 1, endParenthesis); + if (possiblyQuotedArgument.startsWith("\"")) + return possiblyQuotedArgument.substring(1, possiblyQuotedArgument.length() - 1); + else + return possiblyQuotedArgument; + }); + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java index b185680d41c..bcbc7cc99e2 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java @@ -2,12 +2,14 @@ package com.yahoo.searchdefinition; import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.deploy.DeployState; import com.yahoo.io.reader.NamedReader; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.query.profile.QueryProfile; import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.search.query.profile.config.QueryProfileXMLReader; import com.yahoo.search.query.profile.types.FieldDescription; +import com.yahoo.search.query.profile.types.QueryProfileType; import com.yahoo.search.query.profile.types.TensorFieldType; import com.yahoo.search.query.ranking.Diversity; import com.yahoo.searchdefinition.document.SDField; @@ -16,7 +18,6 @@ import com.yahoo.searchdefinition.parser.ParseException; import com.yahoo.searchlib.rankingexpression.ExpressionFunction; import com.yahoo.searchlib.rankingexpression.FeatureList; import com.yahoo.searchlib.rankingexpression.RankingExpression; -import com.yahoo.searchlib.rankingexpression.evaluation.FeatureNames; import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue; import com.yahoo.searchlib.rankingexpression.evaluation.TypeMapContext; import com.yahoo.searchlib.rankingexpression.evaluation.Value; @@ -677,10 +678,10 @@ public class RankProfile implements Serializable, Cloneable { * Returns a copy of this where the content is optimized for execution. * Compiled profiles should never be modified. */ - public RankProfile compile() { + public RankProfile compile(QueryProfileRegistry queryProfiles) { try { RankProfile compiled = this.clone(); - compiled.compileThis(); + compiled.compileThis(queryProfiles); return compiled; } catch (IllegalArgumentException e) { @@ -688,7 +689,7 @@ public class RankProfile implements Serializable, Cloneable { } } - private void compileThis() { + private void compileThis(QueryProfileRegistry queryProfiles) { parseExpressions(); checkNameCollisions(getMacros(), getConstants()); @@ -697,13 +698,14 @@ public class RankProfile implements Serializable, Cloneable { for (Map.Entry<String, Macro> macroEntry : getMacros().entrySet()) { Macro compiledMacro = macroEntry.getValue().clone(); compiledMacro.setRankingExpression(compile(macroEntry.getValue().getRankingExpression(), - getConstants(), Collections.<String, Macro>emptyMap())); + queryProfiles, + getConstants(), Collections.<String, Macro>emptyMap())); compiledMacros.put(macroEntry.getKey(), compiledMacro); } macros = compiledMacros; Map<String, Macro> inlineMacros = keepInline(compiledMacros); - firstPhaseRanking = compile(this.getFirstPhaseRanking(), getConstants(), inlineMacros); - secondPhaseRanking = compile(this.getSecondPhaseRanking(), getConstants(), inlineMacros); + firstPhaseRanking = compile(this.getFirstPhaseRanking(), queryProfiles, getConstants(), inlineMacros); + secondPhaseRanking = compile(this.getSecondPhaseRanking(), queryProfiles, getConstants(), inlineMacros); } private void checkNameCollisions(Map<String, Macro> macros, Map<String, Value> constants) { @@ -723,12 +725,14 @@ public class RankProfile implements Serializable, Cloneable { } private RankingExpression compile(RankingExpression expression, + QueryProfileRegistry queryProfiles, Map<String, Value> constants, Map<String, Macro> inlineMacros) { if (expression == null) return null; Map<String, String> rankPropertiesOutput = new HashMap<>(); RankProfileTransformContext context = new RankProfileTransformContext(this, + queryProfiles, constants, inlineMacros, rankPropertiesOutput); @@ -743,10 +747,12 @@ public class RankProfile implements Serializable, Cloneable { * Creates a context containing the type information of all constants, attributes and query profiles * referable from this rank profile. */ - public TypeContext typeContext() { + public TypeContext typeContext(QueryProfileRegistry queryProfiles) { TypeMapContext context = new TypeMapContext(); - // Add constants + // Add small constants + getConstants().forEach((k, v) -> context.setType(FeatureNames.asConstantFeature(k), v.type())); + // Add large constants getSearch().getRankingConstants().forEach((k, v) -> context.setType(FeatureNames.asConstantFeature(k), v.getTensorType())); // Add attributes @@ -755,37 +761,24 @@ public class RankProfile implements Serializable, Cloneable { } // Add query features from rank profile types reached from the "default" profile - QueryProfile profile = queryProfilesOf(getSearch().sourceApplication()).getComponent("default"); - if (profile != null && profile.getType() != null) { - profile.listTypes(CompoundName.empty, Collections.emptyMap()).forEach((prefix, queryProfileType) -> { - for (FieldDescription field : queryProfileType.declaredFields().values()) { - TensorType type = TensorType.empty; // assume the empty (aka double) type by default - if (field.getType() instanceof TensorFieldType) - type = ((TensorFieldType)field.getType()).type(); - - String feature = FeatureNames.asQueryFeature(prefix.append(field.getName()).toString()); - context.setType(feature, type); - } - }); + for (QueryProfileType queryProfileType : queryProfiles.getTypeRegistry().allComponents()) { + for (FieldDescription field : queryProfileType.declaredFields().values()) { + TensorType type = field.getType().asTensorType(); + String feature = FeatureNames.asQueryFeature(field.getName()); + TensorType existingType = context.getType(feature); + if (existingType != null) + type = existingType.dimensionwiseGeneralizationWith(type).orElseThrow( () -> + new IllegalArgumentException(queryProfileType + " contains query feature " + feature + + " with type " + field.getType().asTensorType() + + ", but this is already defined " + + "in another query profile with type " + context.getType(feature))); + context.setType(feature, type); + } } return context; } - private QueryProfileRegistry queryProfilesOf(ApplicationPackage applicationPackage) { - List<NamedReader> queryProfileFiles = null; - List<NamedReader> queryProfileTypeFiles = null; - try { - queryProfileFiles = applicationPackage.getQueryProfileFiles(); - queryProfileTypeFiles = applicationPackage.getQueryProfileTypeFiles(); - return new QueryProfileXMLReader().read(queryProfileTypeFiles, queryProfileFiles); - } - finally { - NamedReader.closeAll(queryProfileFiles); - NamedReader.closeAll(queryProfileTypeFiles); - } - } - /** * A rank setting. The identity of a rank setting is its field name and type (not value). * A rank setting is immutable. diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java index ba0ede77b55..762c0fec838 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java @@ -8,6 +8,7 @@ import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.document.DocumentTypeManager; import com.yahoo.io.IOUtils; import com.yahoo.io.reader.NamedReader; +import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.derived.SearchOrderer; import com.yahoo.searchdefinition.document.SDDocumentType; import com.yahoo.searchdefinition.parser.ParseException; @@ -32,8 +33,6 @@ import java.util.List; * all available search definitions, using the importXXX() methods, 2) provide the available rank types and rank * expressions, using the setRankXXX() methods, 3) invoke the {@link #build()} method, and 4) retrieve the built * search objects using the {@link #getSearch(String)} method. - * - * @author TODO: Who created this? */ // TODO: This should be cleaned up and more or maybe completely taken over by MockApplicationPackage public class SearchBuilder { @@ -43,23 +42,35 @@ public class SearchBuilder { private ApplicationPackage app = null; private boolean isBuilt = false; private DocumentModel model = new DocumentModel(); - private RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + private final RankProfileRegistry rankProfileRegistry; + private final QueryProfileRegistry queryProfileRegistry; + /** For testing only */ public SearchBuilder() { - this.app = MockApplicationPackage.createEmpty(); + this(MockApplicationPackage.createEmpty(), new RankProfileRegistry(), new QueryProfileRegistry()); } + /** For testing only */ public SearchBuilder(ApplicationPackage app) { - this.app = app; + this(app, new RankProfileRegistry(), new QueryProfileRegistry()); } - public SearchBuilder(ApplicationPackage app, RankProfileRegistry rankProfileRegistry) { - this.app = app; - this.rankProfileRegistry = rankProfileRegistry; + /** For testing only */ + public SearchBuilder(RankProfileRegistry rankProfileRegistry) { + this(MockApplicationPackage.createEmpty(), rankProfileRegistry, new QueryProfileRegistry()); } - public SearchBuilder(RankProfileRegistry rankProfileRegistry) { - this(MockApplicationPackage.createEmpty(), rankProfileRegistry); + /** For testing only */ + public SearchBuilder(RankProfileRegistry rankProfileRegistry, QueryProfileRegistry queryProfileRegistry) { + this(MockApplicationPackage.createEmpty(), rankProfileRegistry, queryProfileRegistry); + } + + public SearchBuilder(ApplicationPackage app, + RankProfileRegistry rankProfileRegistry, + QueryProfileRegistry queryProfileRegistry) { + this.app = app; + this.rankProfileRegistry = rankProfileRegistry; + this.queryProfileRegistry = queryProfileRegistry; } /** @@ -164,12 +175,12 @@ public class SearchBuilder { String rawName = rawSearch.getName(); if (rawSearch.isProcessed()) { throw new IllegalArgumentException("A search definition with a search section called '" + rawName + - "' has already been processed."); + "' has already been processed."); } for (Search search : searchList) { if (rawName.equals(search.getName())) { throw new IllegalArgumentException("A search definition with a search section called '" + rawName + - "' has already been added."); + "' has already been added."); } } searchList.add(rawSearch); @@ -309,7 +320,7 @@ public class SearchBuilder { builder.build(); return builder; } - + /** * Convenience factory method to import and build a {@link Search} object from a file. Only for testing. * @@ -319,7 +330,7 @@ public class SearchBuilder { * @throws ParseException if there was a problem parsing the file content. */ public static SearchBuilder createFromFile(String fileName) throws IOException, ParseException { - return createFromFile(fileName, new BaseDeployLogger(), new RankProfileRegistry()); + return createFromFile(fileName, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfileRegistry()); } /** @@ -332,19 +343,28 @@ public class SearchBuilder { * @throws IOException if there was a problem reading the file. * @throws ParseException if there was a problem parsing the file content. */ - public static SearchBuilder createFromFile(String fileName, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry) + public static SearchBuilder createFromFile(String fileName, + DeployLogger deployLogger, + RankProfileRegistry rankProfileRegistry, + QueryProfileRegistry queryprofileRegistry) throws IOException, ParseException { - SearchBuilder builder = new SearchBuilder(MockApplicationPackage.createEmpty(), rankProfileRegistry); + SearchBuilder builder = new SearchBuilder(MockApplicationPackage.createEmpty(), + rankProfileRegistry, + queryprofileRegistry); builder.importFile(fileName); builder.build(deployLogger, new QueryProfiles()); return builder; } public static SearchBuilder createFromDirectory(String dir) throws IOException, ParseException { - return createFromDirectory(dir, new RankProfileRegistry()); - } - public static SearchBuilder createFromDirectory(String dir, RankProfileRegistry rankProfileRegistry) throws IOException, ParseException { - SearchBuilder builder = new SearchBuilder(MockApplicationPackage.fromSearchDefinitionDirectory(dir), rankProfileRegistry); + return createFromDirectory(dir, new RankProfileRegistry(), new QueryProfileRegistry()); + } + public static SearchBuilder createFromDirectory(String dir, + RankProfileRegistry rankProfileRegistry, + QueryProfileRegistry queryProfileRegistry) throws IOException, ParseException { + SearchBuilder builder = new SearchBuilder(MockApplicationPackage.fromSearchDefinitionDirectory(dir), + rankProfileRegistry, + queryProfileRegistry); for (Iterator<Path> i = Files.list(new File(dir).toPath()).filter(p -> p.getFileName().toString().endsWith(".sd")).iterator(); i.hasNext(); ) { builder.importFile(i.next()); } @@ -353,7 +373,7 @@ public class SearchBuilder { } // TODO: The build methods below just call the create methods above - remove - + /** * Convenience factory method to import and build a {@link Search} object from a file. Only for testing. * @@ -363,7 +383,7 @@ public class SearchBuilder { * @throws ParseException Thrown if there was a problem parsing the file content. */ public static Search buildFromFile(String fileName) throws IOException, ParseException { - return buildFromFile(fileName, new BaseDeployLogger(), new RankProfileRegistry()); + return buildFromFile(fileName, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfileRegistry()); } /** @@ -375,9 +395,11 @@ public class SearchBuilder { * @throws IOException Thrown if there was a problem reading the file. * @throws ParseException Thrown if there was a problem parsing the file content. */ - public static Search buildFromFile(String fileName, RankProfileRegistry rankProfileRegistry) + public static Search buildFromFile(String fileName, + RankProfileRegistry rankProfileRegistry, + QueryProfileRegistry queryProfileRegistry) throws IOException, ParseException { - return buildFromFile(fileName, new BaseDeployLogger(), rankProfileRegistry); + return buildFromFile(fileName, new BaseDeployLogger(), rankProfileRegistry, queryProfileRegistry); } /** @@ -390,20 +412,25 @@ public class SearchBuilder { * @throws IOException Thrown if there was a problem reading the file. * @throws ParseException Thrown if there was a problem parsing the file content. */ - public static Search buildFromFile(String fileName, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry) + public static Search buildFromFile(String fileName, + DeployLogger deployLogger, + RankProfileRegistry rankProfileRegistry, + QueryProfileRegistry queryProfileRegistry) throws IOException, ParseException { - return createFromFile(fileName, deployLogger, rankProfileRegistry).getSearch(); + return createFromFile(fileName, deployLogger, rankProfileRegistry, queryProfileRegistry).getSearch(); } - + /** * Convenience factory method to import and build a {@link Search} object from a raw object. * - * @param rawSearch The raw object to build from. - * @return The built {@link SearchBuilder} object. + * @param rawSearch the raw object to build from. + * @return the built {@link SearchBuilder} object. * @see #importRawSearch(Search) */ - public static SearchBuilder createFromRawSearch(Search rawSearch, RankProfileRegistry rankProfileRegistry) { - SearchBuilder builder = new SearchBuilder(rankProfileRegistry); + public static SearchBuilder createFromRawSearch(Search rawSearch, + RankProfileRegistry rankProfileRegistry, + QueryProfileRegistry queryProfileRegistry) { + SearchBuilder builder = new SearchBuilder(rankProfileRegistry, queryProfileRegistry); builder.importRawSearch(rawSearch); builder.build(); return builder; @@ -416,11 +443,18 @@ public class SearchBuilder { * @return The built {@link Search} object. * @see #importRawSearch(Search) */ - public static Search buildFromRawSearch(Search rawSearch, RankProfileRegistry rankProfileRegistry) { - return createFromRawSearch(rawSearch, rankProfileRegistry).getSearch(); + public static Search buildFromRawSearch(Search rawSearch, + RankProfileRegistry rankProfileRegistry, + QueryProfileRegistry queryProfileRegistry) { + return createFromRawSearch(rawSearch, rankProfileRegistry, queryProfileRegistry).getSearch(); } public RankProfileRegistry getRankProfileRegistry() { return rankProfileRegistry; } + + public QueryProfileRegistry getQueryProfileRegistry() { + return queryProfileRegistry; + } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/TypeMapContext.java b/config-model/src/main/java/com/yahoo/searchdefinition/TypeMapContext.java new file mode 100644 index 00000000000..40e9db1413f --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/TypeMapContext.java @@ -0,0 +1,32 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition; + +import com.yahoo.tensor.TensorType; +import com.yahoo.tensor.evaluation.TypeContext; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +/** + * A context which only contains type information. + * + * @author bratseth + */ +public class TypeMapContext implements TypeContext { + + private final Map<String, TensorType> featureTypes = new HashMap<>(); + + public void setType(String name, TensorType type) { + featureTypes.put(FeatureNames.canonicalize(name), type); + } + + @Override + public TensorType getType(String name) { + return featureTypes.get(FeatureNames.canonicalize(name)); + } + + /** Returns an unmodifiable map of the bindings in this */ + public Map<String, TensorType> bindings() { return Collections.unmodifiableMap(featureTypes); } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/UnprocessingSearchBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/UnprocessingSearchBuilder.java index b448005c6a5..1201de86d8d 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/UnprocessingSearchBuilder.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/UnprocessingSearchBuilder.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition; +import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.parser.ParseException; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; @@ -13,16 +14,19 @@ import java.io.IOException; */ public class UnprocessingSearchBuilder extends SearchBuilder { - public UnprocessingSearchBuilder(ApplicationPackage app, RankProfileRegistry rankProfileRegistry) { - super(app, rankProfileRegistry); + public UnprocessingSearchBuilder(ApplicationPackage app, + RankProfileRegistry rankProfileRegistry, + QueryProfileRegistry queryProfileRegistry) { + super(app, rankProfileRegistry, queryProfileRegistry); } public UnprocessingSearchBuilder() { super(); } - public UnprocessingSearchBuilder(RankProfileRegistry rankProfileRegistry) { - super(rankProfileRegistry); + public UnprocessingSearchBuilder(RankProfileRegistry rankProfileRegistry, + QueryProfileRegistry queryProfileRegistry) { + super(rankProfileRegistry, queryProfileRegistry); } @Override diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java index fa202770e26..985087e905b 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java @@ -8,6 +8,7 @@ import com.yahoo.document.DocumenttypesConfig; import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.io.IOUtils; import com.yahoo.protect.Validator; +import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.derived.validation.Validation; @@ -45,8 +46,8 @@ public class DerivedConfiguration { * modified. * @param rankProfileRegistry a {@link com.yahoo.searchdefinition.RankProfileRegistry} */ - public DerivedConfiguration(Search search, RankProfileRegistry rankProfileRegistry) { - this(search, null, new BaseDeployLogger(), rankProfileRegistry); + public DerivedConfiguration(Search search, RankProfileRegistry rankProfileRegistry, QueryProfileRegistry queryProfiles) { + this(search, null, new BaseDeployLogger(), rankProfileRegistry, queryProfiles); } /** @@ -60,8 +61,12 @@ public class DerivedConfiguration { * @param deployLogger a {@link DeployLogger} for logging when * doing operations on this * @param rankProfileRegistry a {@link com.yahoo.searchdefinition.RankProfileRegistry} + * @param queryProfiles the query profiles of this application */ - public DerivedConfiguration(Search search, List<Search> abstractSearchList, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry) { + public DerivedConfiguration(Search search, List<Search> abstractSearchList, + DeployLogger deployLogger, + RankProfileRegistry rankProfileRegistry, + QueryProfileRegistry queryProfiles) { Validator.ensureNotNull("Search definition", search); if ( ! search.isProcessed()) { throw new IllegalArgumentException("Search '" + search.getName() + "' not processed."); @@ -83,7 +88,7 @@ public class DerivedConfiguration { summaries = new Summaries(search, deployLogger); summaryMap = new SummaryMap(search, summaries); juniperrc = new Juniperrc(search); - rankProfileList = new RankProfileList(search, attributeFields, rankProfileRegistry); + rankProfileList = new RankProfileList(search, attributeFields, rankProfileRegistry, queryProfiles); indexingScript = new IndexingScript(search); indexInfo = new IndexInfo(search); indexSchema = new IndexSchema(search); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java index 5a8996d4e53..77645331d9e 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.derived; +import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.searchdefinition.RankProfile; @@ -22,19 +23,27 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ * @param search the search definition this is a rank profile from * @param attributeFields the attribute fields to create a ranking for */ - public RankProfileList(Search search, AttributeFields attributeFields, RankProfileRegistry rankProfileRegistry) { + public RankProfileList(Search search, + AttributeFields attributeFields, + RankProfileRegistry rankProfileRegistry, + QueryProfileRegistry queryProfiles) { setName(search.getName()); - deriveRankProfiles(rankProfileRegistry, search, attributeFields); + deriveRankProfiles(rankProfileRegistry, queryProfiles, search, attributeFields); } - private void deriveRankProfiles(RankProfileRegistry rankProfileRegistry, Search search, AttributeFields attributeFields) { - RawRankProfile defaultProfile = new RawRankProfile(rankProfileRegistry.getRankProfile(search, "default"), attributeFields); + private void deriveRankProfiles(RankProfileRegistry rankProfileRegistry, + QueryProfileRegistry queryProfiles, + Search search, + AttributeFields attributeFields) { + RawRankProfile defaultProfile = new RawRankProfile(rankProfileRegistry.getRankProfile(search, "default"), + queryProfiles, + attributeFields); rankProfiles.put(defaultProfile.getName(), defaultProfile); for (RankProfile rank : rankProfileRegistry.localRankProfiles(search)) { if ("default".equals(rank.getName())) continue; - RawRankProfile rawRank = new RawRankProfile(rank, attributeFields); + RawRankProfile rawRank = new RawRankProfile(rank, queryProfiles, attributeFields); rankProfiles.put(rawRank.getName(), rawRank); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java index 3c7b99afefc..ea02f960800 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java @@ -4,6 +4,7 @@ package com.yahoo.searchdefinition.derived; import com.google.common.collect.ImmutableList; import com.yahoo.collections.Pair; import com.yahoo.compress.Compressor; +import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.document.RankType; import com.yahoo.searchdefinition.RankProfile; import com.yahoo.searchlib.rankingexpression.ExpressionFunction; @@ -46,9 +47,9 @@ public class RawRankProfile implements RankProfilesConfig.Producer { /** * Creates a raw rank profile from the given rank profile */ - public RawRankProfile(RankProfile rankProfile, AttributeFields attributeFields) { + public RawRankProfile(RankProfile rankProfile, QueryProfileRegistry queryProfiles, AttributeFields attributeFields) { this.name = rankProfile.getName(); - compressedProperties = compress(removePartFromKeys(new Deriver(rankProfile, attributeFields).derive())); + compressedProperties = compress(removePartFromKeys(new Deriver(rankProfile, queryProfiles, attributeFields).derive())); } private List<Pair<String, String>> removePartFromKeys(Map<String, String> map) { @@ -153,8 +154,8 @@ public class RawRankProfile implements RankProfilesConfig.Producer { /** * Creates a raw rank profile from the given rank profile */ - public Deriver(RankProfile rankProfile, AttributeFields attributeFields) { - this.rankProfile = rankProfile.compile(); + public Deriver(RankProfile rankProfile, QueryProfileRegistry queryProfiles, AttributeFields attributeFields) { + this.rankProfile = rankProfile.compile(queryProfiles); deriveRankingFeatures(this.rankProfile); deriveRankTypeSetting(this.rankProfile, attributeFields); deriveFilterFields(this.rankProfile); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ConstantTensorTransformer.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ConstantTensorTransformer.java index 543ac78fc86..6b9d6c3e3ac 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ConstantTensorTransformer.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ConstantTensorTransformer.java @@ -34,7 +34,7 @@ public class ConstantTensorTransformer extends ExpressionTransformer<RankProfile } private ExpressionNode transformFeature(ReferenceNode node, RankProfileTransformContext context) { - if (!node.getArguments().isEmpty()) { + if ( ! node.getArguments().isEmpty()) { return transformArguments(node, context); } else { return transformConstantReference(node, context); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/RankProfileTransformContext.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/RankProfileTransformContext.java index 7fcd2ed357a..5da5b3dabda 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/RankProfileTransformContext.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/RankProfileTransformContext.java @@ -1,6 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.expressiontransforms; +import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.RankProfile; import com.yahoo.searchlib.rankingexpression.evaluation.Value; import com.yahoo.searchlib.rankingexpression.transform.TransformContext; @@ -15,20 +16,24 @@ import java.util.Map; public class RankProfileTransformContext extends TransformContext { private final RankProfile rankProfile; + private final QueryProfileRegistry queryProfiles; private final Map<String, RankProfile.Macro> inlineMacros; private final Map<String, String> rankPropertiesOutput; public RankProfileTransformContext(RankProfile rankProfile, - Map<String, Value> constants, - Map<String, RankProfile.Macro> inlineMacros, - Map<String, String> rankPropertiesOutput) { + QueryProfileRegistry queryProfiles, + Map<String, Value> constants, + Map<String, RankProfile.Macro> inlineMacros, + Map<String, String> rankPropertiesOutput) { super(constants); this.rankProfile = rankProfile; + this.queryProfiles = queryProfiles; this.inlineMacros = inlineMacros; this.rankPropertiesOutput = rankPropertiesOutput; } public RankProfile rankProfile() { return rankProfile; } + public QueryProfileRegistry queryProfiles() { return queryProfiles; } public Map<String, RankProfile.Macro> inlineMacros() { return inlineMacros; } public Map<String, String> rankPropertiesOutput() { return rankPropertiesOutput; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TensorFlowFeatureConverter.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TensorFlowFeatureConverter.java index da85e9f65ec..2b997aa25f2 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TensorFlowFeatureConverter.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TensorFlowFeatureConverter.java @@ -2,14 +2,19 @@ package com.yahoo.searchdefinition.expressiontransforms; import com.google.common.base.Joiner; +import com.yahoo.collections.Pair; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.io.IOUtils; import com.yahoo.path.Path; +import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.RankProfile; import com.yahoo.searchdefinition.RankingConstant; import com.yahoo.searchlib.rankingexpression.RankingExpression; +import com.yahoo.searchlib.rankingexpression.evaluation.DoubleValue; +import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue; +import com.yahoo.searchlib.rankingexpression.evaluation.Value; import com.yahoo.searchlib.rankingexpression.integration.tensorflow.TensorFlowImporter; import com.yahoo.searchlib.rankingexpression.integration.tensorflow.TensorFlowModel; import com.yahoo.searchlib.rankingexpression.integration.tensorflow.TensorFlowModel.Signature; @@ -24,15 +29,18 @@ import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; import com.yahoo.tensor.serialization.TypedBinaryFormat; +import java.io.BufferedReader; import java.io.File; import java.io.IOException; import java.io.StringReader; import java.io.UncheckedIOException; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.logging.Logger; /** * Replaces instances of the tensorflow(model-path, signature, output) @@ -44,6 +52,8 @@ import java.util.Optional; // TODO: Avoid name conflicts across models for constants public class TensorFlowFeatureConverter extends ExpressionTransformer<RankProfileTransformContext> { + private static final Logger log = Logger.getLogger(TensorFlowFeatureConverter.class.getName()); + private final TensorFlowImporter tensorFlowImporter = new TensorFlowImporter(); /** A cache of imported models indexed by model path. This avoids importing the same model multiple times. */ @@ -68,14 +78,16 @@ public class TensorFlowFeatureConverter extends ExpressionTransformer<RankProfil if (store.hasStoredModel()) return transformFromStoredModel(store, context.rankProfile()); else // not converted yet - access TensorFlow model files - return transformFromTensorFlowModel(store, context.rankProfile()); + return transformFromTensorFlowModel(store, context.rankProfile(), context.queryProfiles()); } catch (IllegalArgumentException | UncheckedIOException e) { throw new IllegalArgumentException("Could not use tensorflow model from " + feature, e); } } - private ExpressionNode transformFromTensorFlowModel(ModelStore store, RankProfile profile) { + private ExpressionNode transformFromTensorFlowModel(ModelStore store, + RankProfile profile, + QueryProfileRegistry queryProfiles) { TensorFlowModel model = importedModels.computeIfAbsent(store.arguments().modelPath(), k -> tensorFlowImporter.importModel(store.tensorFlowModelDir())); @@ -83,19 +95,24 @@ public class TensorFlowFeatureConverter extends ExpressionTransformer<RankProfil Signature signature = chooseSignature(model, store.arguments().signature()); String output = chooseOutput(signature, store.arguments().output()); RankingExpression expression = model.expressions().get(output); - verifyRequiredMacros(expression, model.requiredMacros(), profile); + verifyRequiredMacros(expression, model.requiredMacros(), profile, queryProfiles); store.writeConverted(expression); - model.constants().forEach((k, v) -> transformConstant(store, profile, k, v)); + model.smallConstants().forEach((k, v) -> transformSmallConstant(store, profile, k, v)); + model.largeConstants().forEach((k, v) -> transformLargeConstant(store, profile, k, v)); return expression.getRoot(); } private ExpressionNode transformFromStoredModel(ModelStore store, RankProfile profile) { - for (RankingConstant constant : store.readRankingConstants()) { + for (Pair<String, Tensor> constant : store.readSmallConstants()) + profile.addConstant(constant.getFirst(), asValue(constant.getSecond())); + + for (RankingConstant constant : store.readLargeConstants()) { if ( ! profile.getSearch().getRankingConstants().containsKey(constant.getName())) profile.getSearch().addRankingConstant(constant); } + return store.readConverted().getRoot(); } @@ -152,12 +169,19 @@ public class TensorFlowFeatureConverter extends ExpressionTransformer<RankProfil } } - private void transformConstant(ModelStore store, RankProfile profile, String constantName, Tensor constantValue) { - Path constantPath = store.writeConstant(constantName, constantValue); + private void transformSmallConstant(ModelStore store, RankProfile profile, String constantName, Tensor constantValue) { + store.writeSmallConstant(constantName, constantValue); + profile.addConstant(constantName, asValue(constantValue)); + } + + private void transformLargeConstant(ModelStore store, RankProfile profile, String constantName, Tensor constantValue) { + Path constantPath = store.writeLargeConstant(constantName, constantValue); - if ( ! profile.getSearch().getRankingConstants().containsKey(constantName)) + if ( ! profile.getSearch().getRankingConstants().containsKey(constantName)) { + log.info("Adding constant '" + constantName + "' of type " + constantValue.type()); profile.getSearch().addRankingConstant(new RankingConstant(constantName, constantValue.type(), constantPath.toString())); + } } private String skippedOutputsDescription(TensorFlowModel.Signature signature) { @@ -172,7 +196,7 @@ public class TensorFlowFeatureConverter extends ExpressionTransformer<RankProfil * and return tensors of the types specified in requiredMacros. */ private void verifyRequiredMacros(RankingExpression expression, Map<String, TensorType> requiredMacros, - RankProfile profile) { + RankProfile profile, QueryProfileRegistry queryProfiles) { List<String> macroNames = new ArrayList<>(); addMacroNamesIn(expression.getRoot(), macroNames); for (String macroName : macroNames) { @@ -184,7 +208,7 @@ public class TensorFlowFeatureConverter extends ExpressionTransformer<RankProfil throw new IllegalArgumentException("Model refers Placeholder '" + macroName + "' of type " + requiredType + " but this macro is not present in " + profile); - TensorType actualType = macro.getRankingExpression().getRoot().type(profile.typeContext()); + TensorType actualType = macro.getRankingExpression().getRoot().type(profile.typeContext(queryProfiles)); if ( actualType == null) throw new IllegalArgumentException("Model refers Placeholder '" + macroName + "' of type " + requiredType + @@ -210,6 +234,13 @@ public class TensorFlowFeatureConverter extends ExpressionTransformer<RankProfil } } + private Value asValue(Tensor tensor) { + if (tensor.type().rank() == 0) + return new DoubleValue(tensor.asDouble()); // the backend gets offended by dimensionless tensors + else + return new TensorValue(tensor); + } + /** * Provides read/write access to the correct directories of the application package given by the feature arguments */ @@ -264,13 +295,13 @@ public class TensorFlowFeatureConverter extends ExpressionTransformer<RankProfil } /** - * Reads the information about all the constants stored in the application package + * Reads the information about all the large (aka ranking) constants stored in the application package * (the constant value itself is replicated with file distribution). */ - public List<RankingConstant> readRankingConstants() { + public List<RankingConstant> readLargeConstants() { try { List<RankingConstant> constants = new ArrayList<>(); - for (ApplicationFile constantFile : application.getFile(arguments.rankingConstantsPath()).listFiles()) { + for (ApplicationFile constantFile : application.getFile(arguments.largeConstantsPath()).listFiles()) { String[] parts = IOUtils.readAll(constantFile.createReader()).split(":"); constants.add(new RankingConstant(parts[0], TensorType.fromSpec(parts[1]), parts[2])); } @@ -287,25 +318,63 @@ public class TensorFlowFeatureConverter extends ExpressionTransformer<RankProfil * * @return the path to the stored constant, relative to the application package root */ - public Path writeConstant(String name, Tensor constant) { + public Path writeLargeConstant(String name, Tensor constant) { Path constantsPath = ApplicationPackage.MODELS_GENERATED_DIR.append(arguments.modelPath).append("constants"); // "tbf" ending for "typed binary format" - recognized by the nodes receiving the file: Path constantPath = constantsPath.append(name + ".tbf"); - Path constantPathCorrected = constantPath; - if (application.getFileReference(Path.fromString("")).getAbsolutePath().endsWith(FilesApplicationPackage.preprocessed) - && ! constantPath.elements().contains(FilesApplicationPackage.preprocessed)) { - constantPathCorrected = Path.fromString(FilesApplicationPackage.preprocessed).append(constantPath); - } // Remember the constant in a file we replicate in ZooKeeper - application.getFile(arguments.rankingConstantsPath().append(name + ".constant")) - .writeFile(new StringReader(name + ":" + constant.type() + ":" + constantPathCorrected)); + application.getFile(arguments.largeConstantsPath().append(name + ".constant")) + .writeFile(new StringReader(name + ":" + constant.type() + ":" + correct(constantPath))); // Write content explicitly as a file on the file system as this is distributed using file distribution createIfNeeded(constantsPath); IOUtils.writeFile(application.getFileReference(constantPath), TypedBinaryFormat.encode(constant)); - return constantPathCorrected; + return correct(constantPath); + } + + private List<Pair<String, Tensor>> readSmallConstants() { + try { + ApplicationFile file = application.getFile(arguments.smallConstantsPath()); + if (!file.exists()) return Collections.emptyList(); + + List<Pair<String, Tensor>> constants = new ArrayList<>(); + BufferedReader reader = new BufferedReader(file.createReader()); + String line; + while (null != (line = reader.readLine())) { + String[] parts = line.split("\t"); + String name = parts[0]; + TensorType type = TensorType.fromSpec(parts[1]); + Tensor tensor = Tensor.from(type, parts[2]); + constants.add(new Pair<>(name, tensor)); + } + return constants; + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + /** + * Append this constant to the single file used for small constants distributed as config + */ + public void writeSmallConstant(String name, Tensor constant) { + // Secret file format for remembering constants: + application.getFile(arguments.smallConstantsPath()).appendFile(name + "\t" + + constant.type().toString() + "\t" + + constant.toString() + "\n"); + } + + /** Workaround for being constructed with the .preprocessed dir as root while later being used outside it */ + private Path correct(Path path) { + if (application.getFileReference(Path.fromString("")).getAbsolutePath().endsWith(FilesApplicationPackage.preprocessed) + && ! path.elements().contains(FilesApplicationPackage.preprocessed)) { + return Path.fromString(FilesApplicationPackage.preprocessed).append(path); + } + else { + return path; + } } private void createIfNeeded(Path path) { @@ -343,7 +412,13 @@ public class TensorFlowFeatureConverter extends ExpressionTransformer<RankProfil public Optional<String> signature() { return signature; } public Optional<String> output() { return output; } - public Path rankingConstantsPath() { + /** Path to the small constants file */ + public Path smallConstantsPath() { + return ApplicationPackage.MODELS_GENERATED_DIR.append(modelPath).append("constants.txt"); + } + + /** Path to the large (ranking) constants directory */ + public Path largeConstantsPath() { return ApplicationPackage.MODELS_GENERATED_REPLICATED_DIR.append(modelPath).append("constants"); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TensorTransformer.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TensorTransformer.java index 5255cdaeba1..0334012e8d9 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TensorTransformer.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TensorTransformer.java @@ -183,7 +183,7 @@ public class TensorTransformer extends ExpressionTransformer<RankProfileTransfor } private void addIfConstant(ReferenceNode node, Context context, RankProfile profile) { - if (!node.getName().equals(ConstantTensorTransformer.CONSTANT)) { + if ( ! node.getName().equals(ConstantTensorTransformer.CONSTANT)) { return; } if (node.children().size() != 1) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java index 3faebbfeae3..7cb9fee3449 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java @@ -6,6 +6,7 @@ import com.yahoo.search.query.profile.types.FieldDescription; import com.yahoo.search.query.profile.types.FieldType; import com.yahoo.search.query.profile.types.QueryProfileType; import com.yahoo.search.query.profile.types.TensorFieldType; +import com.yahoo.searchdefinition.FeatureNames; import com.yahoo.searchdefinition.RankProfile; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Search; @@ -17,8 +18,6 @@ import com.yahoo.searchdefinition.processing.Processor; import com.yahoo.vespa.model.container.search.QueryProfiles; import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import java.util.Optional; /** @@ -30,8 +29,6 @@ import java.util.Optional; */ public class RankProfileTypeSettingsProcessor extends Processor { - private static final Pattern queryFeaturePattern = Pattern.compile("query\\((\\w+)\\)$"); - public RankProfileTypeSettingsProcessor(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { super(search, deployLogger, rankProfileRegistry, queryProfiles); } @@ -41,7 +38,6 @@ public class RankProfileTypeSettingsProcessor extends Processor { processAttributeFields(); processImportedFields(); processQueryProfileTypes(); - } private void processAttributeFields() { @@ -54,18 +50,18 @@ public class RankProfileTypeSettingsProcessor extends Processor { } private void processImportedFields() { - Optional<ImportedFields> importedFields = search.importedFields(); - if (importedFields.isPresent()) { - importedFields.get().fields().forEach((fieldName, field) -> processImportedField(field)); - } + Optional<ImportedFields> importedFields = search.importedFields(); + if (importedFields.isPresent()) { + importedFields.get().fields().forEach((fieldName, field) -> processImportedField(field)); + } } private void processImportedField(ImportedField field) { - SDField targetField = field.targetField(); - Attribute attribute = targetField.getAttributes().get(targetField.getName()); - if (attribute != null && attribute.tensorType().isPresent()) { - addAttributeTypeToRankProfiles(field.fieldName(), attribute.tensorType().get().toString()); - } + SDField targetField = field.targetField(); + Attribute attribute = targetField.getAttributes().get(targetField.getName()); + if (attribute != null && attribute.tensorType().isPresent()) { + addAttributeTypeToRankProfiles(field.fieldName(), attribute.tensorType().get().toString()); + } } private void addAttributeTypeToRankProfiles(String attributeName, String attributeType) { @@ -87,11 +83,8 @@ public class RankProfileTypeSettingsProcessor extends Processor { FieldType fieldType = fieldDescription.getType(); if (fieldType instanceof TensorFieldType) { TensorFieldType tensorFieldType = (TensorFieldType)fieldType; - Matcher matcher = queryFeaturePattern.matcher(fieldName); - if (matcher.matches()) { - String queryFeature = matcher.group(1); - addQueryFeatureTypeToRankProfiles(queryFeature, tensorFieldType.type().toString()); - } + FeatureNames.argumentOf(fieldName).ifPresent(argument -> + addQueryFeatureTypeToRankProfiles(argument, tensorFieldType.asTensorType().toString())); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SearchManager.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SearchManager.java index b7cfad6c052..296e3952454 100644 --- a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SearchManager.java +++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SearchManager.java @@ -5,9 +5,9 @@ import java.util.TreeMap; /** * @author baldersheim - * @since 2010-02-19 */ public class SearchManager { + /// This is the list of all known search definitions private TreeMap<String, SearchDef> defs = new TreeMap<>(); @@ -24,4 +24,5 @@ public class SearchManager { defs.put(def.getName(), def); return this; } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java index ee9cb2d564a..721e1c08989 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java @@ -36,9 +36,9 @@ import java.util.logging.Logger; * by running through the binary 'vespa-verify-ranksetup' * * @author vegardh - * */ public class RankSetupValidator extends Validator { + private static final Logger log = Logger.getLogger(RankSetupValidator.class.getName()); private final boolean force; @@ -60,7 +60,7 @@ public class RankSetupValidator extends Validator { final String name = docDb.getDerivedConfiguration().getSearch().getName(); String searchDir = clusterDir + name + "/"; writeConfigs(searchDir, docDb); - if (!validate("dir:" + searchDir, sc, name, deployState.getDeployLogger(), cfgDir)) { + if ( ! validate("dir:" + searchDir, sc, name, deployState.getDeployLogger(), cfgDir)) { return; } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java index 4925b88e608..c3069f699d2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java @@ -281,11 +281,15 @@ public abstract class IndexedSearchCluster extends SearchCluster } } protected void deriveAllSearchDefinitions(List<SearchDefinitionSpec> localSearches, - List<com.yahoo.searchdefinition.Search> globalSearches) { + List<com.yahoo.searchdefinition.Search> globalSearches) { for (SearchDefinitionSpec spec : localSearches) { com.yahoo.searchdefinition.Search search = spec.getSearchDefinition().getSearch(); - if (!(search instanceof UnproperSearch)) { - DocumentDatabase db = new DocumentDatabase(this, search.getName(), new DerivedConfiguration(search, globalSearches, deployLogger(), getRoot().getDeployState().rankProfileRegistry())); + if ( ! (search instanceof UnproperSearch)) { + DocumentDatabase db = new DocumentDatabase(this, + search.getName(), + new DerivedConfiguration(search, globalSearches, deployLogger(), + getRoot().getDeployState().rankProfileRegistry(), + getRoot().getDeployState().getQueryProfiles().getRegistry())); // TODO: remove explicit adding of user configs when the complete content model is built using builders. db.mergeUserConfigs(spec.getUserConfigs()); documentDbs.add(db); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java index 632e33b85b0..85f98c27365 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java @@ -92,7 +92,9 @@ public class StreamingSearchCluster extends SearchCluster implements } private void deriveSingleSearchDefinition(com.yahoo.searchdefinition.Search localSearch, List<com.yahoo.searchdefinition.Search> globalSearches) { - this.sdConfig = new DerivedConfiguration(localSearch, globalSearches, deployLogger(), getRoot().getDeployState().rankProfileRegistry()); + this.sdConfig = new DerivedConfiguration(localSearch, globalSearches, deployLogger(), + getRoot().getDeployState().rankProfileRegistry(), + getRoot().getDeployState().getQueryProfiles().getRegistry()); } @Override public DerivedConfiguration getSdConfig() { diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj index bf6376983a4..86b136a1dd2 100644 --- a/config-model/src/main/javacc/SDParser.jj +++ b/config-model/src/main/javacc/SDParser.jj @@ -65,24 +65,29 @@ import org.apache.commons.lang.StringUtils; /** * A search definition parser * - * @author <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a> et.al. + * @author bratseth */ public class SDParser { + private DocumentTypeManager docMan = null; - private ApplicationPackage app = MockApplicationPackage.createEmpty(); - private DeployLogger deployLogger = new BaseDeployLogger(); - private RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + private ApplicationPackage app; + private DeployLogger deployLogger; + private RankProfileRegistry rankProfileRegistry; + /** For testing only */ public SDParser(String input, DeployLogger deployLogger) { this(new SimpleCharStream(input), deployLogger); } + /** For testing only */ public SDParser(SimpleCharStream stream, DeployLogger deployLogger) { - this(stream); - this.deployLogger = deployLogger; + this(stream, deployLogger, MockApplicationPackage.createEmpty(), new RankProfileRegistry()); } - public SDParser(SimpleCharStream stream, DeployLogger deployLogger, ApplicationPackage applicationPackage, RankProfileRegistry rankProfileRegistry) { + public SDParser(SimpleCharStream stream, + DeployLogger deployLogger, + ApplicationPackage applicationPackage, + RankProfileRegistry rankProfileRegistry) { this(stream); this.deployLogger = deployLogger; this.app = applicationPackage; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/FeatureNamesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/FeatureNamesTestCase.java new file mode 100644 index 00000000000..1f60ad870ec --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/FeatureNamesTestCase.java @@ -0,0 +1,58 @@ +/* + * // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + * + * + */ +package com.yahoo.searchdefinition; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +/** + * Tests rank feature names. + * + * @author bratseth + */ +public class FeatureNamesTestCase { + + @Test + public void testCanonicalization() { + assertFalse(FeatureNames.canonicalizeIfValid("foo").isPresent()); + assertEquals("query(bar)", FeatureNames.canonicalize("query(bar)")); + assertEquals("query(bar)", FeatureNames.canonicalize("query('bar')")); + assertEquals("constant(bar)", FeatureNames.canonicalize("constant(\"bar\")")); + assertEquals("query(\"ba.r\")", FeatureNames.canonicalize("query(ba.r)")); + assertEquals("query(\"ba.r\")", FeatureNames.canonicalize("query('ba.r')")); + assertEquals("attribute(\"ba.r\")", FeatureNames.canonicalize("attribute(\"ba.r\")")); + } + + @Test + public void testArgument() { + assertFalse(FeatureNames.argumentOf("foo(bar)").isPresent()); + assertFalse(FeatureNames.argumentOf("foo(bar.baz)").isPresent()); + assertEquals("bar", FeatureNames.argumentOf("query(bar)").get()); + assertEquals("bar.baz", FeatureNames.argumentOf("query(bar.baz)").get()); + assertEquals("bar", FeatureNames.argumentOf("attribute(bar)").get()); + assertEquals("bar.baz", FeatureNames.argumentOf("attribute(bar.baz)").get()); + assertEquals("bar", FeatureNames.argumentOf("constant(bar)").get()); + assertEquals("bar.baz", FeatureNames.argumentOf("constant(bar.baz)").get()); + } + + @Test + public void testConstantFeature() { + assertEquals("constant(\"foo/bar\")", FeatureNames.asConstantFeature("foo/bar")); + } + + @Test + public void testAttributeFeature() { + assertEquals("attribute(foo)", FeatureNames.asAttributeFeature("foo")); + } + + @Test + public void testQueryFeature() { + assertEquals("query(\"foo.bar\")", FeatureNames.asQueryFeature("foo.bar")); + } + +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectRankingExpressionFileRefTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectRankingExpressionFileRefTestCase.java index 27bab31d709..bff34411d44 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectRankingExpressionFileRefTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectRankingExpressionFileRefTestCase.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition; +import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.derived.DerivedConfiguration; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; @@ -19,8 +20,10 @@ public class IncorrectRankingExpressionFileRefTestCase extends SearchDefinitionT public void testIncorrectRef() throws IOException, ParseException { try { RankProfileRegistry registry = new RankProfileRegistry(); - Search search = SearchBuilder.buildFromFile("src/test/examples/incorrectrankingexpressionfileref.sd", registry); - new DerivedConfiguration(search, registry); // rank profile parsing happens during deriving + Search search = SearchBuilder.buildFromFile("src/test/examples/incorrectrankingexpressionfileref.sd", + registry, + new QueryProfileRegistry()); + new DerivedConfiguration(search, registry, new QueryProfileRegistry()); // cause rank profile parsing fail("parsing should have failed"); } catch (IllegalArgumentException e) { e.printStackTrace(); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java index 960a3b7d6db..442c8bd41bd 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java @@ -33,6 +33,7 @@ import static org.junit.Assert.assertTrue; * @author bratseth */ public class RankProfileTestCase extends SearchDefinitionTestCase { + @Test public void testRankProfileInheritance() { Search search = new Search("test", null); @@ -90,7 +91,7 @@ public class RankProfileTestCase extends SearchDefinitionTestCase { assertEquals(8, rankProfile.getNumThreadsPerSearch()); assertEquals(70, rankProfile.getMinHitsPerThread()); assertEquals(1200, rankProfile.getNumSearchPartitions()); - RawRankProfile rawRankProfile = new RawRankProfile(rankProfile, attributeFields); + RawRankProfile rawRankProfile = new RawRankProfile(rankProfile, new QueryProfileRegistry(), attributeFields); assertTrue(findProperty(rawRankProfile.configProperties(), "vespa.matching.termwise_limit").isPresent()); assertEquals("0.78", findProperty(rawRankProfile.configProperties(), "vespa.matching.termwise_limit").get()); assertTrue(findProperty(rawRankProfile.configProperties(), "vespa.matching.numthreadspersearch").isPresent()); @@ -125,7 +126,7 @@ public class RankProfileTestCase extends SearchDefinitionTestCase { } private static void assertAttributeTypeSettings(RankProfile profile, Search search) { - RawRankProfile rawProfile = new RawRankProfile(profile, new AttributeFields(search)); + RawRankProfile rawProfile = new RawRankProfile(profile, new QueryProfileRegistry(), new AttributeFields(search)); assertEquals("tensor(x[10])", findProperty(rawProfile.configProperties(), "vespa.type.attribute.a").get()); assertEquals("tensor(y{})", findProperty(rawProfile.configProperties(), "vespa.type.attribute.b").get()); assertEquals("tensor(x[])", findProperty(rawProfile.configProperties(), "vespa.type.attribute.c").get()); @@ -167,7 +168,7 @@ public class RankProfileTestCase extends SearchDefinitionTestCase { } private static void assertQueryFeatureTypeSettings(RankProfile profile, Search search) { - RawRankProfile rawProfile = new RawRankProfile(profile, new AttributeFields(search)); + RawRankProfile rawProfile = new RawRankProfile(profile, new QueryProfileRegistry(), new AttributeFields(search)); assertEquals("tensor(x[10])", findProperty(rawProfile.configProperties(), "vespa.type.query.tensor1").get()); assertEquals("tensor(y{})", findProperty(rawProfile.configProperties(), "vespa.type.query.tensor2").get()); assertFalse(findProperty(rawProfile.configProperties(), "vespa.type.query.tensor3").isPresent()); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankPropertiesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankPropertiesTestCase.java index c42336b300b..15ddef60807 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankPropertiesTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankPropertiesTestCase.java @@ -3,6 +3,7 @@ package com.yahoo.searchdefinition; import com.yahoo.collections.Pair; import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.derived.AttributeFields; import com.yahoo.searchdefinition.derived.RawRankProfile; import com.yahoo.searchdefinition.parser.ParseException; @@ -59,7 +60,7 @@ public class RankPropertiesTestCase extends SearchDefinitionTestCase { assertEquals("query(a) = 1500", parent.getRankProperties().get(0).toString()); // Check derived model - RawRankProfile rawParent = new RawRankProfile(parent, attributeFields); + RawRankProfile rawParent = new RawRankProfile(parent, new QueryProfileRegistry(), attributeFields); assertEquals("(query(a),1500)", rawParent.configProperties().get(0).toString()); } @@ -69,7 +70,9 @@ public class RankPropertiesTestCase extends SearchDefinitionTestCase { assertEquals("query(a) = 2000", parent.getRankProperties().get(0).toString()); // Check derived model - RawRankProfile rawChild = new RawRankProfile(rankProfileRegistry.getRankProfile(search, "child"), attributeFields); + RawRankProfile rawChild = new RawRankProfile(rankProfileRegistry.getRankProfile(search, "child"), + new QueryProfileRegistry(), + attributeFields); assertEquals("(query(a),2000)", rawChild.configProperties().get(0).toString()); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java index 1f4f18b5a47..e94880e61c7 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java @@ -3,6 +3,7 @@ package com.yahoo.searchdefinition; import com.yahoo.collections.Pair; import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.yolean.Exceptions; import com.yahoo.searchdefinition.derived.AttributeFields; import com.yahoo.searchdefinition.derived.RawRankProfile; @@ -24,6 +25,7 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase @Test public void testConstants() throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + QueryProfileRegistry queryProfileRegistry = new QueryProfileRegistry(); SearchBuilder builder = new SearchBuilder(rankProfileRegistry); builder.importString( "search test {\n" + @@ -67,17 +69,19 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase "}\n"); builder.build(); Search s = builder.getSearch(); - RankProfile parent = rankProfileRegistry.getRankProfile(s, "parent").compile(); + RankProfile parent = rankProfileRegistry.getRankProfile(s, "parent").compile(queryProfileRegistry); assertEquals("0.0", parent.getFirstPhaseRanking().getRoot().toString()); - RankProfile child1 = rankProfileRegistry.getRankProfile(s, "child1").compile(); + RankProfile child1 = rankProfileRegistry.getRankProfile(s, "child1").compile(queryProfileRegistry); assertEquals("6.5", child1.getFirstPhaseRanking().getRoot().toString()); assertEquals("11.5", child1.getSecondPhaseRanking().getRoot().toString()); - RankProfile child2 = rankProfileRegistry.getRankProfile(s, "child2").compile(); + RankProfile child2 = rankProfileRegistry.getRankProfile(s, "child2").compile(queryProfileRegistry); assertEquals("16.6", child2.getFirstPhaseRanking().getRoot().toString()); assertEquals("foo: 14.0", child2.getMacros().get("foo").getRankingExpression().toString()); - List<Pair<String, String>> rankProperties = new RawRankProfile(child2, new AttributeFields(s)).configProperties(); + List<Pair<String, String>> rankProperties = new RawRankProfile(child2, + queryProfileRegistry, + new AttributeFields(s)).configProperties(); assertEquals("(rankingExpression(foo).rankingScript,14.0)", rankProperties.get(0).toString()); assertEquals("(rankingExpression(firstphase).rankingScript,16.6)", rankProperties.get(2).toString()); } @@ -107,7 +111,7 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase builder.build(); Search s = builder.getSearch(); try { - rankProfileRegistry.getRankProfile(s, "test").compile(); + rankProfileRegistry.getRankProfile(s, "test").compile(new QueryProfileRegistry()); fail("Should have caused an exception"); } catch (IllegalArgumentException e) { @@ -169,7 +173,8 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase RankProfile profile = rankProfileRegistry.getRankProfile(s, "test"); profile.parseExpressions(); // TODO: Do differently assertEquals("safeLog(popShareSlowDecaySignal,myValue)", profile.getMacros().get("POP_SLOW_SCORE").getRankingExpression().getRoot().toString()); - assertEquals("safeLog(popShareSlowDecaySignal,-9.21034037)", profile.compile().getMacros().get("POP_SLOW_SCORE").getRankingExpression().getRoot().toString()); + assertEquals("safeLog(popShareSlowDecaySignal,-9.21034037)", + profile.compile(new QueryProfileRegistry()).getMacros().get("POP_SLOW_SCORE").getRankingExpression().getRoot().toString()); } @Test @@ -191,7 +196,8 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase builder.build(); Search s = builder.getSearch(); RankProfile profile = rankProfileRegistry.getRankProfile(s, "test"); - assertEquals("k1 + (k2 + k3) / 100000000.0", profile.compile().getMacros().get("rank_default").getRankingExpression().getRoot().toString()); + assertEquals("k1 + (k2 + k3) / 100000000.0", + profile.compile(new QueryProfileRegistry()).getMacros().get("rank_default").getRankingExpression().getRoot().toString()); } @Test @@ -213,7 +219,8 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase builder.build(); Search s = builder.getSearch(); RankProfile profile = rankProfileRegistry.getRankProfile(s, "test"); - assertEquals("0.5 + 50 * (attribute(rating_yelp) - 3)", profile.compile().getMacros().get("rank_default").getRankingExpression().getRoot().toString()); + assertEquals("0.5 + 50 * (attribute(rating_yelp) - 3)", + profile.compile(new QueryProfileRegistry()).getMacros().get("rank_default").getRankingExpression().getRoot().toString()); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionInliningTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionInliningTestCase.java index a36ac63b2b5..97e1ab9aeb9 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionInliningTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionInliningTestCase.java @@ -2,6 +2,7 @@ package com.yahoo.searchdefinition; import com.yahoo.collections.Pair; +import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.derived.AttributeFields; import com.yahoo.searchdefinition.derived.RawRankProfile; import com.yahoo.searchdefinition.parser.ParseException; @@ -59,10 +60,12 @@ public class RankingExpressionInliningTestCase extends SearchDefinitionTestCase builder.build(); Search s = builder.getSearch(); - RankProfile parent = rankProfileRegistry.getRankProfile(s, "parent").compile(); - assertEquals("7.0 * (3 + attribute(a) + attribute(b) * (attribute(a) * 3 + if (7.0 < attribute(a), 1, 2) == 0))", parent.getFirstPhaseRanking().getRoot().toString()); - RankProfile child = rankProfileRegistry.getRankProfile(s, "child").compile(); - assertEquals("7.0 * (9 + attribute(a))", child.getFirstPhaseRanking().getRoot().toString()); + RankProfile parent = rankProfileRegistry.getRankProfile(s, "parent").compile(new QueryProfileRegistry()); + assertEquals("7.0 * (3 + attribute(a) + attribute(b) * (attribute(a) * 3 + if (7.0 < attribute(a), 1, 2) == 0))", + parent.getFirstPhaseRanking().getRoot().toString()); + RankProfile child = rankProfileRegistry.getRankProfile(s, "child").compile(new QueryProfileRegistry()); + assertEquals("7.0 * (9 + attribute(a))", + child.getFirstPhaseRanking().getRoot().toString()); } @Test public void testConstants() throws ParseException { @@ -116,26 +119,39 @@ public class RankingExpressionInliningTestCase extends SearchDefinitionTestCase builder.build(); Search s = builder.getSearch(); - RankProfile parent = rankProfileRegistry.getRankProfile(s, "parent").compile(); + RankProfile parent = rankProfileRegistry.getRankProfile(s, "parent").compile(new QueryProfileRegistry()); assertEquals("17.0", parent.getFirstPhaseRanking().getRoot().toString()); assertEquals("0.0", parent.getSecondPhaseRanking().getRoot().toString()); - List<Pair<String, String>> parentRankProperties = new RawRankProfile(parent, new AttributeFields(s)).configProperties(); - assertEquals("(rankingExpression(foo).rankingScript,10.0)", parentRankProperties.get(0).toString()); - assertEquals("(rankingExpression(firstphase).rankingScript,17.0)", parentRankProperties.get(2).toString()); - assertEquals("(rankingExpression(secondphase).rankingScript,0.0)", parentRankProperties.get(4).toString()); + List<Pair<String, String>> parentRankProperties = new RawRankProfile(parent, + new QueryProfileRegistry(), + new AttributeFields(s)).configProperties(); + assertEquals("(rankingExpression(foo).rankingScript,10.0)", + parentRankProperties.get(0).toString()); + assertEquals("(rankingExpression(firstphase).rankingScript,17.0)", + parentRankProperties.get(2).toString()); + assertEquals("(rankingExpression(secondphase).rankingScript,0.0)", + parentRankProperties.get(4).toString()); - RankProfile child = rankProfileRegistry.getRankProfile(s, "child").compile(); + RankProfile child = rankProfileRegistry.getRankProfile(s, "child").compile(new QueryProfileRegistry()); assertEquals("31.0 + bar + arg(4.0)", child.getFirstPhaseRanking().getRoot().toString()); assertEquals("24.0", child.getSecondPhaseRanking().getRoot().toString()); - List<Pair<String, String>> childRankProperties = new RawRankProfile(child, new AttributeFields(s)).configProperties(); - for (Object o : childRankProperties) System.out.println(o); - assertEquals("(rankingExpression(foo).rankingScript,12.0)", childRankProperties.get(0).toString()); - assertEquals("(rankingExpression(bar).rankingScript,14.0)", childRankProperties.get(1).toString()); - assertEquals("(rankingExpression(boz).rankingScript,3.0)", childRankProperties.get(2).toString()); - assertEquals("(rankingExpression(baz).rankingScript,9.0 + rankingExpression(boz))", childRankProperties.get(3).toString()); - assertEquals("(rankingExpression(arg).rankingScript,a1 * 2)", childRankProperties.get(4).toString()); - assertEquals("(rankingExpression(firstphase).rankingScript,31.0 + rankingExpression(bar) + rankingExpression(arg@))", censorBindingHash(childRankProperties.get(7).toString())); - assertEquals("(rankingExpression(secondphase).rankingScript,24.0)", childRankProperties.get(9).toString()); + List<Pair<String, String>> childRankProperties = new RawRankProfile(child, + new QueryProfileRegistry(), + new AttributeFields(s)).configProperties(); + assertEquals("(rankingExpression(foo).rankingScript,12.0)", + childRankProperties.get(0).toString()); + assertEquals("(rankingExpression(bar).rankingScript,14.0)", + childRankProperties.get(1).toString()); + assertEquals("(rankingExpression(boz).rankingScript,3.0)", + childRankProperties.get(2).toString()); + assertEquals("(rankingExpression(baz).rankingScript,9.0 + rankingExpression(boz))", + childRankProperties.get(3).toString()); + assertEquals("(rankingExpression(arg).rankingScript,a1 * 2)", + childRankProperties.get(4).toString()); + assertEquals("(rankingExpression(firstphase).rankingScript,31.0 + rankingExpression(bar) + rankingExpression(arg@))", + censorBindingHash(childRankProperties.get(7).toString())); + assertEquals("(rankingExpression(secondphase).rankingScript,24.0)", + childRankProperties.get(9).toString()); } /** diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java index 92896429f26..5100ac15c40 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java @@ -2,6 +2,7 @@ package com.yahoo.searchdefinition; import com.yahoo.collections.Pair; +import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.derived.AttributeFields; import com.yahoo.searchdefinition.derived.RawRankProfile; import com.yahoo.searchdefinition.parser.ParseException; @@ -40,13 +41,16 @@ public class RankingExpressionShadowingTestCase extends SearchDefinitionTestCase "}\n"); builder.build(); Search s = builder.getSearch(); - RankProfile test = rankProfileRegistry.getRankProfile(s, "test").compile(); - List<Pair<String, String>> testRankProperties = new RawRankProfile(test, new AttributeFields(s)).configProperties(); - for (Object o : testRankProperties) - System.out.println(o); - assertEquals("(rankingExpression(sin).rankingScript,x * x)", testRankProperties.get(0).toString()); - assertEquals("(rankingExpression(sin@).rankingScript,2 * 2)", censorBindingHash(testRankProperties.get(1).toString())); - assertEquals("(vespa.rank.firstphase,rankingExpression(sin@))", censorBindingHash(testRankProperties.get(2).toString())); + RankProfile test = rankProfileRegistry.getRankProfile(s, "test").compile(new QueryProfileRegistry()); + List<Pair<String, String>> testRankProperties = new RawRankProfile(test, + new QueryProfileRegistry(), + new AttributeFields(s)).configProperties(); + assertEquals("(rankingExpression(sin).rankingScript,x * x)", + testRankProperties.get(0).toString()); + assertEquals("(rankingExpression(sin@).rankingScript,2 * 2)", + censorBindingHash(testRankProperties.get(1).toString())); + assertEquals("(vespa.rank.firstphase,rankingExpression(sin@))", + censorBindingHash(testRankProperties.get(2).toString())); } @@ -80,19 +84,28 @@ public class RankingExpressionShadowingTestCase extends SearchDefinitionTestCase "}\n"); builder.build(); Search s = builder.getSearch(); - RankProfile test = rankProfileRegistry.getRankProfile(s, "test").compile(); - List<Pair<String, String>> testRankProperties = new RawRankProfile(test, new AttributeFields(s)).configProperties(); - for (Object o : testRankProperties) - System.out.println(o); - assertEquals("(rankingExpression(tan).rankingScript,x * x)", testRankProperties.get(0).toString()); - assertEquals("(rankingExpression(tan@).rankingScript,x * x)", censorBindingHash(testRankProperties.get(1).toString())); - assertEquals("(rankingExpression(cos).rankingScript,rankingExpression(tan@))", censorBindingHash(testRankProperties.get(2).toString())); - assertEquals("(rankingExpression(cos@).rankingScript,rankingExpression(tan@))", censorBindingHash(testRankProperties.get(3).toString())); - assertEquals("(rankingExpression(sin).rankingScript,rankingExpression(cos@))", censorBindingHash(testRankProperties.get(4).toString())); - assertEquals("(rankingExpression(tan@).rankingScript,2 * 2)", censorBindingHash(testRankProperties.get(5).toString())); - assertEquals("(rankingExpression(cos@).rankingScript,rankingExpression(tan@))", censorBindingHash(testRankProperties.get(6).toString())); - assertEquals("(rankingExpression(sin@).rankingScript,rankingExpression(cos@))", censorBindingHash(testRankProperties.get(7).toString())); - assertEquals("(vespa.rank.firstphase,rankingExpression(sin@))", censorBindingHash(testRankProperties.get(8).toString())); + RankProfile test = rankProfileRegistry.getRankProfile(s, "test").compile(new QueryProfileRegistry()); + List<Pair<String, String>> testRankProperties = new RawRankProfile(test, + new QueryProfileRegistry(), + new AttributeFields(s)).configProperties(); + assertEquals("(rankingExpression(tan).rankingScript,x * x)", + testRankProperties.get(0).toString()); + assertEquals("(rankingExpression(tan@).rankingScript,x * x)", + censorBindingHash(testRankProperties.get(1).toString())); + assertEquals("(rankingExpression(cos).rankingScript,rankingExpression(tan@))", + censorBindingHash(testRankProperties.get(2).toString())); + assertEquals("(rankingExpression(cos@).rankingScript,rankingExpression(tan@))", + censorBindingHash(testRankProperties.get(3).toString())); + assertEquals("(rankingExpression(sin).rankingScript,rankingExpression(cos@))", + censorBindingHash(testRankProperties.get(4).toString())); + assertEquals("(rankingExpression(tan@).rankingScript,2 * 2)", + censorBindingHash(testRankProperties.get(5).toString())); + assertEquals("(rankingExpression(cos@).rankingScript,rankingExpression(tan@))", + censorBindingHash(testRankProperties.get(6).toString())); + assertEquals("(rankingExpression(sin@).rankingScript,rankingExpression(cos@))", + censorBindingHash(testRankProperties.get(7).toString())); + assertEquals("(vespa.rank.firstphase,rankingExpression(sin@))", + censorBindingHash(testRankProperties.get(8).toString())); } @@ -120,15 +133,20 @@ public class RankingExpressionShadowingTestCase extends SearchDefinitionTestCase "}\n"); builder.build(); Search s = builder.getSearch(); - RankProfile test = rankProfileRegistry.getRankProfile(s, "test").compile(); - List<Pair<String, String>> testRankProperties = new RawRankProfile(test, new AttributeFields(s)).configProperties(); - for (Object o : testRankProperties) - System.out.println(o); - assertEquals("(rankingExpression(sin).rankingScript,x * x)", testRankProperties.get(0).toString()); - assertEquals("(rankingExpression(sin@).rankingScript,4.0 * 4.0)", censorBindingHash(testRankProperties.get(1).toString())); - assertEquals("(rankingExpression(sin@).rankingScript,cos(5.0) * cos(5.0))", censorBindingHash(testRankProperties.get(2).toString())); - assertEquals("(vespa.rank.firstphase,rankingExpression(firstphase))", censorBindingHash(testRankProperties.get(3).toString())); - assertEquals("(rankingExpression(firstphase).rankingScript,cos(rankingExpression(sin@)) + rankingExpression(sin@))", censorBindingHash(testRankProperties.get(4).toString())); + RankProfile test = rankProfileRegistry.getRankProfile(s, "test").compile(new QueryProfileRegistry()); + List<Pair<String, String>> testRankProperties = new RawRankProfile(test, + new QueryProfileRegistry(), + new AttributeFields(s)).configProperties(); + assertEquals("(rankingExpression(sin).rankingScript,x * x)", + testRankProperties.get(0).toString()); + assertEquals("(rankingExpression(sin@).rankingScript,4.0 * 4.0)", + censorBindingHash(testRankProperties.get(1).toString())); + assertEquals("(rankingExpression(sin@).rankingScript,cos(5.0) * cos(5.0))", + censorBindingHash(testRankProperties.get(2).toString())); + assertEquals("(vespa.rank.firstphase,rankingExpression(firstphase))", + censorBindingHash(testRankProperties.get(3).toString())); + assertEquals("(rankingExpression(firstphase).rankingScript,cos(rankingExpression(sin@)) + rankingExpression(sin@))", + censorBindingHash(testRankProperties.get(4).toString())); } @@ -162,16 +180,22 @@ public class RankingExpressionShadowingTestCase extends SearchDefinitionTestCase "}\n"); builder.build(); Search s = builder.getSearch(); - RankProfile test = rankProfileRegistry.getRankProfile(s, "test").compile(); - List<Pair<String, String>> testRankProperties = new RawRankProfile(test, new AttributeFields(s)).configProperties(); - for (Object o : testRankProperties) - System.out.println(o); - assertEquals("(rankingExpression(relu).rankingScript,max(1.0,x))", testRankProperties.get(0).toString()); - assertEquals("(rankingExpression(relu@).rankingScript,max(1.0,reduce(query(q) * constant(W_hidden), sum, input) + constant(b_input)))", censorBindingHash(testRankProperties.get(1).toString())); - assertEquals("(rankingExpression(hidden_layer).rankingScript,rankingExpression(relu@))", censorBindingHash(testRankProperties.get(2).toString())); - assertEquals("(rankingExpression(final_layer).rankingScript,sigmoid(reduce(rankingExpression(hidden_layer) * constant(W_final), sum, hidden) + constant(b_final)))", testRankProperties.get(3).toString()); - assertEquals("(vespa.rank.secondphase,rankingExpression(secondphase))", testRankProperties.get(4).toString()); - assertEquals("(rankingExpression(secondphase).rankingScript,reduce(rankingExpression(final_layer), sum))", testRankProperties.get(5).toString()); + RankProfile test = rankProfileRegistry.getRankProfile(s, "test").compile(new QueryProfileRegistry()); + List<Pair<String, String>> testRankProperties = new RawRankProfile(test, + new QueryProfileRegistry(), + new AttributeFields(s)).configProperties(); + assertEquals("(rankingExpression(relu).rankingScript,max(1.0,x))", + testRankProperties.get(0).toString()); + assertEquals("(rankingExpression(relu@).rankingScript,max(1.0,reduce(query(q) * constant(W_hidden), sum, input) + constant(b_input)))", + censorBindingHash(testRankProperties.get(1).toString())); + assertEquals("(rankingExpression(hidden_layer).rankingScript,rankingExpression(relu@))", + censorBindingHash(testRankProperties.get(2).toString())); + assertEquals("(rankingExpression(final_layer).rankingScript,sigmoid(reduce(rankingExpression(hidden_layer) * constant(W_final), sum, hidden) + constant(b_final)))", + testRankProperties.get(3).toString()); + assertEquals("(vespa.rank.secondphase,rankingExpression(secondphase))", + testRankProperties.get(4).toString()); + assertEquals("(rankingExpression(secondphase).rankingScript,reduce(rankingExpression(final_layer), sum))", + testRankProperties.get(5).toString()); } private String censorBindingHash(String s) { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java index b652bcd6ce6..a07fea69592 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition; +import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.derived.DerivedConfiguration; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Ignore; @@ -23,7 +24,7 @@ public class RankingExpressionValidationTestCase extends SearchDefinitionTestCas try { RankProfileRegistry registry = new RankProfileRegistry(); Search search = importWithExpression(expression, registry); - new DerivedConfiguration(search, registry); // rank profile parsing happens during deriving + new DerivedConfiguration(search, registry, new QueryProfileRegistry()); // cause rank profile parsing fail("No exception on incorrect ranking expression " + expression); } catch (IllegalArgumentException e) { // Success diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/SearchImporterTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/SearchImporterTestCase.java index f9b62799171..d1a5bbad217 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/SearchImporterTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/SearchImporterTestCase.java @@ -4,6 +4,7 @@ package com.yahoo.searchdefinition; import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.document.DataType; import com.yahoo.document.Document; +import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.document.*; import com.yahoo.searchdefinition.parser.ParseException; import com.yahoo.searchdefinition.processing.MakeAliases; @@ -26,7 +27,7 @@ public class SearchImporterTestCase extends SearchDefinitionTestCase { @Test public void testSimpleImporting() throws IOException, ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SearchBuilder sb = new UnprocessingSearchBuilder(rankProfileRegistry); + SearchBuilder sb = new UnprocessingSearchBuilder(rankProfileRegistry, new QueryProfileRegistry()); sb.importFile("src/test/examples/simple.sd"); sb.build(); Search search = sb.getSearch(); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java index 84edb50aed3..3d2bce62713 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java @@ -78,12 +78,16 @@ public abstract class AbstractExportingTestCase extends SearchDefinitionTestCase } protected DerivedConfiguration derive(String dirName, String searchDefinitionName, SearchBuilder builder) throws IOException { - DerivedConfiguration config = new DerivedConfiguration(builder.getSearch(searchDefinitionName), builder.getRankProfileRegistry()); + DerivedConfiguration config = new DerivedConfiguration(builder.getSearch(searchDefinitionName), + builder.getRankProfileRegistry(), + builder.getQueryProfileRegistry()); return export(dirName, builder, config); } protected DerivedConfiguration derive(String dirName, SearchBuilder builder, Search search) throws IOException { - DerivedConfiguration config = new DerivedConfiguration(search, builder.getRankProfileRegistry()); + DerivedConfiguration config = new DerivedConfiguration(search, + builder.getRankProfileRegistry(), + builder.getQueryProfileRegistry()); return export(dirName, builder, config); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/EmptyRankProfileTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/EmptyRankProfileTestCase.java index 4e9a802841a..21467776ad9 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/EmptyRankProfileTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/EmptyRankProfileTestCase.java @@ -2,6 +2,7 @@ package com.yahoo.searchdefinition.derived; import com.yahoo.document.DataType; +import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.SearchBuilder; @@ -21,7 +22,7 @@ import java.io.IOException; public class EmptyRankProfileTestCase extends SearchDefinitionTestCase { @Test - public void testDeriving() throws IOException, ParseException { + public void testDeriving() { Search search = new Search("test", null); RankProfileRegistry rankProfileRegistry = RankProfileRegistry.createRankProfileRegistryWithBuiltinRankProfiles(search); SDDocumentType doc = new SDDocumentType("test"); @@ -32,7 +33,8 @@ public class EmptyRankProfileTestCase extends SearchDefinitionTestCase { doc.addField(field); doc.addField(new SDField("c", DataType.STRING)); - search = SearchBuilder.buildFromRawSearch(search, rankProfileRegistry); - new DerivedConfiguration(search, rankProfileRegistry); + search = SearchBuilder.buildFromRawSearch(search, rankProfileRegistry, new QueryProfileRegistry()); + new DerivedConfiguration(search, rankProfileRegistry, new QueryProfileRegistry()); } + } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/LiteralBoostTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/LiteralBoostTestCase.java index 2cb7c0e86b9..77ad5051c19 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/LiteralBoostTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/LiteralBoostTestCase.java @@ -3,6 +3,7 @@ package com.yahoo.searchdefinition.derived; import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.document.DataType; +import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.RankProfile; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Search; @@ -40,7 +41,7 @@ public class LiteralBoostTestCase extends AbstractExportingTestCase { other.addRankSetting(new RankProfile.RankSetting("a", RankProfile.RankSetting.Type.LITERALBOOST, 333)); Processing.process(search, new BaseDeployLogger(), rankProfileRegistry, new QueryProfiles()); - DerivedConfiguration derived=new DerivedConfiguration(search, rankProfileRegistry); + DerivedConfiguration derived=new DerivedConfiguration(search, rankProfileRegistry, new QueryProfileRegistry()); // Check attribute fields derived.getAttributeFields(); // TODO: assert content @@ -70,8 +71,8 @@ public class LiteralBoostTestCase extends AbstractExportingTestCase { rankProfileRegistry.addRankProfile(other); other.addRankSetting(new RankProfile.RankSetting("a", RankProfile.RankSetting.Type.LITERALBOOST, 333)); - search = SearchBuilder.buildFromRawSearch(search, rankProfileRegistry); - DerivedConfiguration derived = new DerivedConfiguration(search, rankProfileRegistry); + search = SearchBuilder.buildFromRawSearch(search, rankProfileRegistry, new QueryProfileRegistry()); + DerivedConfiguration derived = new DerivedConfiguration(search, rankProfileRegistry, new QueryProfileRegistry()); // Check il script addition assertIndexing(Arrays.asList("clear_state | guard { input a | tokenize normalize stem:\"SHORTEST\" | index a; }", @@ -97,8 +98,8 @@ public class LiteralBoostTestCase extends AbstractExportingTestCase { field2.parseIndexingScript("{ summary | index }"); field2.setLiteralBoost(20); - search = SearchBuilder.buildFromRawSearch(search, rankProfileRegistry); - new DerivedConfiguration(search, rankProfileRegistry); + search = SearchBuilder.buildFromRawSearch(search, rankProfileRegistry, new QueryProfileRegistry()); + new DerivedConfiguration(search, rankProfileRegistry, new QueryProfileRegistry()); assertIndexing(Arrays.asList("clear_state | guard { input title | tokenize normalize stem:\"SHORTEST\" | summary title | index title; }", "clear_state | guard { input body | tokenize normalize stem:\"SHORTEST\" | summary body | index body; }", "clear_state | guard { input title | tokenize | index title_literal; }", diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SimpleInheritTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SimpleInheritTestCase.java index f7794d3439e..f4edc1dd0ae 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SimpleInheritTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SimpleInheritTestCase.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.derived; +import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.SearchBuilder; import com.yahoo.searchdefinition.parser.ParseException; @@ -31,7 +32,9 @@ public class SimpleInheritTestCase extends AbstractExportingTestCase { toDir.mkdirs(); deleteContent(toDir); - DerivedConfiguration config = new DerivedConfiguration(search, builder.getRankProfileRegistry()); + DerivedConfiguration config = new DerivedConfiguration(search, + builder.getRankProfileRegistry(), + new QueryProfileRegistry()); config.export(toDirName); checkDir(toDirName, expectedResultsDirName); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java index d09f56c31b3..3d2e1d11e28 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java @@ -3,6 +3,7 @@ package com.yahoo.searchdefinition.derived; import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.document.DataType; +import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.SearchDefinitionTestCase; @@ -20,9 +21,7 @@ import static org.junit.Assert.assertFalse; */ public class TypeConversionTestCase extends SearchDefinitionTestCase { - /** - * Tests that exact-string stuff is not spilled over to the default index - */ + /** Tests that exact-string stuff is not spilled over to the default index */ @Test public void testExactStringToStringTypeConversion() { Search search = new Search("test", null); @@ -34,7 +33,7 @@ public class TypeConversionTestCase extends SearchDefinitionTestCase { document.addField(a); Processing.process(search, new BaseDeployLogger(), rankProfileRegistry, new QueryProfiles()); - DerivedConfiguration derived = new DerivedConfiguration(search, rankProfileRegistry); + DerivedConfiguration derived = new DerivedConfiguration(search, rankProfileRegistry, new QueryProfileRegistry()); IndexInfo indexInfo = derived.getIndexInfo(); assertFalse(indexInfo.hasCommand("default", "compact-to-term")); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSearchFieldsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSearchFieldsTestCase.java index 3ffa89b612b..d743f60201e 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSearchFieldsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSearchFieldsTestCase.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.processing; +import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.SearchBuilder; import com.yahoo.searchdefinition.SearchDefinitionTestCase; @@ -98,7 +99,7 @@ public class ImplicitSearchFieldsTestCase extends SearchDefinitionTestCase { sb.importFile("src/test/examples/nextgen/simple.sd"); sb.build(); assertNotNull(sb.getSearch()); - new DerivedConfiguration(sb.getSearch(), sb.getRankProfileRegistry()); + new DerivedConfiguration(sb.getSearch(), sb.getRankProfileRegistry(), new QueryProfileRegistry()); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java index 86094458b34..800697b3430 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java @@ -3,6 +3,7 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.RankProfile; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Search; @@ -22,20 +23,22 @@ import static org.junit.Assert.assertEquals; class RankProfileSearchFixture { private RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + private final QueryProfileRegistry queryProfileRegistry; private Search search; RankProfileSearchFixture(String rankProfiles) throws ParseException { - this(MockApplicationPackage.createEmpty(), rankProfiles); + this(MockApplicationPackage.createEmpty(), new QueryProfileRegistry(), rankProfiles); } - RankProfileSearchFixture(ApplicationPackage applicationpackage, String rankProfiles) throws ParseException { - this(applicationpackage, rankProfiles, null, null); + RankProfileSearchFixture(ApplicationPackage applicationpackage, QueryProfileRegistry queryProfileRegistry, + String rankProfiles) throws ParseException { + this(applicationpackage, queryProfileRegistry, rankProfiles, null, null); } - RankProfileSearchFixture(ApplicationPackage applicationpackage, + RankProfileSearchFixture(ApplicationPackage applicationpackage, QueryProfileRegistry queryProfileRegistry, String rankProfiles, String constant, String field) throws ParseException { - SearchBuilder builder = new SearchBuilder(applicationpackage, rankProfileRegistry); + SearchBuilder builder = new SearchBuilder(applicationpackage, rankProfileRegistry, new QueryProfileRegistry()); String sdContent = "search test {\n" + " " + (constant != null ? constant : "") + "\n" + " document test {\n" + @@ -47,6 +50,7 @@ class RankProfileSearchFixture { builder.importString(sdContent); builder.build(); search = builder.getSearch(); + this.queryProfileRegistry = queryProfileRegistry; } public void assertFirstPhaseExpression(String expExpression, String rankProfile) { @@ -68,7 +72,7 @@ class RankProfileSearchFixture { } public RankProfile rankProfile(String rankProfile) { - return rankProfileRegistry.getRankProfile(search, rankProfile).compile(); + return rankProfileRegistry.getRankProfile(search, rankProfile).compile(queryProfileRegistry); } public Search search() { return search; } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankPropertyVariablesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankPropertyVariablesTestCase.java index bdfad96ef87..df2bcca63dd 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankPropertyVariablesTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankPropertyVariablesTestCase.java @@ -2,6 +2,7 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.RankProfile.RankProperty; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Search; @@ -16,10 +17,14 @@ import java.util.List; import static org.junit.Assert.fail; public class RankPropertyVariablesTestCase extends SearchDefinitionTestCase { + @Test public void testRankPropVariables() throws IOException, ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - Search search = SearchBuilder.buildFromFile("src/test/examples/rankpropvars.sd", new BaseDeployLogger(), rankProfileRegistry); + Search search = SearchBuilder.buildFromFile("src/test/examples/rankpropvars.sd", + new BaseDeployLogger(), + rankProfileRegistry, + new QueryProfileRegistry()); assertRankPropEquals(rankProfileRegistry.getRankProfile(search, "other").getRankProperties(), "$testvar1", "foo"); assertRankPropEquals(rankProfileRegistry.getRankProfile(search, "other").getRankProperties(), "$testvar_2", "bar"); assertRankPropEquals(rankProfileRegistry.getRankProfile(search, "other").getRankProperties(), "$testvarOne23", "baz"); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java index e6db6abdc95..7246b22b0f8 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java @@ -8,6 +8,7 @@ import com.yahoo.io.GrowableByteBuffer; import com.yahoo.io.IOUtils; import com.yahoo.io.reader.NamedReader; import com.yahoo.path.Path; +import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.RankingConstant; import com.yahoo.searchdefinition.parser.ParseException; import com.yahoo.tensor.Tensor; @@ -148,6 +149,7 @@ public class RankingExpressionWithTensorFlowTestCase { try { RankProfileSearchFixture search = new RankProfileSearchFixture( new StoringApplicationPackage(applicationDir), + new QueryProfileRegistry(), " rank-profile my_profile {\n" + " first-phase {\n" + " expression: tensorflow('mnist_softmax/saved')" + @@ -290,6 +292,7 @@ public class RankingExpressionWithTensorFlowTestCase { try { return new RankProfileSearchFixture( application, + application.getQueryProfiles(), " rank-profile my_profile {\n" + " macro Placeholder() {\n" + " expression: " + placeholderExpression + @@ -310,19 +313,14 @@ public class RankingExpressionWithTensorFlowTestCase { private final File root; - /** The content of the single query profile and type present in this, or null if none */ - private final String queryProfile, queryProfileType; - StoringApplicationPackage(Path applicationPackageWritableRoot) { this(applicationPackageWritableRoot, null, null); } StoringApplicationPackage(Path applicationPackageWritableRoot, String queryProfile, String queryProfileType) { super(null, null, Collections.emptyList(), null, - null, null, false); + null, null, false, queryProfile, queryProfileType); this.root = new File(applicationPackageWritableRoot.toString()); - this.queryProfile = queryProfile; - this.queryProfileType = queryProfileType; } @Override @@ -335,18 +333,6 @@ public class RankingExpressionWithTensorFlowTestCase { return new StoringApplicationPackageFile(file, Path.fromString(root.toString())); } - @Override - public List<NamedReader> getQueryProfileFiles() { - if (queryProfile == null) return Collections.emptyList(); - return Collections.singletonList(new NamedReader("default.xml", new StringReader(queryProfile))); - } - - @Override - public List<NamedReader> getQueryProfileTypeFiles() { - if (queryProfileType == null) return Collections.emptyList(); - return Collections.singletonList(new NamedReader("root.xml", new StringReader(queryProfileType))); - } - } private static class StoringApplicationPackageFile extends ApplicationFile { @@ -413,6 +399,17 @@ public class RankingExpressionWithTensorFlowTestCase { } @Override + public ApplicationFile appendFile(String value) { + try { + IOUtils.writeFile(file, value, true); + return this; + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @Override public List<ApplicationFile> listFiles(PathFilter filter) { if ( ! isDirectory()) return Collections.emptyList(); return Arrays.stream(file.listFiles()).filter(f -> filter.accept(Path.fromString(f.toString()))) diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java index cdb77f5d0e1..18c3e43ae7e 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java @@ -2,6 +2,7 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.collections.Pair; +import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.*; import com.yahoo.searchdefinition.derived.DerivedConfiguration; import com.yahoo.searchdefinition.derived.AttributeFields; @@ -16,13 +17,14 @@ import java.util.Map; import static org.junit.Assert.assertEquals; -// TODO: WHO? public class RankingExpressionsTestCase extends SearchDefinitionTestCase { @Test public void testMacros() throws IOException, ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - Search search = SearchBuilder.createFromDirectory("src/test/examples/rankingexpressionfunction", rankProfileRegistry).getSearch(); + Search search = SearchBuilder.createFromDirectory("src/test/examples/rankingexpressionfunction", + rankProfileRegistry, + new QueryProfileRegistry()).getSearch(); final RankProfile macrosRankProfile = rankProfileRegistry.getRankProfile(search, "macros"); macrosRankProfile.parseExpressions(); final Map<String, RankProfile.Macro> macros = macrosRankProfile.getMacros(); @@ -35,7 +37,9 @@ public class RankingExpressionsTestCase extends SearchDefinitionTestCase { assertEquals("78 + closeness(distance)", macros.get("artistmatch").getTextualExpression().trim()); assertEquals(0, macros.get("artistmatch").getFormalParams().size()); - List<Pair<String, String>> rankProperties = new RawRankProfile(macrosRankProfile, new AttributeFields(search)).configProperties(); + List<Pair<String, String>> rankProperties = new RawRankProfile(macrosRankProfile, + new QueryProfileRegistry(), + new AttributeFields(search)).configProperties(); assertEquals(6, rankProperties.size()); assertEquals("rankingExpression(titlematch$).rankingScript", rankProperties.get(0).getFirst()); @@ -57,8 +61,10 @@ public class RankingExpressionsTestCase extends SearchDefinitionTestCase { @Test(expected = IllegalArgumentException.class) public void testThatIncludingFileInSubdirFails() throws IOException, ParseException { RankProfileRegistry registry = new RankProfileRegistry(); - Search search = SearchBuilder.createFromDirectory("src/test/examples/rankingexpressioninfile", registry).getSearch(); - new DerivedConfiguration(search, registry); // rank profile parsing happens during deriving + Search search = SearchBuilder.createFromDirectory("src/test/examples/rankingexpressioninfile", + registry, + new QueryProfileRegistry()).getSearch(); + new DerivedConfiguration(search, registry, new QueryProfileRegistry()); // rank profile parsing happens during deriving } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java index 475fee62177..c18cfcfe1aa 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java @@ -91,7 +91,7 @@ public class TensorTransformTestCase extends SearchDefinitionTestCase { private void assertContainsExpression(String expr, String transformedExpression) throws ParseException { - assertTrue("Expected expression '" + transformedExpression + "' not found", + assertTrue("Expected expression '" + transformedExpression + "' found", containsExpression(expr, transformedExpression)); } @@ -169,10 +169,10 @@ public class TensorTransformTestCase extends SearchDefinitionTestCase { "}\n"); builder.build(new BaseDeployLogger(), setupQueryProfileTypes()); Search s = builder.getSearch(); - RankProfile test = rankProfileRegistry.getRankProfile(s, "test").compile(); - List<Pair<String, String>> testRankProperties = new RawRankProfile(test, new AttributeFields(s)).configProperties(); - for (Object o : testRankProperties) - System.out.println(o); + RankProfile test = rankProfileRegistry.getRankProfile(s, "test").compile(new QueryProfileRegistry()); + List<Pair<String, String>> testRankProperties = new RawRankProfile(test, + new QueryProfileRegistry(), + new AttributeFields(s)).configProperties(); return testRankProperties; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFile.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFile.java index 717fb88e5dc..affc2e03e2b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFile.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFile.java @@ -95,7 +95,6 @@ class ZKApplicationFile extends ApplicationFile { @Override public ApplicationFile writeFile(Reader input) { - // foo/bar/baz.txt String zkPath = getZKPath(path); try { String data = IOUtils.readAll(input); @@ -112,6 +111,21 @@ class ZKApplicationFile extends ApplicationFile { } @Override + public ApplicationFile appendFile(String value) { + String zkPath = getZKPath(path); + String status = ContentStatusNew; + if (zkApp.exists(zkPath)) { + status = ContentStatusChanged; + } + String existingData = zkApp.getData(zkPath); + if (existingData == null) + existingData = ""; + zkApp.putData(zkPath, existingData + value); + writeMetaFile(value, status); + return this; + } + + @Override public List<ApplicationFile> listFiles(PathFilter filter) { String userPath = getZKPath(path); List<ApplicationFile> ret = new ArrayList<>(); diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/VespaHeaders.java b/container-core/src/main/java/com/yahoo/container/jdisc/VespaHeaders.java index ecb36ade5eb..5fa1a57569b 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/VespaHeaders.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/VespaHeaders.java @@ -6,6 +6,7 @@ import static com.yahoo.container.protect.Error.BACKEND_COMMUNICATION_ERROR; import static com.yahoo.container.protect.Error.BAD_REQUEST; import static com.yahoo.container.protect.Error.FORBIDDEN; import static com.yahoo.container.protect.Error.ILLEGAL_QUERY; +import static com.yahoo.container.protect.Error.INSUFFICIENT_STORAGE; import static com.yahoo.container.protect.Error.INTERNAL_SERVER_ERROR; import static com.yahoo.container.protect.Error.INVALID_QUERY_PARAMETER; import static com.yahoo.container.protect.Error.NOT_FOUND; @@ -13,7 +14,6 @@ import static com.yahoo.container.protect.Error.NO_BACKENDS_IN_SERVICE; import static com.yahoo.container.protect.Error.TIMEOUT; import static com.yahoo.container.protect.Error.UNAUTHORIZED; -import java.net.URLDecoder; import java.util.Iterator; import com.yahoo.collections.Tuple2; @@ -159,6 +159,8 @@ public final class VespaHeaders { return new Tuple2<>(true, Response.Status.BAD_REQUEST); if (error.getCode() == INTERNAL_SERVER_ERROR.code) return new Tuple2<>(true, Response.Status.INTERNAL_SERVER_ERROR); + if (error.getCode() == INSUFFICIENT_STORAGE.code) + return new Tuple2<>(true, Response.Status.INSUFFICIENT_STORAGE); return NO_MATCH; } diff --git a/container-core/src/main/java/com/yahoo/container/protect/Error.java b/container-core/src/main/java/com/yahoo/container/protect/Error.java index 46b49b1623a..b39a33b1346 100644 --- a/container-core/src/main/java/com/yahoo/container/protect/Error.java +++ b/container-core/src/main/java/com/yahoo/container/protect/Error.java @@ -26,7 +26,8 @@ public enum Error { FORBIDDEN(15), NOT_FOUND(16), BAD_REQUEST(17), - INTERNAL_SERVER_ERROR(18); + INTERNAL_SERVER_ERROR(18), + INSUFFICIENT_STORAGE(19); public final int code; diff --git a/container-search/src/main/java/com/yahoo/fs4/GetDocSumsPacket.java b/container-search/src/main/java/com/yahoo/fs4/GetDocSumsPacket.java index a40f5d55317..c1b0e9ccb37 100644 --- a/container-search/src/main/java/com/yahoo/fs4/GetDocSumsPacket.java +++ b/container-search/src/main/java/com/yahoo/fs4/GetDocSumsPacket.java @@ -102,10 +102,9 @@ public class GetDocSumsPacket extends Packet { buffer.putInt((int)features); buffer.putInt(0); //Unused, was docstamp long timeLeft = query.getTimeLeft(); - final int minTimeout = 50; - buffer.putInt(Math.max(minTimeout, (int)timeLeft)); + buffer.putInt(Math.max(1, (int)timeLeft)); // Safety to avoid sending down 0 or negative number if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Timeout from query(" + query.getTimeout() + "), sent to backend: " + Math.max(minTimeout, timeLeft)); + log.log(LogLevel.DEBUG, "Timeout from query(" + query.getTimeout() + "), sent to backend: " + timeLeft); } if (queryPacketData != null) diff --git a/container-search/src/main/java/com/yahoo/fs4/QueryPacket.java b/container-search/src/main/java/com/yahoo/fs4/QueryPacket.java index 1513cf2213c..8a14ea5c343 100644 --- a/container-search/src/main/java/com/yahoo/fs4/QueryPacket.java +++ b/container-search/src/main/java/com/yahoo/fs4/QueryPacket.java @@ -123,8 +123,7 @@ public class QueryPacket extends Packet { ignoreableOffset = buffer.position() - relativeZero; IntegerCompressor.putCompressedPositiveNumber(getOffset(), buffer); IntegerCompressor.putCompressedPositiveNumber(getHits(), buffer); - // store the cutoff time in the tag object, and then do a similar Math.max there - buffer.putInt(Math.max(50, (int)query.getTimeLeft())); + buffer.putInt(Math.max(1, (int)query.getTimeLeft())); // Safety to avoid sending down 0 or negative number ignoreableSize = buffer.position() - relativeZero - ignoreableOffset; buffer.putInt(getFlagInt()); int startOfFieldToSave = buffer.position(); diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldDescription.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldDescription.java index 6be0dac6448..e8d8ecb02bb 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldDescription.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldDescription.java @@ -4,6 +4,7 @@ package com.yahoo.search.query.profile.types; import com.google.common.collect.ImmutableList; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.query.profile.QueryProfile; +import com.yahoo.tensor.TensorType; import java.util.Arrays; import java.util.Collections; diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldType.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldType.java index 69c07843681..3bfd33668e6 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldType.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldType.java @@ -6,6 +6,7 @@ import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; import com.yahoo.search.yql.YqlQuery; import com.yahoo.tensor.Tensor; +import com.yahoo.tensor.TensorType; /** * Superclass of query type field types. @@ -43,6 +44,12 @@ public abstract class FieldType { public abstract Object convertFrom(Object o, CompiledQueryProfileRegistry registry); /** + * Returns this type as a tensor type: The true tensor type is this is a tensor field an an empty type - + * interpreted as a double in numerical contexts - otherwise + */ + public TensorType asTensorType() { return TensorType.empty; } + + /** * Returns the field type for a given string name. * * @param typeString a type string - a primitive name, "query-profile" or "query-profile:profile-name" diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/TensorFieldType.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/TensorFieldType.java index 53ab7aefafd..9699a72cb31 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/types/TensorFieldType.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/TensorFieldType.java @@ -21,7 +21,8 @@ public class TensorFieldType extends FieldType { } /** Returns information about the type of tensor this will hold */ - public TensorType type() { return type; } + @Override + public TensorType asTensorType() { return type; } @Override public Class getValueClass() { return Tensor.class; } diff --git a/container-search/src/test/java/com/yahoo/fs4/test/QueryTestCase.java b/container-search/src/test/java/com/yahoo/fs4/test/QueryTestCase.java index f47c8e9e0b8..c7359ce2f94 100644 --- a/container-search/src/test/java/com/yahoo/fs4/test/QueryTestCase.java +++ b/container-search/src/test/java/com/yahoo/fs4/test/QueryTestCase.java @@ -35,7 +35,7 @@ public class QueryTestCase extends junit.framework.TestCase { 0,0,0,6, // Features 2, 8, - 0,0,0,50, // querytimeout + 0,0,0,1, // querytimeout 0,0,0x40,0x01, // qflags 7, 'd', 'e', 'f', 'a', 'u', 'l', 't', @@ -57,7 +57,7 @@ public class QueryTestCase extends junit.framework.TestCase { 0,0,0,6, // Features 2, 8, - 0,0,0,50, // querytimeout + 0,0,0,1, // querytimeout 0,0,0x40,0x01, // QFlags 3, 't','w','o', // Ranking @@ -162,7 +162,7 @@ public class QueryTestCase extends junit.framework.TestCase { 0,0,0,-122, // Features 2, // offset 8, // maxhits - 0,0,0,50, // querytimeout + 0,0,0,1, // querytimeout 0,0,0x40,0x01, // qflags 7, 'd', 'e', 'f', 'a', 'u', 'l', 't', @@ -233,7 +233,7 @@ public class QueryTestCase extends junit.framework.TestCase { 0,0,0,6, // Features 2, 8, - 0,0,0,50, // querytimeout + 0,0,0,1, // querytimeout 0,0,0x40,0x01, // qflags 7, 'd', 'e', 'f', 'a', 'u', 'l', 't', @@ -252,7 +252,7 @@ public class QueryTestCase extends junit.framework.TestCase { 0,0,0,6, // Features 2, 8, - 0,0,0,50, // querytimeout + 0,0,0,1, // querytimeout 0,0,0x40,0x01, // qflags 7, 'd', 'e', 'f', 'a', 'u', 'l', 't', diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/InstancesReply.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/InstancesReply.java index ff0c155460e..ebacafd75c4 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/InstancesReply.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/InstancesReply.java @@ -15,4 +15,5 @@ public class InstancesReply { public Set<URI> globalRotations; public List<InstanceReference> instances; public String compileVersion; + public String rotationId; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java index d5ce613b98d..5105a016934 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java @@ -7,9 +7,9 @@ import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.integration.MetricsService.ApplicationMetrics; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationRotation; import com.yahoo.vespa.hosted.controller.application.ApplicationVersion; import com.yahoo.vespa.hosted.controller.application.Change; @@ -42,7 +42,7 @@ public class Application { private final Map<ZoneId, Deployment> deployments; private final DeploymentJobs deploymentJobs; private final Change change; - private final boolean outstandingChange; + private final Change outstandingChange; private final Optional<IssueId> ownershipIssueId; private final ApplicationMetrics metrics; private final Optional<RotationId> rotation; @@ -51,14 +51,14 @@ public class Application { public Application(ApplicationId id) { this(id, DeploymentSpec.empty, ValidationOverrides.empty, Collections.emptyMap(), new DeploymentJobs(Optional.empty(), Collections.emptyList(), Optional.empty()), - Change.empty(), false, Optional.empty(), new ApplicationMetrics(0, 0), + Change.empty(), Change.empty(), Optional.empty(), new ApplicationMetrics(0, 0), Optional.empty()); } /** Used from persistence layer: Do not use */ public Application(ApplicationId id, DeploymentSpec deploymentSpec, ValidationOverrides validationOverrides, List<Deployment> deployments, DeploymentJobs deploymentJobs, Change change, - boolean outstandingChange, Optional<IssueId> ownershipIssueId, ApplicationMetrics metrics, + Change outstandingChange, Optional<IssueId> ownershipIssueId, ApplicationMetrics metrics, Optional<RotationId> rotation) { this(id, deploymentSpec, validationOverrides, deployments.stream().collect(Collectors.toMap(Deployment::zone, d -> d)), @@ -67,7 +67,7 @@ public class Application { Application(ApplicationId id, DeploymentSpec deploymentSpec, ValidationOverrides validationOverrides, Map<ZoneId, Deployment> deployments, DeploymentJobs deploymentJobs, Change change, - boolean outstandingChange, Optional<IssueId> ownershipIssueId, ApplicationMetrics metrics, + Change outstandingChange, Optional<IssueId> ownershipIssueId, ApplicationMetrics metrics, Optional<RotationId> rotation) { Objects.requireNonNull(id, "id cannot be null"); Objects.requireNonNull(deploymentSpec, "deploymentSpec cannot be null"); @@ -129,7 +129,7 @@ public class Application { * Returns whether this has an outstanding change (in the source repository), which * has currently not started deploying (because a deployment is (or was) already in progress */ - public boolean hasOutstandingChange() { return outstandingChange; } + public Change outstandingChange() { return outstandingChange; } public Optional<IssueId> ownershipIssueId() { return ownershipIssueId; @@ -157,25 +157,31 @@ public class Application { /** Returns the current version this application has, or if none; should use, in the given zone */ public Version versionIn(ZoneId zone, Controller controller) { return Optional.ofNullable(deployments().get(zone)).map(Deployment::version) // Already deployed in this zone: Use that version - .orElse(oldestDeployedVersion().orElse(controller.systemVersion())); - } - - /** Returns the application version a deployment to this zone should use, or empty if we don't know */ - public Optional<ApplicationVersion> deployApplicationVersionIn(ZoneId zone) { - if (change().application().isPresent()) { - ApplicationVersion version = change().application().get(); - if (version == ApplicationVersion.unknown) - return Optional.empty(); - else - return Optional.of(version); + .orElse(oldestDeployedVersion().orElse(controller.systemVersion())); + } + + /** Returns the Vespa version to use for the given job */ + public Version deployVersionFor(DeploymentJobs.JobType jobType, Controller controller) { + return jobType == DeploymentJobs.JobType.component + ? controller.systemVersion() + : deployVersionIn(jobType.zone(controller.system()).get(), controller); + } + + /** Returns the application version to use for the given job */ + public Optional<ApplicationVersion> deployApplicationVersionFor(DeploymentJobs.JobType jobType, + Controller controller, + boolean currentVersion) { + // Use last successful version if currentVersion is requested (staging deployment) or when we're not deploying + // a new application version + if (currentVersion || !change().application().isPresent()) { + Optional<ApplicationVersion> version = deploymentJobs().lastSuccessfulApplicationVersionFor(jobType); + if (version.isPresent()) { + return version; + } } - - return applicationVersionIn(zone); - } - - /** Returns the application version that is or should be deployed with in the given zone, or empty if unknown. */ - public Optional<ApplicationVersion> applicationVersionIn(ZoneId zone) { - return Optional.ofNullable(deployments().get(zone)).map(Deployment::applicationVersion); + return jobType == DeploymentJobs.JobType.component + ? Optional.empty() + : deployApplicationVersionIn(jobType.zone(controller.system()).get()); } /** Returns the global rotation of this, if present */ @@ -203,8 +209,22 @@ public class Application { return "application '" + id + "'"; } + /** Returns whether changes to this are blocked in the given instant */ public boolean isBlocked(Instant instant) { return ! deploymentSpec().canUpgradeAt(instant) || ! deploymentSpec().canChangeRevisionAt(instant); } + /** Returns the application version a deployment to this zone should use, or empty if we don't know */ + private Optional<ApplicationVersion> deployApplicationVersionIn(ZoneId zone) { + if (change().application().isPresent()) { + return Optional.of(change().application().get()); + } + return applicationVersionIn(zone); + } + + /** Returns the application version that is or should be deployed with in the given zone, or empty if unknown. */ + private Optional<ApplicationVersion> applicationVersionIn(ZoneId zone) { + return Optional.ofNullable(deployments().get(zone)).map(Deployment::applicationVersion); + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 5d75ae4a340..090f25171b1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -23,7 +23,6 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.RevisionId; import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactory; import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsClient; -import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsException; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerClient; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Log; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NoInstanceException; @@ -287,41 +286,51 @@ public class ApplicationController { version = application.deployVersionIn(zone, controller); } - Optional<DeploymentJobs.JobType> jobType = DeploymentJobs.JobType.from(controller.system(), zone); - if (!jobType.isPresent() && !applicationPackageFromDeployer.isPresent()) { - throw new IllegalArgumentException("Unable to determine job type from zone '" + zone + - "' and no application package was given"); - } - - // Determine which application package to use - ApplicationPackage applicationPackage; + // Determine application package to use ApplicationVersion applicationVersion; - if (applicationPackageFromDeployer.isPresent()) { - applicationVersion = toApplicationPackageRevision(applicationPackageFromDeployer.get(), - options.screwdriverBuildJob); - applicationPackage = applicationPackageFromDeployer.get(); - } else { - applicationVersion = application.deployApplicationVersion(jobType.get(), controller) - .orElseThrow(() -> new IllegalArgumentException("Cannot determine application version to use in " + zone)); - applicationPackage = new ApplicationPackage(artifactRepository.getApplicationPackage( - applicationId, applicationVersion.id()) + ApplicationPackage applicationPackage; + Optional<DeploymentJobs.JobType> job = DeploymentJobs.JobType.from(controller.system(), zone); + + // TODO: Simplify after new application version is always available + if (canDownloadReportedApplicationVersion(application) && !canDeployDirectlyTo(zone, options)) { + if (!job.isPresent()) { + throw new IllegalArgumentException("Cannot determine job for zone " + zone); + } + applicationVersion = application.deployApplicationVersionFor(job.get(), controller, + options.deployCurrentVersion) + .orElseThrow(() -> new IllegalArgumentException("Cannot determine application version for " + applicationId + " in " + job.get())); + if (canDownloadArtifact(applicationVersion)) { + applicationPackage = new ApplicationPackage( + artifactRepository.getApplicationPackage(applicationId, applicationVersion.id()) + ); + } else { + applicationPackage = applicationPackageFromDeployer.orElseThrow( + () -> new IllegalArgumentException("Application package with version " + + applicationVersion.id() + " cannot be downloaded, and " + + "no package was given by deployer")); + } + } else { // ..otherwise we use the package sent by the deployer and deduce version from the package + // TODO: Only allow this for environments that are allowed to deploy directly + applicationPackage = applicationPackageFromDeployer.orElseThrow( + () -> new IllegalArgumentException("Application package must be given as new application " + + "version is not known for " + applicationId) ); + applicationVersion = toApplicationPackageRevision(applicationPackage, options.screwdriverBuildJob); } - validate(applicationPackage.deploymentSpec()); - // TODO: Remove after introducing new application version number - if ( ! options.deployCurrentVersion && applicationPackageFromDeployer.isPresent()) { + // TODO: Remove after introducing new application version + if (!options.deployCurrentVersion && !canDownloadReportedApplicationVersion(application)) { if (application.change().application().isPresent()) { - application = application.withDeploying(application.change().with(applicationVersion)); + application = application.withChange(application.change().with(applicationVersion)); } - if (!canDeployDirectlyTo(zone, options) && jobType.isPresent()) { + if (!canDeployDirectlyTo(zone, options) && job.isPresent()) { // Update with (potentially) missing information about what we triggered: // * When someone else triggered the job, we need to store a stand-in triggering event. // * When this is the system test job, we need to record the new application version, // for future use. - JobStatus.JobRun triggering = getOrCreateTriggering(application, version, jobType.get()); - application = application.withJobTriggering(jobType.get(), + JobStatus.JobRun triggering = getOrCreateTriggering(application, version, job.get()); + application = application.withJobTriggering(job.get(), application.change(), triggering.at(), version, @@ -406,7 +415,8 @@ public class ApplicationController { Log logEntry = new Log(); logEntry.level = "WARNING"; logEntry.time = clock.instant().toEpochMilli(); - logEntry.message = "Ignoring deployment of " + get(applicationId) + " to " + zone + " as a deployment is not currently expected"; + logEntry.message = "Ignoring deployment of " + require(applicationId) + " to " + zone + + " as a deployment is not currently expected"; PrepareResponse prepareResponse = new PrepareResponse(); prepareResponse.log = Collections.singletonList(logEntry); prepareResponse.configChangeActions = new ConfigChangeActions(Collections.emptyList(), Collections.emptyList()); @@ -682,6 +692,18 @@ public class ApplicationController { zone.environment().isManuallyDeployed(); } + /** Returns whether artifact for given version number is available in artifact repository */ + private static boolean canDownloadArtifact(ApplicationVersion applicationVersion) { + return applicationVersion.buildNumber().isPresent() && applicationVersion.source().isPresent(); + } + + /** Returns whether component has reported a version number that is availabe in artifact repository */ + private static boolean canDownloadReportedApplicationVersion(Application application) { + return application.deploymentJobs().lastSuccessfulApplicationVersionFor(DeploymentJobs.JobType.component) + .filter(ApplicationController::canDownloadArtifact) + .isPresent(); + } + /** Verify that each of the production zones listed in the deployment spec exist in this system. */ private void validate(DeploymentSpec deploymentSpec) { deploymentSpec.zones().stream() diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java index e744df0da68..950790e26b6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java @@ -6,11 +6,11 @@ import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.api.integration.MetricsService; import com.yahoo.vespa.hosted.controller.api.integration.MetricsService.ApplicationMetrics; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationRotation; import com.yahoo.vespa.hosted.controller.application.ApplicationVersion; import com.yahoo.vespa.hosted.controller.application.Change; @@ -49,7 +49,7 @@ public class LockedApplication extends Application { private LockedApplication(Builder builder) { super(builder.applicationId, builder.deploymentSpec, builder.validationOverrides, builder.deployments, builder.deploymentJobs, builder.deploying, - builder.hasOutstandingChange, builder.ownershipIssueId, builder.metrics, builder.rotation); + builder.outstandingChange, builder.ownershipIssueId, builder.metrics, builder.rotation); } public LockedApplication withProjectId(long projectId) { @@ -60,8 +60,11 @@ public class LockedApplication extends Application { return new LockedApplication(new Builder(this).with(deploymentJobs().with(issueId))); } - public LockedApplication withJobCompletion(DeploymentJobs.JobReport report, Instant notificationTime, Controller controller) { - return new LockedApplication(new Builder(this).with(deploymentJobs().withCompletion(report, notificationTime, controller))); + public LockedApplication withJobCompletion(DeploymentJobs.JobReport report, ApplicationVersion applicationVersion, + Instant notificationTime, Controller controller) { + return new LockedApplication(new Builder(this).with(deploymentJobs().withCompletion( + report, applicationVersion, notificationTime, controller)) + ); } public LockedApplication withJobTriggering(JobType type, Change change, Instant triggerTime, @@ -119,12 +122,12 @@ public class LockedApplication extends Application { return new LockedApplication(new Builder(this).with(validationOverrides)); } - public LockedApplication withDeploying(Change deploying) { - return new LockedApplication(new Builder(this).withDeploying(deploying)); + public LockedApplication withChange(Change change) { + return new LockedApplication(new Builder(this).withChange(change)); } - public LockedApplication withOutstandingChange(boolean outstandingChange) { - return new LockedApplication(new Builder(this).with(outstandingChange)); + public LockedApplication withOutstandingChange(Change outstandingChange) { + return new LockedApplication(new Builder(this).withOutstandingChange(outstandingChange)); } public LockedApplication withOwnershipIssueId(IssueId issueId) { @@ -139,18 +142,6 @@ public class LockedApplication extends Application { return new LockedApplication(new Builder(this).with(rotation)); } - public Version deployVersionFor(DeploymentJobs.JobType jobType, Controller controller) { - return jobType == JobType.component - ? controller.systemVersion() - : deployVersionIn(jobType.zone(controller.system()).get(), controller); - } - - public Optional<ApplicationVersion> deployApplicationVersion(DeploymentJobs.JobType jobType, Controller controller) { - return jobType == JobType.component - ? Optional.empty() - : deployApplicationVersionIn(jobType.zone(controller.system()).get()); - } - /** Don't expose non-leaf sub-objects. */ private LockedApplication with(Deployment deployment) { Map<ZoneId, Deployment> deployments = new LinkedHashMap<>(deployments()); @@ -158,7 +149,6 @@ public class LockedApplication extends Application { return new LockedApplication(new Builder(this).with(deployments)); } - private static class Builder { private final ApplicationId applicationId; @@ -167,7 +157,7 @@ public class LockedApplication extends Application { private Map<ZoneId, Deployment> deployments; private DeploymentJobs deploymentJobs; private Change deploying; - private boolean hasOutstandingChange; + private Change outstandingChange; private Optional<IssueId> ownershipIssueId; private ApplicationMetrics metrics; private Optional<RotationId> rotation; @@ -179,7 +169,7 @@ public class LockedApplication extends Application { this.deployments = application.deployments(); this.deploymentJobs = application.deploymentJobs(); this.deploying = application.change(); - this.hasOutstandingChange = application.hasOutstandingChange(); + this.outstandingChange = application.outstandingChange(); this.ownershipIssueId = application.ownershipIssueId(); this.metrics = application.metrics(); this.rotation = application.rotation().map(ApplicationRotation::id); @@ -205,13 +195,13 @@ public class LockedApplication extends Application { return this; } - private Builder withDeploying(Change deploying) { + private Builder withChange(Change deploying) { this.deploying = deploying; return this; } - private Builder with(boolean hasOutstandingChange) { - this.hasOutstandingChange = hasOutstandingChange; + private Builder withOutstandingChange(Change outstandingChange) { + this.outstandingChange = outstandingChange; return this; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationVersion.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationVersion.java index 304d82b2bec..31ee5fa6d44 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationVersion.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationVersion.java @@ -11,18 +11,19 @@ import java.util.Optional; * @author bratseth * @author mpolden */ -public class ApplicationVersion { +public class ApplicationVersion implements Comparable<ApplicationVersion> { - // TODO: Remove the need for this + /** + * Used in cases where application version cannot be determined, such as manual deployments (e.g. in dev + * environment) + */ public static final ApplicationVersion unknown = new ApplicationVersion(); - // Never changes. Only used to create a valid version number for the bundle + // This never changes and is only used to create a valid semantic version number, as required by application bundles private static final String majorVersion = "1.0"; - // TODO: Remove after introducing new application version + // TODO: Remove after 2018-03-01 private final Optional<String> applicationPackageHash; - - // TODO: Make mandatory private final Optional<SourceRevision> source; private final Optional<Long> buildNumber; @@ -37,11 +38,11 @@ public class ApplicationVersion { Objects.requireNonNull(applicationPackageHash, "applicationPackageHash cannot be null"); Objects.requireNonNull(source, "source cannot be null"); Objects.requireNonNull(buildNumber, "buildNumber cannot be null"); - if (buildNumber.isPresent() && !source.isPresent()) { - throw new IllegalArgumentException("both buildNumber and source must be set if buildNumber is set"); + if (!applicationPackageHash.isPresent() && source.isPresent() != buildNumber.isPresent()) { + throw new IllegalArgumentException("both buildNumber and source must be set together"); } - if ( ! buildNumber.isPresent() && ! applicationPackageHash.isPresent()) { - throw new IllegalArgumentException("applicationPackageHash must be given if buildNumber is unset"); + if (buildNumber.isPresent() && buildNumber.get() <= 0) { + throw new IllegalArgumentException("buildNumber must be > 0"); } this.applicationPackageHash = applicationPackageHash; this.source = source; @@ -68,9 +69,14 @@ public class ApplicationVersion { if (applicationPackageHash.isPresent()) { return applicationPackageHash.get(); } - return String.format("%s.%d-%s", majorVersion, buildNumber.get(), abbreviateCommit(source.get().commit())); + return source.map(sourceRevision -> String.format("%s.%d-%s", majorVersion, buildNumber.get(), + abbreviateCommit(source.get().commit()))) + .orElse("unknown"); } + /** Returns the application package hash, if known */ + public Optional<String> applicationPackageHash() { return applicationPackageHash; } + /** * Returns information about the source of this revision, or empty if the source is not know/defined * (which is the case for command-line deployment from developers, but never for deployment jobs) @@ -80,23 +86,29 @@ public class ApplicationVersion { /** Returns the build number that built this version */ public Optional<Long> buildNumber() { return buildNumber; } + /** Returns whether this is unknown */ + public boolean isUnknown() { + return this.equals(unknown); + } + @Override - public int hashCode() { return applicationPackageHash.hashCode(); } + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ApplicationVersion that = (ApplicationVersion) o; + return Objects.equals(applicationPackageHash, that.applicationPackageHash) && + Objects.equals(source, that.source) && + Objects.equals(buildNumber, that.buildNumber); + } @Override - public boolean equals(Object other) { - if (this == other) return true; - if ( ! (other instanceof ApplicationVersion)) return false; - return this.applicationPackageHash.equals(((ApplicationVersion)other).applicationPackageHash); + public int hashCode() { + return Objects.hash(applicationPackageHash, source, buildNumber); } @Override public String toString() { - if (buildNumber.isPresent()) { - return "Application package version: " + abbreviateCommit(source.get().commit()) + "-" + buildNumber.get(); - } - return "Application package revision '" + applicationPackageHash + "'" + - (source.isPresent() ? " with " + source.get() : ""); + return "Application package version: " + id() + source.map(s -> ", " + s.toString()).orElse(""); } /** Abbreviate given commit hash to 9 characters */ @@ -104,4 +116,11 @@ public class ApplicationVersion { return hash.length() <= 9 ? hash : hash.substring(0, 9); } + @Override + public int compareTo(ApplicationVersion o) { + if (!buildNumber().isPresent() || !o.buildNumber().isPresent()) { + return 0; // No sorting for application package hash + } + return buildNumber().get().compareTo(o.buildNumber().get()); + } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Change.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Change.java index 13d66c8d083..216025215d3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Change.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Change.java @@ -31,6 +31,10 @@ public final class Change { private Change(Optional<Version> platform, Optional<ApplicationVersion> application) { Objects.requireNonNull(platform, "platform cannot be null"); Objects.requireNonNull(application, "application cannot be null"); + if (application.isPresent() && application.get().isUnknown()) { + // TODO: Require version to be known for application change + //throw new IllegalArgumentException("Application version to deploy must be a known version"); + } this.platform = platform; this.application = application; } @@ -42,7 +46,7 @@ public final class Change { return false; } - /** Returns whether a change shoudl currently be deployed */ + /** Returns whether a change should currently be deployed */ public boolean isPresent() { return platform.isPresent() || application.isPresent(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java index bb7b39eed0f..7e1dcfe2bc6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java @@ -55,11 +55,13 @@ public class DeploymentJobs { } /** Return a new instance with the given completion */ - public DeploymentJobs withCompletion(JobReport report, Instant notificationTime, Controller controller) { + public DeploymentJobs withCompletion(JobReport report, ApplicationVersion applicationVersion, + Instant notificationTime, Controller controller) { Map<JobType, JobStatus> status = new LinkedHashMap<>(this.status); status.compute(report.jobType(), (type, job) -> { if (job == null) job = JobStatus.initial(report.jobType()); - return job.withCompletion(report.buildNumber(), report.jobError(), notificationTime, controller); + return job.withCompletion(report.buildNumber(), applicationVersion, report.jobError(), notificationTime, + controller); }); return new DeploymentJobs(Optional.of(report.projectId()), status, issueId); } @@ -129,12 +131,11 @@ public class DeploymentJobs { return true; // other environments do not have any preconditions } - /** Returns whether the job of the given type has completed successfully for the given change */ - public boolean isSuccessful(Change change, JobType jobType) { + /** Returns the last successful application version for the given job */ + public Optional<ApplicationVersion> lastSuccessfulApplicationVersionFor(JobType jobType) { return Optional.ofNullable(jobStatus().get(jobType)) - .flatMap(JobStatus::lastSuccess) - .filter(status -> status.lastCompletedWas(change)) - .isPresent(); + .flatMap(JobStatus::lastSuccess) + .map(JobStatus.JobRun::applicationVersion); } /** @@ -146,6 +147,14 @@ public class DeploymentJobs { public Optional<IssueId> issueId() { return issueId; } + /** Returns whether the job of the given type has completed successfully for the given change */ + private boolean isSuccessful(Change change, JobType jobType) { + return Optional.ofNullable(jobStatus().get(jobType)) + .flatMap(JobStatus::lastSuccess) + .filter(status -> status.lastCompletedWas(change)) + .isPresent(); + } + private static Optional<Long> requireId(Optional<Long> id, String message) { Objects.requireNonNull(id, message); if ( ! id.isPresent()) { @@ -254,11 +263,16 @@ public class DeploymentJobs { Objects.requireNonNull(sourceRevision, "sourceRevision cannot be null"); Objects.requireNonNull(jobError, "jobError cannot be null"); + if (jobType == JobType.component && !sourceRevision.isPresent()) { + // TODO: Throw after 2018-03-01 + //throw new IllegalArgumentException("sourceRevision is required for job " + jobType); + } + this.applicationId = applicationId; this.projectId = projectId; this.buildNumber = buildNumber; this.jobType = jobType; - this.sourceRevision = sourceRevision; // TODO: Require non-empty source revision if jobType == component + this.sourceRevision = sourceRevision; this.jobError = jobError; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobList.java index 41060a7af4c..fb9ab8735d8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobList.java @@ -172,7 +172,7 @@ public class JobList { private static boolean failingApplicationChange(JobStatus job) { if ( job.isSuccess()) return false; if ( ! job.lastSuccess().isPresent()) return true; // An application which never succeeded is surely bad. - if ( job.lastSuccess().get().applicationVersion() == ApplicationVersion.unknown) return true; // Indicates the component job, which is always an application change. + if ( job.lastSuccess().get().applicationVersion().isUnknown()) return true; // Indicates the component job, which is always an application change. if ( ! job.firstFailing().get().version().equals(job.lastSuccess().get().version())) return false; // Version change may be to blame. return ! job.firstFailing().get().applicationVersion().equals(job.lastSuccess().get().applicationVersion()); // Return whether there is an application change. } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobStatus.java index e165d3c9fe5..e963fde8f94 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobStatus.java @@ -57,27 +57,31 @@ public class JobStatus { public JobStatus withTriggering(Version version, ApplicationVersion applicationVersion, boolean upgrade, String reason, Instant triggerTime) { - return new JobStatus(type, jobError, Optional.of(new JobRun(-1, version, applicationVersion, upgrade, reason, triggerTime)), + return new JobStatus(type, jobError, Optional.of(new JobRun(-1, version, applicationVersion, upgrade, + reason, triggerTime)), lastCompleted, firstFailing, lastSuccess); } - public JobStatus withCompletion(long runId, Optional<DeploymentJobs.JobError> jobError, Instant completionTime, Controller controller) { + public JobStatus withCompletion(long runId, Optional<DeploymentJobs.JobError> jobError, Instant completionTime, + Controller controller) { + return withCompletion(runId, ApplicationVersion.unknown, jobError, completionTime, controller); + } + + public JobStatus withCompletion(long runId, ApplicationVersion applicationVersion, + Optional<DeploymentJobs.JobError> jobError, Instant completionTime, + Controller controller) { Version version; - ApplicationVersion applicationVersion; boolean upgrade; String reason; if (type == DeploymentJobs.JobType.component) { // not triggered by us version = controller.systemVersion(); - applicationVersion = ApplicationVersion.unknown; upgrade = false; reason = "Application commit"; - } - else if (! lastTriggered.isPresent()) { + } else if (! lastTriggered.isPresent()) { throw new IllegalStateException("Got notified about completion of " + this + ", but that has neither been triggered nor deployed"); - } - else { + } else { version = lastTriggered.get().version(); applicationVersion = lastTriggered.get().applicationVersion(); upgrade = lastTriggered.get().upgrade(); @@ -197,7 +201,7 @@ public class JobStatus { /** Returns the Vespa version used on this run */ public Version version() { return version; } - /** Returns the application version used for this run, or empty when not known */ + /** Returns the application version used in this run */ public ApplicationVersion applicationVersion() { return applicationVersion; } /** Returns a human-readable reason for this particular job run */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index 1beab1307c1..067d8d41f53 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -78,7 +78,8 @@ public class DeploymentTrigger { */ public void triggerFromCompletion(JobReport report) { applications().lockOrThrow(report.applicationId(), application -> { - application = application.withJobCompletion(report, clock.instant(), controller); + ApplicationVersion applicationVersion = applicationVersionFrom(report); + application = application.withJobCompletion(report, applicationVersion, clock.instant(), controller); application = application.withProjectId(report.projectId()); // Handle successful starting and ending @@ -87,20 +88,17 @@ public class DeploymentTrigger { if (acceptNewApplicationVersionNow(application)) { // Set this as the change we are doing, unless we are already pushing a platform change if ( ! ( application.change().platform().isPresent())) { - ApplicationVersion applicationVersion = ApplicationVersion.unknown; - if (report.sourceRevision().isPresent()) - applicationVersion = ApplicationVersion.from(report.sourceRevision().get(), report.buildNumber()); - application = application.withDeploying(Change.of(applicationVersion)); + application = application.withChange(Change.of(applicationVersion)); } } else { // postpone - applications().store(application.withOutstandingChange(true)); + applications().store(application.withOutstandingChange(Change.of(applicationVersion))); return; } } else if (deploymentComplete(application)) { // change completed - application = application.withDeploying(Change.empty()); + application = application.withChange(Change.empty()); } } @@ -139,7 +137,7 @@ public class DeploymentTrigger { } if (change.application().isPresent()) { // If we don't yet know the application version we are deploying, then we are not complete - if (change.application().get() == ApplicationVersion.unknown) return false; + if (change.application().get().isUnknown()) return false; if ( ! change.application().get().equals(deployment.applicationVersion())) return false; } } @@ -260,9 +258,9 @@ public class DeploymentTrigger { if (application.change().isPresent() && ! application.deploymentJobs().hasFailures()) throw new IllegalArgumentException("Could not start " + change + " on " + application + ": " + application.change() + " is already in progress"); - application = application.withDeploying(change); + application = application.withChange(change); if (change.application().isPresent()) - application = application.withOutstandingChange(false); + application = application.withOutstandingChange(Change.empty()); application = trigger(JobType.systemTest, application, false, change.toString()); applications().store(application); }); @@ -276,7 +274,7 @@ public class DeploymentTrigger { public void cancelChange(ApplicationId applicationId) { applications().lockOrThrow(applicationId, application -> { buildSystem.removeJobs(application.id()); - applications().store(application.withDeploying(Change.empty())); + applications().store(application.withChange(Change.empty())); }); } @@ -355,10 +353,17 @@ public class DeploymentTrigger { application.change(), clock.instant(), application.deployVersionFor(jobType, controller), - application.deployApplicationVersion(jobType, controller).orElse(ApplicationVersion.unknown), + application.deployApplicationVersionFor(jobType, controller, false) + .orElse(ApplicationVersion.unknown), reason); } + /** Create application version from job report */ + private ApplicationVersion applicationVersionFrom(JobReport report) { + return report.sourceRevision().map(sr -> ApplicationVersion.from(sr, report.buildNumber())) + .orElse(ApplicationVersion.unknown); + } + /** Returns true if the given proposed job triggering should be effected */ private boolean allowedTriggering(JobType jobType, LockedApplication application) { // Note: We could make a more fine-grained and more correct determination about whether to block diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployer.java index 3dd63a511e1..9f1738f9560 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployer.java @@ -4,8 +4,6 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.application.ApplicationList; -import com.yahoo.vespa.hosted.controller.application.ApplicationVersion; -import com.yahoo.vespa.hosted.controller.application.Change; import java.time.Duration; @@ -24,9 +22,9 @@ public class OutstandingChangeDeployer extends Maintainer { protected void maintain() { ApplicationList applications = ApplicationList.from(controller().applications().asList()).notPullRequest(); for (Application application : applications.asList()) { - if (application.hasOutstandingChange() && ! application.change().isPresent()) + if (!application.change().isPresent() && application.outstandingChange().isPresent()) controller().applications().deploymentTrigger().triggerChange(application.id(), - Change.of(ApplicationVersion.unknown)); + application.outstandingChange()); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java index 652f95a2d13..670338d10e5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java @@ -6,15 +6,16 @@ import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; +import com.yahoo.slime.Type; import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.api.integration.MetricsService.ApplicationMetrics; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationVersion; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.ClusterInfo; @@ -61,6 +62,7 @@ public class ApplicationSerializer { private final String environmentField = "environment"; private final String regionField = "region"; private final String deployTimeField = "deployTime"; + private final String applicationBuildNumberField = "applicationBuildNumber"; private final String applicationPackageRevisionField = "applicationPackageRevision"; private final String applicationPackageHashField = "applicationPackageHash"; private final String sourceRevisionField = "sourceRevision"; @@ -125,8 +127,8 @@ public class ApplicationSerializer { root.setString(validationOverridesField, application.validationOverrides().xmlForm()); deploymentsToSlime(application.deployments().values(), root.setArray(deploymentsField)); toSlime(application.deploymentJobs(), root.setObject(deploymentJobsField)); - toSlime(application.change(), root); - root.setBool(outstandingChangeField, application.hasOutstandingChange()); + toSlime(application.change(), root, deployingField); + toSlime(application.outstandingChange(), root, outstandingChangeField); application.ownershipIssueId().ifPresent(issueId -> root.setString(ownershipIssueIdField, issueId.value())); root.setDouble(queryQualityField, application.metrics().queryServiceQuality()); root.setDouble(writeQualityField, application.metrics().writeServiceQuality()); @@ -198,9 +200,15 @@ public class ApplicationSerializer { } private void toSlime(ApplicationVersion applicationVersion, Cursor object) { - object.setString(applicationPackageHashField, applicationVersion.id()); - if (applicationVersion.source().isPresent()) + if (applicationVersion.buildNumber().isPresent() && applicationVersion.source().isPresent()) { + object.setLong(applicationBuildNumberField, applicationVersion.buildNumber().get()); toSlime(applicationVersion.source().get(), object.setObject(sourceRevisionField)); + } else if (applicationVersion.applicationPackageHash().isPresent()) { // TODO: Remove after 2018-03-01 + object.setString(applicationPackageHashField, applicationVersion.applicationPackageHash().get()); + if (applicationVersion.source().isPresent()){ + toSlime(applicationVersion.source().get(), object.setObject(sourceRevisionField)); + } + } } private void toSlime(SourceRevision sourceRevision, Cursor object) { @@ -236,20 +244,19 @@ public class ApplicationSerializer { Cursor object = parent.setObject(jobRunObjectName); object.setLong(jobRunIdField, jobRun.get().id()); object.setString(versionField, jobRun.get().version().toString()); - if ( jobRun.get().applicationVersion() != ApplicationVersion.unknown) - toSlime(jobRun.get().applicationVersion(), object.setObject(revisionField)); + toSlime(jobRun.get().applicationVersion(), object.setObject(revisionField)); object.setBool(upgradeField, jobRun.get().upgrade()); object.setString(reasonField, jobRun.get().reason()); object.setLong(atField, jobRun.get().at().toEpochMilli()); } - private void toSlime(Change deploying, Cursor parentObject) { + private void toSlime(Change deploying, Cursor parentObject, String fieldName) { if ( ! deploying.isPresent()) return; - Cursor object = parentObject.setObject(deployingField); + Cursor object = parentObject.setObject(fieldName); if (deploying.platform().isPresent()) object.setString(versionField, deploying.platform().get().toString()); - if (deploying.application().isPresent() && deploying.application().get() != ApplicationVersion.unknown) + if (deploying.application().isPresent()) toSlime(deploying.application().get(), object); } @@ -264,7 +271,7 @@ public class ApplicationSerializer { List<Deployment> deployments = deploymentsFromSlime(root.field(deploymentsField)); DeploymentJobs deploymentJobs = deploymentJobsFromSlime(root.field(deploymentJobsField)); Change deploying = changeFromSlime(root.field(deployingField)); - boolean outstandingChange = root.field(outstandingChangeField).asBool(); + Change outstandingChange = outstandingChangeFromSlime(root.field(outstandingChangeField)); Optional<IssueId> ownershipIssueId = optionalString(root.field(ownershipIssueIdField)).map(IssueId::from); ApplicationMetrics metrics = new ApplicationMetrics(root.field(queryQualityField).asDouble(), root.field(writeQualityField).asDouble()); @@ -342,10 +349,17 @@ public class ApplicationSerializer { private ApplicationVersion applicationVersionFromSlime(Inspector object) { if ( ! object.valid()) return ApplicationVersion.unknown; - String applicationPackageHash = object.field(applicationPackageHashField).asString(); + Optional<String> applicationPackageHash = optionalString(object.field(applicationPackageHashField)); + Optional<Long> applicationBuildNumber = optionalLong(object.field(applicationBuildNumberField)); Optional<SourceRevision> sourceRevision = sourceRevisionFromSlime(object.field(sourceRevisionField)); - return sourceRevision.isPresent() ? ApplicationVersion.from(applicationPackageHash, sourceRevision.get()) - : ApplicationVersion.from(applicationPackageHash); + if (applicationPackageHash.isPresent()) { // TODO: Remove after 2018-03-01 + return sourceRevision.map(sr -> ApplicationVersion.from(applicationPackageHash.get(), sr)) + .orElseGet(() -> ApplicationVersion.from(applicationPackageHash.get())); + } + if (!sourceRevision.isPresent() || !applicationBuildNumber.isPresent()) { + return ApplicationVersion.unknown; + } + return ApplicationVersion.from(sourceRevision.get(), applicationBuildNumber.get()); } private Optional<SourceRevision> sourceRevisionFromSlime(Inspector object) { @@ -369,13 +383,23 @@ public class ApplicationSerializer { Change change = Change.empty(); if (versionFieldValue.valid()) change = Change.of(Version.fromString(versionFieldValue.asString())); - if (object.field(applicationPackageHashField).valid()) + if (object.field(applicationBuildNumberField).valid() || + object.field(applicationPackageHashField).valid()) // TODO: Remove after 2018-03-01 change = change.with(applicationVersionFromSlime(object)); if ( ! change.isPresent()) // A deploy object with no fields -> unknown application change change = Change.of(ApplicationVersion.unknown); return change; } + // TODO: Remove and inline after 2018-03-01 + private Change outstandingChangeFromSlime(Inspector object) { + if (object.type() == Type.BOOL) { + boolean outstandingChange = object.asBool(); + return outstandingChange ? Change.of(ApplicationVersion.unknown) : Change.empty(); + } + return changeFromSlime(object); + } + private List<JobStatus> jobStatusListFromSlime(Inspector array) { List<JobStatus> jobStatusList = new ArrayList<>(); array.traverse((ArrayTraverser) (int i, Inspector item) -> jobStatusList.add(jobStatusFromSlime(item))); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 9019e843c2c..2105ea1d3d9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -475,9 +475,10 @@ public class ApplicationApiHandler extends LoggingRequestHandler { } private void toSlime(ApplicationVersion applicationVersion, Cursor object) { - object.setString("hash", applicationVersion.id()); - if (applicationVersion.source().isPresent()) + if (!applicationVersion.isUnknown()) { + object.setString("hash", applicationVersion.id()); sourceRevisionToSlime(applicationVersion.source(), object.setObject("source")); + } } private void sourceRevisionToSlime(Optional<SourceRevision> revision, Cursor object) { @@ -967,7 +968,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { private void toSlime(JobStatus.JobRun jobRun, Cursor object) { object.setLong("id", jobRun.id()); object.setString("version", jobRun.version().toFullString()); - if (jobRun.applicationVersion() != ApplicationVersion.unknown) + if (!jobRun.applicationVersion().isUnknown()) toSlime(jobRun.applicationVersion(), object.setObject("revision")); object.setString("reason", jobRun.reason()); object.setLong("at", jobRun.at().toEpochMilli()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ArtifactRepositoryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ArtifactRepositoryMock.java index efbc10e8deb..cfe60cd3e96 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ArtifactRepositoryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ArtifactRepositoryMock.java @@ -14,11 +14,22 @@ import java.util.Objects; */ public class ArtifactRepositoryMock implements ArtifactRepository { - private final Map<Integer, byte[]> repository = new HashMap<>(); + private final Map<Integer, Artifact> repository = new HashMap<>(); public ArtifactRepositoryMock put(ApplicationId applicationId, ApplicationPackage applicationPackage, String applicationVersion) { - repository.put(artifactHash(applicationId, applicationVersion), applicationPackage.zippedContent()); + repository.put(artifactHash(applicationId, applicationVersion), + new Artifact(applicationPackage.zippedContent())); + return this; + } + + public int hits(ApplicationId applicationId, String applicationVersion) { + Artifact artifact = repository.get(artifactHash(applicationId, applicationVersion)); + return artifact == null ? 0 : artifact.hits; + } + + public ArtifactRepository resetHits() { + repository.values().forEach(Artifact::resetHits); return this; } @@ -29,11 +40,32 @@ public class ArtifactRepositoryMock implements ArtifactRepository { throw new IllegalArgumentException("No application package found for " + applicationId + " with version " + applicationVersion); } - return repository.get(artifactHash); + Artifact artifact = repository.get(artifactHash); + artifact.recordHit(); + return artifact.data; } private static int artifactHash(ApplicationId applicationId, String applicationVersion) { return Objects.hash(applicationId, applicationVersion); } + private class Artifact { + + private final byte[] data; + private int hits = 0; + + private Artifact(byte[] data) { + this.data = data; + } + + private void recordHit() { + hits++; + } + + private void resetHits() { + hits = 0; + } + + } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ConfigServerClientMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ConfigServerClientMock.java index bd5aef1ec3a..08f96195d2a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ConfigServerClientMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ConfigServerClientMock.java @@ -42,8 +42,8 @@ public class ConfigServerClientMock extends AbstractComponent implements ConfigS private final Map<URI, Version> versions = new HashMap<>(); private Version defaultVersion = new Version(6, 1, 0); - private RuntimeException prepareException = null; private Version lastPrepareVersion = null; + private RuntimeException prepareException = null; /** The version given in the previous prepare call, or empty if no call has been made */ public Optional<Version> lastPrepareVersion() { @@ -72,7 +72,11 @@ public class ConfigServerClientMock extends AbstractComponent implements ConfigS public void setDefaultVersion(Version version) { this.defaultVersion = version; } - + + public Version getDefaultVersion() { + return defaultVersion; + } + @Override public PreparedApplication prepare(DeploymentId deployment, DeployOptions deployOptions, Set<String> rotationCnames, Set<String> rotationNames, byte[] content) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index 916494a0636..1f2a9c4e910 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -102,7 +102,7 @@ public class ControllerTest { .build(); // staging job - succeeding - Version version1 = Version.fromString("6.1"); // Set in config server mock + Version version1 = tester.defaultVespaVersion(); Application app1 = tester.createApplication("app1", "tenant1", 1, 11L); tester.notifyJobCompletion(component, app1, true); assertEquals("Application version is currently not known", @@ -205,129 +205,64 @@ public class ControllerTest { assertNull("Deployment job was removed", applications.require(app1.id()).deploymentJobs().jobStatus().get(productionCorpUsEast1)); } - // TODO: Replace above test with this one after introducing new application version number @Test - public void testDeploymentWithApplicationVersion() { - // Setup system + public void testDeploymentApplicationVersion() { DeploymentTester tester = new DeploymentTester(); - ApplicationController applications = tester.controller().applications(); - Version version1 = Version.fromString("6.1"); // Set in config server mock - Application app1 = tester.createApplication("app1", "tenant1", 1, 11L); - - // Component runs, uploads artifact and notifies completion + Application app = tester.createApplication("app1", "tenant1", 1, 11L); ApplicationPackage applicationPackage = new ApplicationPackageBuilder() .environment(Environment.prod) .region("corp-us-east-1") .region("us-east-3") .build(); - SourceRevision source = new SourceRevision("repo", "branch", "deadbeef"); - String expectedVersionString = "1.0.37-deadbeef"; - tester.artifactRepository().put(app1.id(), applicationPackage, expectedVersionString); - tester.notifyJobCompletion(component, app1, Optional.empty(), Optional.of(source), 37); - ApplicationVersion expectedVersion = ApplicationVersion.from(source, 37); - assertEquals(expectedVersionString, tester.controller().applications() - .require(app1.id()) - .change().application().get().id()); - - // Deploy without application package - tester.deployAndNotify(app1, true, systemTest); - tester.deployAndNotify(app1, true, stagingTest); - assertEquals(4, applications.require(app1.id()).deploymentJobs().jobStatus().size()); - assertStatus(JobStatus.initial(stagingTest) - .withTriggering(version1, expectedVersion, false, "", tester.clock().instant().minus(Duration.ofMillis(1))) - .withCompletion(42, Optional.empty(), tester.clock().instant(), tester.controller()), app1.id(), tester.controller()); + SourceRevision source = new SourceRevision("repo", "master", "commit1"); - // Causes first deployment job to be triggered - assertStatus(JobStatus.initial(productionCorpUsEast1) - .withTriggering(version1, expectedVersion, false, "", tester.clock().instant()), app1.id(), tester.controller()); - tester.clock().advance(Duration.ofSeconds(1)); + ApplicationVersion applicationVersion = ApplicationVersion.from(source, 101); + tester.artifactRepository().put(app.id(), applicationPackage, applicationVersion.id()); + runDeployment(tester, app.id(), applicationVersion, Optional.empty(), Optional.of(source), 101); + assertEquals("Artifact is downloaded twice in staging and once for other zones", 5, + tester.artifactRepository().hits(app.id(), applicationVersion.id())); - // production job (failing) - tester.deployAndNotify(app1, false, productionCorpUsEast1); - assertEquals(4, applications.require(app1.id()).deploymentJobs().jobStatus().size()); - - JobStatus expectedJobStatus = JobStatus.initial(productionCorpUsEast1) - .withTriggering(version1, expectedVersion, false, "", tester.clock().instant()) - .withCompletion(42, Optional.of(JobError.unknown), tester.clock().instant(), tester.controller()); - - assertStatus(expectedJobStatus, app1.id(), tester.controller()); - - // Simulate restart - tester.restartController(); - applications = tester.controller().applications(); - - assertNotNull(tester.controller().tenants().tenant(new TenantId("tenant1"))); - assertNotNull(applications.get(ApplicationId.from(TenantName.from("tenant1"), - ApplicationName.from("application1"), - InstanceName.from("default")))); - assertEquals(4, applications.require(app1.id()).deploymentJobs().jobStatus().size()); - - - tester.clock().advance(Duration.ofHours(1)); - - tester.notifyJobCompletion(productionCorpUsEast1, app1, false); // Need to complete the job, or new jobs won't start. - - // Component is triggered again - tester.artifactRepository().put(app1.id(), applicationPackage, "1.0.38-deadbeef"); - tester.notifyJobCompletion(component, app1, Optional.empty(), Optional.of(source), 38); - tester.deployAndNotify(app1, Optional.empty(), true, false, systemTest); - expectedVersion = ApplicationVersion.from(source, 38); - assertStatus(JobStatus.initial(systemTest) - .withTriggering(version1, expectedVersion, false, "", tester.clock().instant().minus(Duration.ofMillis(1))) - .withCompletion(42, Optional.empty(), tester.clock().instant(), tester.controller()), app1.id(), tester.controller()); - tester.deployAndNotify(app1, Optional.empty(), true, true, stagingTest); - - // production job succeeding now - tester.deployAndNotify(app1, Optional.empty(), true, true, productionCorpUsEast1); - expectedJobStatus = expectedJobStatus - .withTriggering(version1, expectedVersion, false, "", tester.clock().instant().minus(Duration.ofMillis(1))) - .withCompletion(42, Optional.empty(), tester.clock().instant(), tester.controller()); - assertStatus(expectedJobStatus, app1.id(), tester.controller()); - - // causes triggering of next production job - assertStatus(JobStatus.initial(productionUsEast3) - .withTriggering(version1, expectedVersion, false, "", tester.clock().instant()), - app1.id(), tester.controller()); - tester.deployAndNotify(app1, Optional.empty(), true, true, productionUsEast3); - - assertEquals(5, applications.get(app1.id()).get().deploymentJobs().jobStatus().size()); - - // prod zone removal is not allowed - applicationPackage = new ApplicationPackageBuilder() - .environment(Environment.prod) - .region("us-east-3") - .build(); - tester.artifactRepository().put(app1.id(), applicationPackage, "1.0.56-cafed00d"); - source = new SourceRevision("repo", "branch", "cafed00d"); - tester.notifyJobCompletion(component, app1, Optional.empty(), Optional.of(source), 56); - try { - tester.deploy(systemTest, app1, Optional.empty(), false); - fail("Expected exception due to unallowed production deployment removal"); - } - catch (IllegalArgumentException e) { - assertEquals("deployment-removal: application 'tenant1.app1' is deployed in corp-us-east-1, but does not include this zone in deployment.xml", e.getMessage()); - } - assertNotNull("Zone was not removed", - applications.require(app1.id()).deployments().get(productionCorpUsEast1.zone(SystemName.main).get())); - JobStatus jobStatus = applications.require(app1.id()).deploymentJobs().jobStatus().get(productionCorpUsEast1); - assertNotNull("Deployment job was not removed", jobStatus); - assertEquals(42, jobStatus.lastCompleted().get().id()); - assertEquals("staging-test completed", jobStatus.lastCompleted().get().reason()); + // Application is upgraded. This makes deployment orchestration pick the last successful application version in + // zones which do not have permanent deployments, e.g. test and staging + runUpgrade(tester, app.id(), applicationVersion); + } - // prod zone removal is allowed with override - applicationPackage = new ApplicationPackageBuilder() - .allow(ValidationId.deploymentRemoval) - .upgradePolicy("default") + @Test + public void testDeploymentApplicationVersionMigration() { + DeploymentTester tester = new DeploymentTester(); + Application app = tester.createApplication("app1", "tenant1", 1, 11L); + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() .environment(Environment.prod) + .region("corp-us-east-1") .region("us-east-3") .build(); - tester.artifactRepository().put(app1.id(), applicationPackage, "1.0.103-c00ffefe"); - source = new SourceRevision("repo", "branch", "c00ffefe"); - tester.notifyJobCompletion(component, app1, Optional.empty(), Optional.of(source), 103); - tester.deployAndNotify(app1, Optional.empty(), true, true, systemTest); - assertNull("Zone was removed", - applications.require(app1.id()).deployments().get(productionCorpUsEast1.zone(SystemName.main).get())); - assertNull("Deployment job was removed", applications.require(app1.id()).deploymentJobs().jobStatus().get(productionCorpUsEast1)); + SourceRevision source = new SourceRevision("repo", "master", "commit1"); + + // Scenario 1: Old fashioned deployment. Simulates existing production deployments + ApplicationVersion v0 = ApplicationVersion.from(applicationPackage.hash(), source); + runDeployment(tester, app.id(), v0, Optional.of(applicationPackage), Optional.empty(), 100); + assertEquals("Nothing downloaded from repository", 0, + tester.artifactRepository().hits(app.id(), v0.id())); + + // Scenario 2: New application version number is reported and package is downloaded by controller. In staging, + // the application package from the deployer is used as v0 cannot be downloaded from repository. + ApplicationVersion v1 = ApplicationVersion.from(source, 101); + tester.artifactRepository().put(app.id(), applicationPackage, v1.id()); + runDeployment(tester, app.id(), v1, Optional.of(applicationPackage), Optional.of(source), 101); + assertEquals("Artifact is downloaded once per zone", 4, + tester.artifactRepository().hits(app.id(), v1.id())); + assertEquals("v0 is never downloaded", 0, + tester.artifactRepository().hits(app.id(), v0.id())); + tester.artifactRepository().resetHits(); + + // Scenario 3: Both application versions are available in repository + ApplicationVersion v2 = ApplicationVersion.from(source, 102); + tester.artifactRepository().put(app.id(), applicationPackage, v2.id()); + runDeployment(tester, app.id(), v2, Optional.empty(), Optional.of(source), 102); + assertEquals("Previous artifact is downloaded once", 1, + tester.artifactRepository().hits(app.id(), v1.id())); + assertEquals("Artifact is downloaded once per zone", 4, + tester.artifactRepository().hits(app.id(), v2.id())); } @Test @@ -666,19 +601,20 @@ public class ControllerTest { @Test public void testDeployUntestedChangeFails() { - ControllerTester tester = new ControllerTester(); + DeploymentTester tester = new DeploymentTester(); ApplicationController applications = tester.controller().applications(); - TenantId tenant = tester.createTenant("tenant1", "domain1", 11L); - Application app = tester.createApplication(tenant, "app1", "default", 1); + TenantId tenant = tester.controllerTester().createTenant("tenant1", "domain1", 11L); + Application app = tester.controllerTester().createApplication(tenant, "app1", "default", 1); + tester.deployCompletely(app, applicationPackage); tester.controller().applications().lockOrThrow(app.id(), application -> { - application = application.withDeploying(Change.of(Version.fromString("6.3"))); + application = application.withChange(Change.of(Version.fromString("6.3"))); applications.store(application); try { - tester.deploy(app, ZoneId.from("prod", "us-east-3")); + tester.controllerTester().deploy(app, ZoneId.from("prod", "corp-us-east-1"), applicationPackage); fail("Expected exception"); } catch (IllegalArgumentException e) { - assertEquals("Rejecting deployment of application 'tenant1.app1' to zone prod.us-east-3 as upgrade to 6.3 is not tested", e.getMessage()); + assertEquals("Rejecting deployment of application 'tenant1.app1' to zone prod.corp-us-east-1 as upgrade to 6.3 is not tested", e.getMessage()); } }); } @@ -919,6 +855,60 @@ public class ControllerTest { } + private void runUpgrade(DeploymentTester tester, ApplicationId application, ApplicationVersion version) { + Version next = Version.fromString("6.2"); + tester.upgradeSystem(next); + runDeployment(tester, tester.applications().require(application), version, Optional.of(next), Optional.empty()); + } + + private void runDeployment(DeploymentTester tester, ApplicationId application, ApplicationVersion version, + Optional<ApplicationPackage> applicationPackage, Optional<SourceRevision> sourceRevision, + long initialBuildNumber) { + Application app = tester.applications().require(application); + tester.notifyJobCompletion(component, app, Optional.empty(), sourceRevision, initialBuildNumber); + ApplicationVersion change = sourceRevision.map(sr -> ApplicationVersion.from(sr, initialBuildNumber)) + .orElse(ApplicationVersion.unknown); + assertEquals(change.id(), tester.controller().applications() + .require(application) + .change().application().get().id()); + runDeployment(tester, app, version, Optional.empty(), applicationPackage); + } + + private void runDeployment(DeploymentTester tester, Application app, ApplicationVersion version, + Optional<Version> upgrade, Optional<ApplicationPackage> applicationPackage) { + Version vespaVersion = upgrade.orElseGet(tester::defaultVespaVersion); + + // Deploy in test + tester.deployAndNotify(app, applicationPackage, true, true, systemTest); + tester.deployAndNotify(app, applicationPackage, true, true, stagingTest); + assertStatus(JobStatus.initial(stagingTest) + .withTriggering(vespaVersion, version, upgrade.isPresent(), "", + tester.clock().instant().minus(Duration.ofMillis(1))) + .withCompletion(42, Optional.empty(), tester.clock().instant(), + tester.controller()), app.id(), tester.controller()); + + // Deploy in production + tester.deployAndNotify(app, applicationPackage, true, true, productionCorpUsEast1); + assertStatus(JobStatus.initial(productionCorpUsEast1) + .withTriggering(vespaVersion, version, upgrade.isPresent(), "", + tester.clock().instant().minus(Duration.ofMillis(1))) + .withCompletion(42, Optional.empty(), tester.clock().instant(), + tester.controller()), app.id(), tester.controller()); + tester.deployAndNotify(app, applicationPackage, true, true, productionUsEast3); + assertStatus(JobStatus.initial(productionUsEast3) + .withTriggering(vespaVersion, version, upgrade.isPresent(), "", + tester.clock().instant().minus(Duration.ofMillis(1))) + .withCompletion(42, Optional.empty(), tester.clock().instant(), + tester.controller()), app.id(), tester.controller()); + + // Verify deployed version + app = tester.controller().applications().require(app.id()); + for (Deployment deployment : app.productionDeployments().values()) { + assertEquals(version, deployment.applicationVersion()); + upgrade.ifPresent(v -> assertEquals(v, deployment.version())); + } + } + @Test public void testDeploymentOfNewInstanceWithIllegalApplicationName() { ControllerTester tester = new ControllerTester(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java index d0dfe825558..c923542b8c9 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java @@ -104,11 +104,15 @@ public class DeploymentTester { } public void upgradeSystem(Version version) { - controllerTester().configServer().setDefaultVersion(version); + configServer().setDefaultVersion(version); updateVersionStatus(version); upgrader().maintain(); } + public Version defaultVespaVersion() { + return configServer().getDefaultVersion(); + } + public Application createApplication(String applicationName, String tenantName, long projectId, Long propertyId) { TenantId tenant = tester.createTenant(tenantName, UUID.randomUUID().toString(), propertyId); return tester.createApplication(tenant, applicationName, "default", projectId); @@ -253,10 +257,6 @@ public class DeploymentTester { deployCurrentVersion)); } - public void deployAndNotify(Application application, boolean success, JobType... job) { - deployAndNotify(application, Optional.empty(), success, true, job); - } - public void deployAndNotify(Application application, String upgradePolicy, boolean success, JobType... jobs) { deployAndNotify(application, applicationPackage(upgradePolicy), success, true, jobs); } @@ -276,9 +276,17 @@ public class DeploymentTester { consumeJobs(application, expectOnlyTheseJobs, jobs); for (JobType job : jobs) { if (success) { + // Staging deploys twice, once with current version and once with new version + if (job == JobType.stagingTest) { + deploy(job, application, applicationPackage, true); + } deploy(job, application, applicationPackage, false); } notifyJobCompletion(job, application, success); + // Deactivate test deployments after deploy. This replicates the behaviour of the tenant pipeline + if (job.isTest()) { + controller().applications().deactivate(application, job.zone(controller().system()).get()); + } } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java index 5c61e43f9cf..029eb335d82 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java @@ -320,7 +320,7 @@ public class DeploymentTriggerTest { new JobControl(tester.controllerTester().curator())); LockedApplication app = (LockedApplication)tester.createAndDeploy("default0", 3, "default"); // Store that we are upgrading but don't start the system-tests job - tester.controller().applications().store(app.withDeploying(Change.of(Version.fromString("6.2")))); + tester.controller().applications().store(app.withChange(Change.of(Version.fromString("6.2")))); assertEquals(0, tester.buildSystem().jobs().size()); readyJobsTrigger.run(); assertEquals(1, tester.buildSystem().jobs().size()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java index 3d34e78c759..4bed276fcac 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java @@ -2,15 +2,18 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; +import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.api.integration.BuildService; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; +import com.yahoo.vespa.hosted.controller.application.SourceRevision; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; import org.junit.Test; import java.time.Duration; import java.util.List; +import java.util.Optional; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -34,9 +37,15 @@ public class OutstandingChangeDeployerTest { tester.deploymentTrigger().triggerChange(tester.application("app1").id(), Change.of(version)); assertEquals(Change.of(version), tester.application("app1").change()); - assertFalse(tester.application("app1").hasOutstandingChange()); - tester.notifyJobCompletion(DeploymentJobs.JobType.component, tester.application("app1"), true); - assertTrue(tester.application("app1").hasOutstandingChange()); + assertFalse(tester.application("app1").outstandingChange().isPresent()); + tester.notifyJobCompletion(DeploymentJobs.JobType.component, tester.application("app1"), + Optional.empty(), Optional.of(new SourceRevision("repo", "master", + "cafed00d")), + 42); + + Application app = tester.application("app1"); + assertTrue(app.outstandingChange().isPresent()); + assertEquals("1.0.42-cafed00d", app.outstandingChange().application().get().id()); assertEquals(1, tester.buildSystem().jobs().size()); deployer.maintain(); @@ -50,7 +59,7 @@ public class OutstandingChangeDeployerTest { assertEquals(1, jobs.size()); assertEquals(11, jobs.get(0).projectId()); assertEquals(DeploymentJobs.JobType.systemTest.jobName(), jobs.get(0).jobName()); - assertFalse(tester.application("app1").hasOutstandingChange()); + assertFalse(tester.application("app1").outstandingChange().isPresent()); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java index 9a945281789..9c19cdb66ac 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java @@ -62,9 +62,9 @@ public class ApplicationSerializerTest { "</validation-overrides>"); List<Deployment> deployments = new ArrayList<>(); - ApplicationVersion applicationVersion1 = ApplicationVersion.from("appHash1"); + ApplicationVersion applicationVersion1 = ApplicationVersion.from(new SourceRevision("repo1", "branch1", "commit1"), 31); ApplicationVersion applicationVersion2 = ApplicationVersion - .from("appHash2", new SourceRevision("repo1", "branch1", "commit1")); + .from(new SourceRevision("repo1", "branch1", "commit1"), 32); deployments.add(new Deployment(zone1, applicationVersion1, Version.fromString("1.2.3"), Instant.ofEpochMilli(3))); // One deployment without cluster info and utils deployments.add(new Deployment(zone2, applicationVersion2, Version.fromString("1.2.3"), Instant.ofEpochMilli(5), createClusterUtils(3, 0.2), createClusterInfo(3, 4),new DeploymentMetrics(2,3,4,5,6))); @@ -86,7 +86,7 @@ public class ApplicationSerializerTest { validationOverrides, deployments, deploymentJobs, Change.of(Version.fromString("6.7")), - true, + Change.of(ApplicationVersion.from(new SourceRevision("repo", "master", "deadcafe"), 42)), Optional.of(IssueId.from("1234")), new MetricsService.ApplicationMetrics(0.5, 0.9), Optional.of(new RotationId("my-rotation"))); @@ -113,7 +113,7 @@ public class ApplicationSerializerTest { assertEquals( original.deploymentJobs().jobStatus().get(DeploymentJobs.JobType.stagingTest), serialized.deploymentJobs().jobStatus().get(DeploymentJobs.JobType.stagingTest)); - assertEquals(original.hasOutstandingChange(), serialized.hasOutstandingChange()); + assertEquals(original.outstandingChange(), serialized.outstandingChange()); assertEquals(original.ownershipIssueId(), serialized.ownershipIssueId()); @@ -148,22 +148,29 @@ public class ApplicationSerializerTest { assertEquals(6, serialized.deployments().get(zone2).metrics().writeLatencyMillis(), Double.MIN_VALUE); { // test more deployment serialization cases - Application original2 = writable(original).withDeploying(Change.of(ApplicationVersion.from("hash1"))); + Application original2 = writable(original).withChange(Change.of(ApplicationVersion.from("hash1"))); Application serialized2 = applicationSerializer.fromSlime(applicationSerializer.toSlime(original2)); assertEquals(original2.change(), serialized2.change()); assertEquals(serialized2.change().application().get().source(), original2.change().application().get().source()); - Application original3 = writable(original).withDeploying(Change.of(ApplicationVersion.from("hash1", - new SourceRevision("a", "b", "c")))); + Application original3 = writable(original).withChange(Change.of(ApplicationVersion.from("hash1", + new SourceRevision("a", "b", "c")))); Application serialized3 = applicationSerializer.fromSlime(applicationSerializer.toSlime(original3)); - assertEquals(original3.change(), serialized2.change()); + assertEquals(original3.change(), serialized3.change()); assertEquals(serialized3.change().application().get().source(), original3.change().application().get().source()); - - Application original4 = writable(original).withDeploying(Change.empty()); + Application original4 = writable(original).withChange(Change.empty()); Application serialized4 = applicationSerializer.fromSlime(applicationSerializer.toSlime(original4)); assertEquals(original4.change(), serialized4.change()); + + Application original5 = writable(original).withChange(Change.of(ApplicationVersion.unknown)); + Application serialized5 = applicationSerializer.fromSlime(applicationSerializer.toSlime(original5)); + assertEquals(original5.change(), serialized5.change()); + + Application original6 = writable(original).withOutstandingChange(Change.of(ApplicationVersion.unknown)); + Application serialized6 = applicationSerializer.fromSlime(applicationSerializer.toSlime(original6)); + assertEquals(original6.outstandingChange(), serialized6.outstandingChange()); } } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java index 57e7070296f..312f8a65364 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java @@ -9,14 +9,22 @@ import com.yahoo.document.DocumentTypeManager; import com.yahoo.document.DocumentTypeManagerConfigurer; import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; import com.yahoo.documentapi.metrics.DocumentProtocolMetricSet; -import com.yahoo.messagebus.*; +import com.yahoo.messagebus.ErrorCode; +import com.yahoo.messagebus.Protocol; +import com.yahoo.messagebus.Reply; +import com.yahoo.messagebus.Routable; import com.yahoo.messagebus.metrics.MetricSet; import com.yahoo.messagebus.routing.RoutingContext; import com.yahoo.messagebus.routing.RoutingNodeIterator; import com.yahoo.messagebus.routing.RoutingPolicy; import com.yahoo.text.Utf8String; -import java.util.*; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Set; import java.util.logging.Logger; /** @@ -207,7 +215,7 @@ public class DocumentProtocol implements Protocol { * NORMAL categories. Traffic in the HIGH end will be usually be prioritized over important maintenance operations. * Traffic in the LOW end will be prioritized after these operations.</p> */ - public static enum Priority { + public enum Priority { HIGHEST(0), VERY_HIGH(1), HIGH_1(2), @@ -300,7 +308,6 @@ public class DocumentProtocol implements Protocol { VersionSpecification version52 = new VersionSpecification(5, 115); VersionSpecification version6 = new VersionSpecification(6, 999); // TODO change once stable protocol - // TODO ensure version semantics List<VersionSpecification> from50 = Arrays.asList(version50, version51, version52, version6); List<VersionSpecification> from51 = Arrays.asList(version51, version52, version6); List<VersionSpecification> from52 = Arrays.asList(version52, version6); diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages60TestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages60TestCase.java index 6bb9207edd0..b522147124c 100644 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages60TestCase.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages60TestCase.java @@ -122,6 +122,4 @@ public class Messages60TestCase extends Messages52TestCase { } } - // TODO want to test that non-default bucket space fails to encode with old version - } diff --git a/eval/src/apps/tensor_conformance/generate.cpp b/eval/src/apps/tensor_conformance/generate.cpp index f70c472cbcd..f3a99f8a36c 100644 --- a/eval/src/apps/tensor_conformance/generate.cpp +++ b/eval/src/apps/tensor_conformance/generate.cpp @@ -193,6 +193,8 @@ void generate_tensor_concat(TestBuilder &dst) { spec({x(2),y(2),z(3)}, Seq({1.0, 2.0, 3.0, 1.0, 2.0, 3.0, 4.0, 4.0, 4.0, 5.0, 5.0, 5.0}))); dst.add("concat(a,b,x)", {{"a", spec(y(3), Seq({1.0, 2.0, 3.0}))}, {"b", spec(y(2), Seq({4.0, 5.0}))}}, spec({x(2), y(2)}, Seq({1.0, 2.0, 4.0, 5.0}))); + dst.add("concat(concat(a,b,x),concat(c,d,x),y)", {{"a", spec(1.0)}, {"b", spec(2.0)}, {"c", spec(3.0)}, {"d", spec(4.0)}}, + spec({x(2), y(2)}, Seq({1.0, 3.0, 2.0, 4.0}))); } //----------------------------------------------------------------------------- diff --git a/eval/src/apps/tensor_conformance/test_spec.json b/eval/src/apps/tensor_conformance/test_spec.json index 513d5e8e902..335c4bcb939 100644 --- a/eval/src/apps/tensor_conformance/test_spec.json +++ b/eval/src/apps/tensor_conformance/test_spec.json @@ -1235,6 +1235,7 @@ {"expression":"concat(a,b,x)","inputs":{"a":"0x02020178020179023FF0000000000000400000000000000040080000000000004010000000000000","b":"0x020101780240140000000000004018000000000000"},"result":{"expect":"0x02020178040179023FF00000000000004000000000000000400800000000000040100000000000004014000000000000401400000000000040180000000000004018000000000000"}} {"expression":"concat(a,b,x)","inputs":{"a":"0x0201017A033FF000000000000040000000000000004008000000000000","b":"0x020101790240100000000000004014000000000000"},"result":{"expect":"0x0203017802017902017A033FF0000000000000400000000000000040080000000000003FF000000000000040000000000000004008000000000000401000000000000040100000000000004010000000000000401400000000000040140000000000004014000000000000"}} {"expression":"concat(a,b,x)","inputs":{"a":"0x02010179033FF000000000000040000000000000004008000000000000","b":"0x020101790240100000000000004014000000000000"},"result":{"expect":"0x02020178020179023FF0000000000000400000000000000040100000000000004014000000000000"}} +{"expression":"concat(concat(a,b,x),concat(c,d,x),y)","inputs":{"a":"0x02003FF0000000000000","b":"0x02004000000000000000","c":"0x02004008000000000000","d":"0x02004010000000000000"},"result":{"expect":"0x02020178020179023FF0000000000000400800000000000040000000000000004010000000000000"}} {"expression":"rename(a,x,y)","inputs":{"a":"0x02010178053FF00000000000004000000000000000400800000000000040100000000000004014000000000000"},"result":{"expect":"0x02010179053FF00000000000004000000000000000400800000000000040100000000000004014000000000000"}} {"expression":"rename(a,y,x)","inputs":{"a":"0x0202017905017A053FF000000000000040000000000000004008000000000000401000000000000040140000000000004018000000000000401C00000000000040200000000000004022000000000000402400000000000040260000000000004028000000000000402A000000000000402C000000000000402E0000000000004030000000000000403100000000000040320000000000004033000000000000403400000000000040350000000000004036000000000000403700000000000040380000000000004039000000000000"},"result":{"expect":"0x0202017805017A053FF000000000000040000000000000004008000000000000401000000000000040140000000000004018000000000000401C00000000000040200000000000004022000000000000402400000000000040260000000000004028000000000000402A000000000000402C000000000000402E0000000000004030000000000000403100000000000040320000000000004033000000000000403400000000000040350000000000004036000000000000403700000000000040380000000000004039000000000000"}} {"expression":"rename(a,z,x)","inputs":{"a":"0x0202017905017A053FF000000000000040000000000000004008000000000000401000000000000040140000000000004018000000000000401C00000000000040200000000000004022000000000000402400000000000040260000000000004028000000000000402A000000000000402C000000000000402E0000000000004030000000000000403100000000000040320000000000004033000000000000403400000000000040350000000000004036000000000000403700000000000040380000000000004039000000000000"},"result":{"expect":"0x02020178050179053FF000000000000040180000000000004026000000000000403000000000000040350000000000004000000000000000401C00000000000040280000000000004031000000000000403600000000000040080000000000004020000000000000402A0000000000004032000000000000403700000000000040100000000000004022000000000000402C0000000000004033000000000000403800000000000040140000000000004024000000000000402E00000000000040340000000000004039000000000000"}} @@ -1244,4 +1245,4 @@ {"expression":"tensor(x[10])(x+1)","inputs":{},"result":{"expect":"0x020101780A3FF000000000000040000000000000004008000000000000401000000000000040140000000000004018000000000000401C000000000000402000000000000040220000000000004024000000000000"}} {"expression":"tensor(x[5],y[4])(x*4+(y+1))","inputs":{},"result":{"expect":"0x02020178050179043FF000000000000040000000000000004008000000000000401000000000000040140000000000004018000000000000401C00000000000040200000000000004022000000000000402400000000000040260000000000004028000000000000402A000000000000402C000000000000402E00000000000040300000000000004031000000000000403200000000000040330000000000004034000000000000"}} {"expression":"tensor(x[5],y[4])(x==y)","inputs":{},"result":{"expect":"0x02020178050179043FF000000000000000000000000000000000000000000000000000000000000000000000000000003FF000000000000000000000000000000000000000000000000000000000000000000000000000003FF000000000000000000000000000000000000000000000000000000000000000000000000000003FF00000000000000000000000000000000000000000000000000000000000000000000000000000"}} -{"num_tests":1246} +{"num_tests":1247} diff --git a/eval/src/tests/tensor/dense_dot_product_function/dense_dot_product_function_test.cpp b/eval/src/tests/tensor/dense_dot_product_function/dense_dot_product_function_test.cpp index abf51d57b9a..71bbacc7806 100644 --- a/eval/src/tests/tensor/dense_dot_product_function/dense_dot_product_function_test.cpp +++ b/eval/src/tests/tensor/dense_dot_product_function/dense_dot_product_function_test.cpp @@ -31,10 +31,10 @@ makeTensor(size_t numCells, double cellBias) double calcDotProduct(const DenseTensor &lhs, const DenseTensor &rhs) { - size_t numCells = std::min(lhs.cells().size(), rhs.cells().size()); + size_t numCells = std::min(lhs.cellsRef().size(), rhs.cellsRef().size()); double result = 0; for (size_t i = 0; i < numCells; ++i) { - result += (lhs.cells()[i] * rhs.cells()[i]); + result += (lhs.cellsRef()[i] * rhs.cellsRef()[i]); } return result; } diff --git a/eval/src/tests/tensor/dense_tensor_builder/dense_tensor_builder_test.cpp b/eval/src/tests/tensor/dense_tensor_builder/dense_tensor_builder_test.cpp index 61efdbe6d22..ae6166f9d24 100644 --- a/eval/src/tests/tensor/dense_tensor_builder/dense_tensor_builder_test.cpp +++ b/eval/src/tests/tensor/dense_tensor_builder/dense_tensor_builder_test.cpp @@ -10,6 +10,15 @@ using vespalib::IllegalArgumentException; using Builder = DenseTensorBuilder; using vespalib::eval::TensorSpec; using vespalib::eval::ValueType; +using vespalib::ConstArrayRef; + +template <typename T> std::vector<T> make_vector(const ConstArrayRef<T> &ref) { + std::vector<T> vec; + for (const T &t: ref) { + vec.push_back(t); + } + return vec; +} void assertTensor(const std::vector<ValueType::Dimension> &expDims, @@ -18,7 +27,7 @@ assertTensor(const std::vector<ValueType::Dimension> &expDims, { const DenseTensor &realTensor = dynamic_cast<const DenseTensor &>(tensor); EXPECT_EQUAL(ValueType::tensor_type(expDims), realTensor.type()); - EXPECT_EQUAL(expCells, realTensor.cells()); + EXPECT_EQUAL(expCells, make_vector(realTensor.cellsRef())); } void diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor.cpp b/eval/src/vespa/eval/tensor/dense/dense_tensor.cpp index 9693e89bb75..e775385b623 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_tensor.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_tensor.cpp @@ -25,12 +25,12 @@ void checkCellsSize(const DenseTensor &arg) { auto cellsSize = calcCellsSize(arg.fast_type()); - if (arg.cells().size() != cellsSize) { + if (arg.cellsRef().size() != cellsSize) { throw IllegalStateException(make_string("Wrong cell size, " "expected=%zu, " "actual=%zu", cellsSize, - arg.cells().size())); + arg.cellsRef().size())); } } diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor.h b/eval/src/vespa/eval/tensor/dense/dense_tensor.h index c45d3c7ccb6..0da5f570674 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_tensor.h +++ b/eval/src/vespa/eval/tensor/dense/dense_tensor.h @@ -19,7 +19,6 @@ class DenseTensor : public DenseTensorView public: typedef std::unique_ptr<DenseTensor> UP; using Cells = std::vector<double>; - using CellsIterator = DenseTensorCellsIterator; private: eval::ValueType _type; @@ -32,8 +31,6 @@ public: DenseTensor(const eval::ValueType &type_in, Cells &&cells_in); DenseTensor(eval::ValueType &&type_in, Cells &&cells_in); bool operator==(const DenseTensor &rhs) const; - const Cells &cells() const { return _cells; } - }; } diff --git a/eval/src/vespa/eval/tensor/serialization/dense_binary_format.cpp b/eval/src/vespa/eval/tensor/serialization/dense_binary_format.cpp index a2d600aa0c9..feb811a92de 100644 --- a/eval/src/vespa/eval/tensor/serialization/dense_binary_format.cpp +++ b/eval/src/vespa/eval/tensor/serialization/dense_binary_format.cpp @@ -22,7 +22,7 @@ makeValueType(std::vector<eval::ValueType::Dimension> &&dimensions) { } void -DenseBinaryFormat::serialize(nbostream &stream, const DenseTensor &tensor) +DenseBinaryFormat::serialize(nbostream &stream, const DenseTensorView &tensor) { stream.putInt1_4Bytes(tensor.fast_type().dimensions().size()); size_t cellsSize = 1; @@ -31,7 +31,7 @@ DenseBinaryFormat::serialize(nbostream &stream, const DenseTensor &tensor) stream.putInt1_4Bytes(dimension.size); cellsSize *= dimension.size; } - const DenseTensor::Cells &cells = tensor.cells(); + DenseTensorView::CellsRef cells = tensor.cellsRef(); assert(cells.size() == cellsSize); for (const auto &value : cells) { stream << value; diff --git a/eval/src/vespa/eval/tensor/serialization/dense_binary_format.h b/eval/src/vespa/eval/tensor/serialization/dense_binary_format.h index 13efc945880..8019648ffcb 100644 --- a/eval/src/vespa/eval/tensor/serialization/dense_binary_format.h +++ b/eval/src/vespa/eval/tensor/serialization/dense_binary_format.h @@ -10,6 +10,7 @@ class nbostream; namespace tensor { class DenseTensor; +class DenseTensorView; /** * Class for serializing a dense tensor. @@ -17,7 +18,7 @@ class DenseTensor; class DenseBinaryFormat { public: - static void serialize(nbostream &stream, const DenseTensor &tensor); + static void serialize(nbostream &stream, const DenseTensorView &tensor); static std::unique_ptr<DenseTensor> deserialize(nbostream &stream); }; diff --git a/eval/src/vespa/eval/tensor/serialization/typed_binary_format.cpp b/eval/src/vespa/eval/tensor/serialization/typed_binary_format.cpp index c242f44df94..5db4f0aeb12 100644 --- a/eval/src/vespa/eval/tensor/serialization/typed_binary_format.cpp +++ b/eval/src/vespa/eval/tensor/serialization/typed_binary_format.cpp @@ -19,7 +19,7 @@ namespace tensor { void TypedBinaryFormat::serialize(nbostream &stream, const Tensor &tensor) { - if (auto denseTensor = dynamic_cast<const DenseTensor *>(&tensor)) { + if (auto denseTensor = dynamic_cast<const DenseTensorView *>(&tensor)) { stream.putInt1_4Bytes(DENSE_BINARY_FORMAT_TYPE); DenseBinaryFormat::serialize(stream, *denseTensor); } else if (auto wrapped = dynamic_cast<const WrappedSimpleTensor *>(&tensor)) { diff --git a/metrics/src/vespa/metrics/metricmanager.cpp b/metrics/src/vespa/metrics/metricmanager.cpp index 5530e468007..39a9fdefc39 100644 --- a/metrics/src/vespa/metrics/metricmanager.cpp +++ b/metrics/src/vespa/metrics/metricmanager.cpp @@ -150,11 +150,16 @@ MetricManager::removeMetricUpdateHook(UpdateHook& hook) LOG(warning, "Update hook not registered"); } +bool +MetricManager::isInitialized() const { + return static_cast<bool>(_configHandle); +} + void MetricManager::init(const config::ConfigUri & uri, FastOS_ThreadPool& pool, bool startThread) { - if (_configHandle.get()) { + if (isInitialized()) { throw vespalib::IllegalStateException( "The metric manager have already been initialized. " "It can only be initialized once.", VESPA_STRLOC); @@ -164,12 +169,11 @@ MetricManager::init(const config::ConfigUri & uri, FastOS_ThreadPool& pool, _configHandle = _configSubscriber->subscribe<Config>(uri.getConfigId()); _configSubscriber->nextConfig(); configure(getMetricLock(), _configHandle->getConfig()); - LOG(debug, "Starting worker thread, waiting for first " - "iteration to complete."); + LOG(debug, "Starting worker thread, waiting for first iteration to complete."); if (startThread) { Runnable::start(pool); - // Wait for first iteration to have completed, such that it is safe - // to access snapshots afterwards. + // Wait for first iteration to have completed, such that it is safe + // to access snapshots afterwards. vespalib::MonitorGuard sync(_waiter); while (_lastProcessedTime == 0) { sync.wait(1); diff --git a/metrics/src/vespa/metrics/metricmanager.h b/metrics/src/vespa/metrics/metricmanager.h index 7aeca264328..423eb41a787 100644 --- a/metrics/src/vespa/metrics/metricmanager.h +++ b/metrics/src/vespa/metrics/metricmanager.h @@ -273,6 +273,8 @@ public: MemoryConsumption::UP getMemoryConsumption(const MetricLockGuard & guard) const; + bool isInitialized() const; + private: void takeSnapshots(const MetricLockGuard &, time_t timeToProcess); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/TestTaskContext.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TestTaskContext.java index 1fe63e84605..6806e5096c5 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/TestTaskContext.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TestTaskContext.java @@ -1,9 +1,6 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.task.util.file; - -import com.yahoo.vespa.hosted.node.admin.component.IdempotentTask; -import com.yahoo.vespa.hosted.node.admin.component.TaskContext; +package com.yahoo.vespa.hosted.node.admin.component; import java.util.ArrayList; import java.util.List; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributes.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributes.java index 3910398a040..611e2c32bcd 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributes.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributes.java @@ -24,4 +24,5 @@ public class FileAttributes { public String permissions() { return PosixFilePermissions.toString(attributes.permissions()); } public boolean isRegularFile() { return attributes.isRegularFile(); } public boolean isDirectory() { return attributes.isDirectory(); } + public long size() { return attributes.size(); } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/InputStreamUtil.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/InputStreamUtil.java new file mode 100644 index 00000000000..780102e9c9e --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/InputStreamUtil.java @@ -0,0 +1,40 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.hosted.node.admin.task.util.file; + +import java.io.ByteArrayOutputStream; +import java.io.InputStream; + +import static com.yahoo.vespa.hosted.node.admin.task.util.file.IOExceptionUtil.uncheck; + +/** + * @author hakonhall + */ +public class InputStreamUtil { + private final InputStream inputStream; + + public InputStreamUtil(InputStream inputStream) { + this.inputStream = inputStream; + } + + public InputStream getInputStream() { + return inputStream; + } + + /** + * TODO: Replace usages with Java 9's InputStream::readAllBytes + */ + byte[] readAllBytes() { + // According to https://stackoverflow.com/questions/309424/read-convert-an-inputstream-to-a-string + // all other implementations are much inferior to this in performance. + + ByteArrayOutputStream result = new ByteArrayOutputStream(); + byte[] buffer = new byte[1024]; + int length; + while ((length = uncheck(() -> inputStream.read(buffer))) != -1) { + result.write(buffer, 0, length); + } + + return result.toByteArray(); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess2.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess2.java new file mode 100644 index 00000000000..172203a281a --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess2.java @@ -0,0 +1,16 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.hosted.node.admin.task.util.process; + +/** + * @author hakonhall + */ +interface ChildProcess2 extends AutoCloseable { + void waitForTermination(); + int exitCode(); + String getOutput(); + + /** Close/cleanup any resources held. Must not throw an exception. */ + @Override + void close(); +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess2Impl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess2Impl.java new file mode 100644 index 00000000000..67020270a99 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess2Impl.java @@ -0,0 +1,138 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.hosted.node.admin.task.util.process; + +import com.yahoo.jdisc.Timer; +import com.yahoo.log.LogLevel; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.time.Instant; +import java.util.concurrent.TimeUnit; +import java.util.logging.Logger; + +import static com.yahoo.vespa.hosted.node.admin.task.util.file.IOExceptionUtil.uncheck; + +/** + * @author hakonhall + */ +public class ChildProcess2Impl implements ChildProcess2 { + private static final Logger logger = Logger.getLogger(ChildProcess2Impl.class.getName()); + + private final CommandLine commandLine; + private final ProcessApi2 process; + private final Path outputPath; + private final Timer timer; + + public ChildProcess2Impl(CommandLine commandLine, + ProcessApi2 process, + Path outputPath, + Timer timer) { + this.commandLine = commandLine; + this.process = process; + this.outputPath = outputPath; + this.timer = timer; + } + + @Override + public void waitForTermination() { + Duration timeoutDuration = commandLine.getTimeout(); + Instant timeout = timer.currentTime().plus(timeoutDuration); + long maxOutputBytes = commandLine.getMaxOutputBytes(); + + // How frequently do we want to wake up and check the output file size? + final Duration pollInterval = Duration.ofSeconds(10); + + boolean hasTerminated = false; + while (!hasTerminated) { + Instant now = timer.currentTime(); + long sleepPeriodMillis = pollInterval.toMillis(); + if (now.plusMillis(sleepPeriodMillis).isAfter(timeout)) { + sleepPeriodMillis = Duration.between(now, timeout).toMillis(); + + if (sleepPeriodMillis <= 0) { + gracefullyKill(); + throw new TimeoutChildProcessException( + timeoutDuration, commandLine.toString(), getOutput()); + } + } + + try { + hasTerminated = process.waitFor(sleepPeriodMillis, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + // Ignore, just loop around. + continue; + } + + // Always check output file size to ensure we don't load too much into memory. + long sizeInBytes = uncheck(() -> Files.size(outputPath)); + if (sizeInBytes > maxOutputBytes) { + gracefullyKill(); + throw new LargeOutputChildProcessException( + sizeInBytes, commandLine.toString(), getOutput()); + } + } + } + + @Override + public int exitCode() { + return process.exitValue(); + } + + @Override + public String getOutput() { + byte[] bytes = uncheck(() -> Files.readAllBytes(outputPath)); + return new String(bytes, commandLine.getOutputEncoding()); + } + + @Override + public void close() { + try { + Files.delete(outputPath); + } catch (Throwable t) { + logger.log(LogLevel.WARNING, "Failed to delete " + outputPath, t); + } + } + + Path getOutputPath() { + return outputPath; + } + + private void gracefullyKill() { + process.destroy(); + + Duration maxWaitAfterSigTerm = commandLine.getSigTermGracePeriod(); + Instant timeout = timer.currentTime().plus(maxWaitAfterSigTerm); + if (!waitForTermination(timeout)) { + process.destroyForcibly(); + + // If waiting for the process now takes a long time, it's probably a kernel issue + // or huge core is getting dumped. + Duration maxWaitAfterSigKill = commandLine.getSigKillGracePeriod(); + if (!waitForTermination(timer.currentTime().plus(maxWaitAfterSigKill))) { + throw new UnkillableChildProcessException( + maxWaitAfterSigTerm, + maxWaitAfterSigKill, + commandLine.toString(), + getOutput()); + } + } + } + + /** @return true if process terminated, false on timeout. */ + private boolean waitForTermination(Instant timeout) { + while (true) { + long waitDurationMillis = Duration.between(timer.currentTime(), timeout).toMillis(); + if (waitDurationMillis <= 0) { + return false; + } + + try { + return process.waitFor(waitDurationMillis, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + // ignore + } + } + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcessException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcessException.java new file mode 100644 index 00000000000..b84bd2d8fef --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcessException.java @@ -0,0 +1,66 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.hosted.node.admin.task.util.process; + +/** + * Base class for child process related exceptions, with a util to build an error message + * that includes a large part of the output. + * + * @author hakonhall + */ +@SuppressWarnings("serial") +public abstract class ChildProcessException extends RuntimeException { + private static final int MAX_OUTPUT_PREFIX = 200; + private static final int MAX_OUTPUT_SUFFIX = 200; + // Omitting a number of chars less than 10 or less than 10% would be ridiculous. + private static final int MAX_OUTPUT_SLACK = Math.max(10, (10 * (MAX_OUTPUT_PREFIX + MAX_OUTPUT_SUFFIX)) / 100); + + /** + * An exception with a message of the following format: + * Command 'COMMANDLINE' PROBLEM: stdout/stderr: 'OUTPUT' + * + * If the output of the terminated command is too large it will be sampled. + * + * @param problem E.g. "terminated with exit code 1" + * @param commandLine The command that failed in a concise (e.g. shell-like) format + * @param possiblyHugeOutput The output of the command + */ + protected ChildProcessException(String problem, String commandLine, String possiblyHugeOutput) { + super(makeSnippet( + problem, + commandLine, + possiblyHugeOutput, + MAX_OUTPUT_PREFIX, + MAX_OUTPUT_SUFFIX, + MAX_OUTPUT_SLACK)); + } + + // Package-private instead of private for testing. + static String makeSnippet(String problem, + String commandLine, + String possiblyHugeOutput, + int maxOutputPrefix, + int maxOutputSuffix, + int maxOutputSlack) { + StringBuilder stringBuilder = new StringBuilder() + .append("Command '") + .append(commandLine) + .append("' ") + .append(problem) + .append(": stdout/stderr: '"); + + if (possiblyHugeOutput.length() <= maxOutputPrefix + maxOutputSuffix + maxOutputSlack) { + stringBuilder.append(possiblyHugeOutput); + } else { + stringBuilder.append(possiblyHugeOutput.substring(0, maxOutputPrefix)) + .append("... [") + .append(possiblyHugeOutput.length() - maxOutputPrefix - maxOutputSuffix) + .append(" chars omitted] ...") + .append(possiblyHugeOutput.substring(possiblyHugeOutput.length() - maxOutputSuffix)); + } + + stringBuilder.append("'"); + + return stringBuilder.toString(); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcessFailureException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcessFailureException.java new file mode 100644 index 00000000000..5c6785a646c --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcessFailureException.java @@ -0,0 +1,15 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.hosted.node.admin.task.util.process; + +/** + * The child process terminated with a non-zero exit code. + * + * @author hakonhall + */ +@SuppressWarnings("serial") +public class ChildProcessFailureException extends ChildProcessException { + ChildProcessFailureException(int exitCode, String commandLine, String possiblyHugeOutput) { + super("terminated with exit code " + exitCode, commandLine, possiblyHugeOutput); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandLine.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandLine.java new file mode 100644 index 00000000000..6c4de7ac1e3 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandLine.java @@ -0,0 +1,265 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.hosted.node.admin.task.util.process; + +import com.yahoo.vespa.hosted.node.admin.component.TaskContext; + +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.function.Predicate; +import java.util.logging.Logger; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +/** + * A CommandLine is used to specify and execute a shell-like program in a child process, + * and capture its output. + * + * @author hakonhall + */ +public class CommandLine { + private static Logger logger = Logger.getLogger(CommandLine.class.getName()); + private static Pattern UNESCAPED_ARGUMENT_PATTERN = Pattern.compile("^[a-zA-Z0-9=@%/+:.,_-]+$"); + + /** The default timeout. See setTimeout() for details. */ + public static final Duration DEFAULT_TIMEOUT = Duration.ofMinutes(10); + + /** The default maximum number of output bytes. See setMaxOutputBytes() for details. */ + public static final long DEFAULT_MAX_OUTPUT_BYTES = 1024 * 1024 * 1024; // 1 Gb + + /** + * The default grace period after SIGTERM has been sent during a graceful kill. + * See setSigTermGracePeriod for details. + */ + public static final Duration DEFAULT_SIGTERM_GRACE_PERIOD = Duration.ofMinutes(1); + + /** + * The default grace period after SIGKILL has been sent during a graceful kill. + * See setSigKillGracePeriod for details. + */ + public static final Duration DEFAULT_SIGKILL_GRACE_PERIOD = Duration.ofMinutes(30); + + private final List<String> arguments = new ArrayList<>(); + private final TaskContext taskContext; + private final ProcessFactory processFactory; + + private boolean redirectStderrToStdoutInsteadOfDiscard = true; + private boolean executeSilentlyCalled = false; + private Charset outputEncoding = StandardCharsets.UTF_8; + private Duration timeout = DEFAULT_TIMEOUT; + private long maxOutputBytes = DEFAULT_MAX_OUTPUT_BYTES; + private Duration sigTermGracePeriod = DEFAULT_SIGTERM_GRACE_PERIOD; + private Duration sigKillGracePeriod = DEFAULT_SIGKILL_GRACE_PERIOD; + private Predicate<Integer> successfulExitCodePredicate = code -> code == 0; + + public CommandLine(TaskContext taskContext, ProcessFactory processFactory) { + this.taskContext = taskContext; + this.processFactory = processFactory; + } + + /** Add arguments to the command. The first argument in the first call to add() is the program. */ + public CommandLine add(String... arguments) { return add(Arrays.asList(arguments)); } + + /** Add arguments to the command. The first argument in the first call to add() is the program. */ + public CommandLine add(List<String> arguments) { + this.arguments.addAll(arguments); + return this; + } + + /** + * Execute a shell-like program in a child process: + * - the program is recorded and logged as modifying the system, but see executeSilently(). + * - the program's stderr is redirected to stdout, but see discardStderr(). + * - the program's output is assumed to be UTF-8, but see setOutputEncoding(). + * - the program must terminate with exit code 0, but see ignoreExitCode(). + * - the output of the program will be accessible in the returned CommandResult. + * + * Footnote 1: As a safety measure the size of the output is capped, and the program is + * only allowed to execute up to a timeout. The defaults are set high so you typically do + * not have to worry about reaching these limits, but otherwise see setMaxOutputBytes() + * and setTimeout(), respectively. + * + * Footnote 2: If the child process is forced to be killed due to footnote 1, then + * setSigTermGracePeriod() and setSigKillGracePeriod() can be used to tweak how much time + * is given to the program to shut down. Again, the defaults should be reasonable. + */ + public CommandResult execute() { + taskContext.recordSystemModification(logger, "Executing command: " + toString()); + return doExecute(); + } + + /** + * Same as execute(), except it will not record the program as modifying the system. + * + * If the program is later found to have modified the system, or otherwise worthy of + * a record, call recordSilentExecutionAsSystemModification(). + */ + public CommandResult executeSilently() { + executeSilentlyCalled = true; + return doExecute(); + } + + /** + * Record an already executed executeSilently() as having modified the system. + * For instance with YUM it is not known until after a 'yum install' whether it + * modified the system. + */ + public void recordSilentExecutionAsSystemModification() { + if (!executeSilentlyCalled) { + throw new IllegalStateException("executeSilently has not been called"); + } + // Disallow multiple consecutive calls to this method without an intervening call + // to executeSilently(). + executeSilentlyCalled = false; + + taskContext.recordSystemModification(logger, "Executed command: " + toString()); + } + + /** + * The first argument of the command specifies the program and is either the program's + * filename (in case the environment variable PATH will be used to search for the program + * file) or a path with the last component being the program's filename. + * + * @return The filename of the program. + */ + public String programName() { + if (arguments.isEmpty()) { + throw new IllegalStateException( + "The program name cannot be determined yet as no arguments have been given"); + } + String path = arguments.get(0); + int lastIndex = path.lastIndexOf('/'); + if (lastIndex == -1) { + return path; + } else { + return path.substring(lastIndex + 1); + } + } + + /** Returns a shell-like representation of the command. */ + @Override + public String toString() { + String command = arguments.stream() + .map(CommandLine::maybeEscapeArgument) + .collect(Collectors.joining(" ")); + + // Note: Both of these cannot be confused with an argument since they would + // require escaping. + command += redirectStderrToStdoutInsteadOfDiscard ? " 2>&1" : " 2>/dev/null"; + + return command; + } + + + /** + * By default, stderr is redirected to stderr. This method will instead discard stderr. + */ + public CommandLine discardStderr() { + this.redirectStderrToStdoutInsteadOfDiscard = false; + return this; + } + + /** + * By default, a non-zero exit code will cause the command execution to fail. This method + * will instead ignore the exit code. + */ + public CommandLine ignoreExitCode() { + this.successfulExitCodePredicate = code -> true; + return this; + } + + /** + * By default, the output of the command is parsed as UTF-8. This method will set a + * different encoding. + */ + public CommandLine setOutputEncoding(Charset outputEncoding) { + this.outputEncoding = outputEncoding; + return this; + } + + /** + * By default, the command will be gracefully killed after DEFAULT_TIMEOUT. This method + * overrides that default. + */ + public CommandLine setTimeout(Duration timeout) { + this.timeout = timeout; + return this; + } + + /** + * By default, the command will be gracefully killed if it ever outputs more than + * DEFAULT_MAX_OUTPUT_BYTES. This method overrides that default. + */ + public CommandLine setMaxOutputBytes(long maxOutputBytes) { + this.maxOutputBytes = maxOutputBytes; + return this; + } + + /** + * By default, if the program needs to be gracefully killed it will wait up to + * DEFAULT_SIGTERM_GRACE_PERIOD for the program to exit after it has been killed with + * the SIGTERM signal. + */ + public CommandLine setSigTermGracePeriod(Duration period) { + this.sigTermGracePeriod = period; + return this; + } + + public CommandLine setSigKillGracePeriod(Duration period) { + this.sigTermGracePeriod = period; + return this; + } + // Accessor fields necessary for classes in this package. Could be public if necessary. + List<String> getArguments() { return Collections.unmodifiableList(arguments); } + boolean getRedirectStderrToStdoutInsteadOfDiscard() { return redirectStderrToStdoutInsteadOfDiscard; } + Predicate<Integer> getSuccessfulExitCodePredicate() { return successfulExitCodePredicate; } + Charset getOutputEncoding() { return outputEncoding; } + Duration getTimeout() { return timeout; } + long getMaxOutputBytes() { return maxOutputBytes; } + Duration getSigTermGracePeriod() { return sigTermGracePeriod; } + Duration getSigKillGracePeriod() { return sigKillGracePeriod; } + + private CommandResult doExecute() { + try (ChildProcess2 child = processFactory.spawn(this)) { + child.waitForTermination(); + int exitCode = child.exitCode(); + if (!successfulExitCodePredicate.test(exitCode)) { + throw new ChildProcessFailureException(exitCode, toString(), child.getOutput()); + } + + String output = child.getOutput(); + return new CommandResult(this, exitCode, output); + } + } + + private static String maybeEscapeArgument(String argument) { + if (UNESCAPED_ARGUMENT_PATTERN.matcher(argument).matches()) { + return argument; + } else { + return escapeArgument(argument); + } + } + + private static String escapeArgument(String argument) { + StringBuilder doubleQuoteEscaped = new StringBuilder(argument.length() + 10); + + for (int i = 0; i < argument.length(); ++i) { + char c = argument.charAt(i); + switch (c) { + case '"': + case '\\': + doubleQuoteEscaped.append("\\").append(c); + break; + default: + doubleQuoteEscaped.append(c); + } + } + + return "\"" + doubleQuoteEscaped + "\""; + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandResult.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandResult.java new file mode 100644 index 00000000000..12f0d546b36 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandResult.java @@ -0,0 +1,60 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.hosted.node.admin.task.util.process; + +import java.util.List; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * A CommandResult is the result of the execution of a CommandLine. + * + * @author hakonhall + */ +public class CommandResult { + private static final Pattern NEWLINE = Pattern.compile("\\n"); + + private final CommandLine commandLine; + private final int exitCode; + private final String output; + + CommandResult(CommandLine commandLine, int exitCode, String output) { + this.commandLine = commandLine; + this.exitCode = exitCode; + this.output = output; + } + + public int getExitCode() { + return exitCode; + } + + /** Returns the output with leading and trailing white-space removed. */ + public String getOutput() { return output.trim(); } + + public String getUntrimmedOutput() { return output; } + + /** Returns the output lines of the command, omitting trailing empty lines. */ + public List<String> getOutputLines() { + return getOutputLinesStream().collect(Collectors.toList()); + } + + public Stream<String> getOutputLinesStream() { + if (output.isEmpty()) { + // For some reason an empty string => one-element list. + return Stream.empty(); + } + + // For some reason this removes trailing empty elements, but that's OK with us. + return NEWLINE.splitAsStream(output); + } + + /** + * Convenience method for getting the CommandLine, whose execution resulted in + * this CommandResult instance. + * + * Warning: the CommandLine is mutable and may be changed by the caller of the execution + * through other references! This is just a convenience method for getting that instance. + */ + public CommandLine getCommandLine() { return commandLine; } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/LargeOutputChildProcessException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/LargeOutputChildProcessException.java new file mode 100644 index 00000000000..5c764757e84 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/LargeOutputChildProcessException.java @@ -0,0 +1,15 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.hosted.node.admin.task.util.process; + +/** + * Exception thrown if the output of the child process is larger than the maximum limit. + * + * @author hakonhall + */ +@SuppressWarnings("serial") +public class LargeOutputChildProcessException extends ChildProcessException { + LargeOutputChildProcessException(long maxFileSize, String commandLine, String possiblyHugeOutput) { + super("output more than " + maxFileSize + " bytes", commandLine, possiblyHugeOutput); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessApi2.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessApi2.java new file mode 100644 index 00000000000..124f319e932 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessApi2.java @@ -0,0 +1,17 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.hosted.node.admin.task.util.process; + +import java.util.concurrent.TimeUnit; + +/** + * Process abstraction. + * + * @author hakonhall + */ +public interface ProcessApi2 { + boolean waitFor(long timeout, TimeUnit unit) throws InterruptedException; + int exitValue(); + void destroy(); + void destroyForcibly(); +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessApi2Impl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessApi2Impl.java new file mode 100644 index 00000000000..853558c38e6 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessApi2Impl.java @@ -0,0 +1,36 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.hosted.node.admin.task.util.process; + +import java.util.concurrent.TimeUnit; + +/** + * @author hakonhall + */ +public class ProcessApi2Impl implements ProcessApi2 { + private final Process process; + + ProcessApi2Impl(Process process) { + this.process = process; + } + + @Override + public boolean waitFor(long timeout, TimeUnit unit) throws InterruptedException { + return process.waitFor(timeout, unit); + } + + @Override + public int exitValue() { + return process.exitValue(); + } + + @Override + public void destroy() { + process.destroy(); + } + + @Override + public void destroyForcibly() { + process.destroyForcibly(); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessApiImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessApiImpl.java index e664a68aeff..3620ec9089e 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessApiImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessApiImpl.java @@ -41,7 +41,6 @@ public class ProcessApiImpl implements ProcessApi { @Override public void close() { - // TODO: Should kill process if still alive? processOutputPath.toFile().delete(); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessFactory.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessFactory.java new file mode 100644 index 00000000000..3351563faf5 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessFactory.java @@ -0,0 +1,10 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.hosted.node.admin.task.util.process; + +/** + * @author hakonhall + */ +public interface ProcessFactory { + ChildProcess2 spawn(CommandLine commandLine); +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessFactoryImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessFactoryImpl.java new file mode 100644 index 00000000000..1c7a60a13fc --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessFactoryImpl.java @@ -0,0 +1,89 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.hosted.node.admin.task.util.process; + +import com.yahoo.jdisc.Timer; +import com.yahoo.log.LogLevel; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.FileAttribute; +import java.nio.file.attribute.PosixFilePermission; +import java.nio.file.attribute.PosixFilePermissions; +import java.util.List; +import java.util.Set; +import java.util.logging.Logger; + +import static com.yahoo.vespa.hosted.node.admin.task.util.file.IOExceptionUtil.uncheck; + +/** + * @author hakonhall + */ +public class ProcessFactoryImpl implements ProcessFactory { + private static final Logger logger = Logger.getLogger(ProcessFactoryImpl.class.getName()); + private static final File DEV_NULL = new File("/dev/null"); + + private final ProcessStarter processStarter; + private final Timer timer; + + ProcessFactoryImpl(ProcessStarter processStarter, Timer timer) { + this.processStarter = processStarter; + this.timer = timer; + } + + @Override + public ChildProcess2Impl spawn(CommandLine commandLine) { + List<String> arguments = commandLine.getArguments(); + if (arguments.isEmpty()) { + throw new IllegalArgumentException("No arguments specified - missing program to spawn"); + } + + ProcessBuilder processBuilder = new ProcessBuilder(arguments); + + if (commandLine.getRedirectStderrToStdoutInsteadOfDiscard()) { + processBuilder.redirectErrorStream(true); + } else { + processBuilder.redirectError(ProcessBuilder.Redirect.to(DEV_NULL)); + } + + // The output is redirected to a temporary file because: + // - We could read continuously from process.getInputStream, but that may block + // indefinitely with a faulty program. + // - If we don't read continuously from process.getInputStream, then because + // the underlying channel may be a pipe, the child may be stopped because the pipe + // is full. + // - To honor the timeout, no API can be used that may end up blocking indefinitely. + // + // Therefore, we redirect the output to a file and use waitFor w/timeout. This also + // has the benefit of allowing for inspection of the file during execution, and + // allowing the inspection of the file if it e.g. gets too large to hold in-memory. + + String temporaryFilePrefix = + ProcessFactoryImpl.class.getSimpleName() + "-" + commandLine.programName() + "-"; + + FileAttribute<Set<PosixFilePermission>> fileAttribute = PosixFilePermissions.asFileAttribute( + PosixFilePermissions.fromString("rw-------")); + + Path temporaryFile = uncheck(() -> Files.createTempFile( + temporaryFilePrefix, + ".out", + fileAttribute)); + + try { + processBuilder.redirectOutput(temporaryFile.toFile()); + ProcessApi2 process = processStarter.start(processBuilder); + return new ChildProcess2Impl(commandLine, process, temporaryFile, timer); + } catch (RuntimeException | Error throwable) { + try { + Files.delete(temporaryFile); + } catch (IOException ioException) { + logger.log(LogLevel.WARNING, "Failed to delete temporary file at " + + temporaryFile, ioException); + } + throw throwable; + } + + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessStarter.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessStarter.java new file mode 100644 index 00000000000..0afd4c6ee37 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessStarter.java @@ -0,0 +1,10 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.hosted.node.admin.task.util.process; + +/** + * @author hakonhall + */ +public interface ProcessStarter { + ProcessApi2 start(ProcessBuilder processBuilder); +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessStarterImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessStarterImpl.java new file mode 100644 index 00000000000..2694a2929c4 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessStarterImpl.java @@ -0,0 +1,16 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.hosted.node.admin.task.util.process; + +import static com.yahoo.vespa.hosted.node.admin.task.util.file.IOExceptionUtil.uncheck; + +/** + * @author hakonhall + */ +public class ProcessStarterImpl implements ProcessStarter { + @Override + public ProcessApi2 start(ProcessBuilder processBuilder) { + Process process = uncheck(() -> processBuilder.start()); + return new ProcessApi2Impl(process); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/Terminal.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/Terminal.java new file mode 100644 index 00000000000..849099ab5ca --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/Terminal.java @@ -0,0 +1,14 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.hosted.node.admin.task.util.process; + +import com.yahoo.vespa.hosted.node.admin.component.TaskContext; + +/** + * A Terminal is a light-weight terminal-like interface for executing shell-like programs. + * + * @author hakonhall + */ +public interface Terminal { + CommandLine newCommandLine(TaskContext taskContext); +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TerminalImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TerminalImpl.java new file mode 100644 index 00000000000..422584b8ccd --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TerminalImpl.java @@ -0,0 +1,25 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.process; + +import com.yahoo.jdisc.Timer; +import com.yahoo.vespa.hosted.node.admin.component.TaskContext; + +/** + * @author hakonhall + */ +public class TerminalImpl implements Terminal { + private final ProcessFactory processFactory; + + public TerminalImpl(Timer timer) { + this(new ProcessFactoryImpl(new ProcessStarterImpl(), timer)); + } + + /** For testing. */ + public TerminalImpl(ProcessFactory processFactory) { + this.processFactory = processFactory; + } + + public CommandLine newCommandLine(TaskContext taskContext) { + return new CommandLine(taskContext, processFactory); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TestChildProcess2.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TestChildProcess2.java new file mode 100644 index 00000000000..4e678522168 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TestChildProcess2.java @@ -0,0 +1,52 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.process; + +import java.util.Optional; + +/** + * @author hakonhall + */ +public class TestChildProcess2 implements ChildProcess2 { + private final int exitCode; + private final String output; + private Optional<RuntimeException> exceptionToThrowInWaitForTermination = Optional.empty(); + private boolean closeCalled = false; + + public TestChildProcess2(int exitCode, String output) { + this.exitCode = exitCode; + this.output = output; + } + + public void throwInWaitForTermination(RuntimeException e) { + this.exceptionToThrowInWaitForTermination = Optional.of(e); + } + + @Override + public void waitForTermination() { + if (exceptionToThrowInWaitForTermination.isPresent()) { + throw exceptionToThrowInWaitForTermination.get(); + } + } + + @Override + public int exitCode() { + return exitCode; + } + + @Override + public String getOutput() { + return output; + } + + @Override + public void close() { + if (closeCalled) { + throw new IllegalStateException("close already called"); + } + closeCalled = true; + } + + public boolean closeCalled() { + return closeCalled; + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TestProcessFactory.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TestProcessFactory.java new file mode 100644 index 00000000000..0586797d259 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TestProcessFactory.java @@ -0,0 +1,95 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.process; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.function.Function; + +/** + * @author hakonhall + */ +public class TestProcessFactory implements ProcessFactory { + private static class SpawnCall { + private final String commandDescription; + private final Function<CommandLine, ChildProcess2> callback; + + private SpawnCall(String commandDescription, + Function<CommandLine, ChildProcess2> callback) { + this.commandDescription = commandDescription; + this.callback = callback; + } + } + private final List<SpawnCall> expectedSpawnCalls = new ArrayList<>(); + private final List<CommandLine> spawnCommandLines = new ArrayList<>(); + + /** Forward call to spawn() to callback. */ + public TestProcessFactory interceptSpawn(String commandDescription, + Function<CommandLine, ChildProcess2> callback) { + expectedSpawnCalls.add(new SpawnCall(commandDescription, callback)); + return this; + } + + // Convenience method for the caller to avoid having to create a TestChildProcess2 instance. + public TestProcessFactory expectSpawn(String commandLineString, TestChildProcess2 toReturn) { + return interceptSpawn( + commandLineString, + commandLine -> defaultSpawn(commandLine, commandLineString, toReturn)); + } + + // Convenience method for the caller to avoid having to create a TestChildProcess2 instance. + public TestProcessFactory expectSpawn(String commandLine, int exitCode, String output) { + return expectSpawn(commandLine, new TestChildProcess2(exitCode, output)); + } + + /** Ignore the CommandLine passed to spawn(), just return successfully with the given output. */ + public TestProcessFactory ignoreSpawn(String output) { + return interceptSpawn( + "[call index " + expectedSpawnCalls.size() + "]", + commandLine -> new TestChildProcess2(0, output)); + } + + public TestProcessFactory ignoreSpawn() { + return ignoreSpawn(""); + } + + public void verifyAllCommandsExecuted() { + if (spawnCommandLines.size() < expectedSpawnCalls.size()) { + int missingCommandIndex = spawnCommandLines.size(); + throw new IllegalStateException("Command #" + missingCommandIndex + + " never executed: " + + expectedSpawnCalls.get(missingCommandIndex).commandDescription); + } + } + + /** + * WARNING: CommandLine is mutable, and e.g. reusing a CommandLine for the next call + * would make the CommandLine in this list no longer reflect the original CommandLine. + */ + public List<CommandLine> getMutableCommandLines() { + return spawnCommandLines; + } + + @Override + public ChildProcess2 spawn(CommandLine commandLine) { + String commandLineString = commandLine.toString(); + if (spawnCommandLines.size() + 1 > expectedSpawnCalls.size()) { + throw new IllegalStateException("Too many invocations: " + commandLineString); + } + spawnCommandLines.add(commandLine); + + return expectedSpawnCalls.get(spawnCommandLines.size() - 1).callback.apply(commandLine); + } + + private static ChildProcess2 defaultSpawn(CommandLine commandLine, + String expectedCommandLineString, + ChildProcess2 toReturn) { + String actualCommandLineString = commandLine.toString(); + if (!Objects.equals(actualCommandLineString, expectedCommandLineString)) { + throw new IllegalArgumentException("Expected command line '" + + expectedCommandLineString + "' but got '" + actualCommandLineString + "'"); + } + + return toReturn; + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TestTerminal.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TestTerminal.java new file mode 100644 index 00000000000..57aeeb04532 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TestTerminal.java @@ -0,0 +1,67 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.process; + +import com.yahoo.vespa.hosted.node.admin.component.TaskContext; + +import java.util.function.Function; + +/** + * @author hakonhall + */ +public class TestTerminal implements Terminal { + private final TerminalImpl realTerminal; + private final TestProcessFactory testProcessFactory = new TestProcessFactory(); + + public TestTerminal() { + this.realTerminal = new TerminalImpl(testProcessFactory); + } + + /** Get the TestProcessFactory the terminal was started with. */ + public TestProcessFactory getTestProcessFactory() { return testProcessFactory; } + + /** Forward call to spawn() to callback. */ + public TestTerminal interceptCommand(String commandDescription, + Function<CommandLine, ChildProcess2> callback) { + testProcessFactory.interceptSpawn(commandDescription, callback); + return this; + } + + /** Wraps expectSpawn in TestProcessFactory, provided here as convenience. */ + public TestTerminal expectCommand(String commandLine, TestChildProcess2 toReturn) { + testProcessFactory.expectSpawn(commandLine, toReturn); + return this; + } + + /** Wraps expectSpawn in TestProcessFactory, provided here as convenience. */ + public TestTerminal expectCommand(String commandLine, int exitCode, String output) { + testProcessFactory.expectSpawn(commandLine, new TestChildProcess2(exitCode, output)); + return this; + } + + /** Verifies command line matches commandLine, and returns successfully with output "". */ + public TestTerminal expectCommand(String commandLine) { + expectCommand(commandLine, 0, ""); + return this; + } + + /** Wraps expectSpawn in TestProcessFactory, provided here as convenience. */ + public TestTerminal ignoreCommand(String output) { + testProcessFactory.ignoreSpawn(output); + return this; + } + + /** Wraps expectSpawn in TestProcessFactory, provided here as convenience. */ + public TestTerminal ignoreCommand() { + testProcessFactory.ignoreSpawn(); + return this; + } + + public void verifyAllCommandsExecuted() { + testProcessFactory.verifyAllCommandsExecuted(); + } + + @Override + public CommandLine newCommandLine(TaskContext taskContext) { + return realTerminal.newCommandLine(taskContext); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TimeoutChildProcessException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TimeoutChildProcessException.java new file mode 100644 index 00000000000..df9e2dc3471 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/TimeoutChildProcessException.java @@ -0,0 +1,18 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.hosted.node.admin.task.util.process; + +import java.time.Duration; + +/** + * Exception thrown when a child process has taken too long to terminate, in case it has been + * forcibly killed. + * + * @author hakonhall + */ +@SuppressWarnings("serial") +public class TimeoutChildProcessException extends ChildProcessException { + TimeoutChildProcessException(Duration timeout, String commandLine, String possiblyHugeOutput) { + super("timed out after " + timeout, commandLine, possiblyHugeOutput); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/UnexpectedOutputException2.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/UnexpectedOutputException2.java new file mode 100644 index 00000000000..e786452c0ef --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/UnexpectedOutputException2.java @@ -0,0 +1,16 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.hosted.node.admin.task.util.process; + +/** + * @author hakonhall + */ +@SuppressWarnings("serial") +public class UnexpectedOutputException2 extends ChildProcessException { + /** + * @param problem Problem description, e.g. "Output is not of the form ^NAME=VALUE$" + */ + public UnexpectedOutputException2(String problem, String commandLine, String possiblyHugeOutput) { + super("output was not of the expected format: " + problem, commandLine, possiblyHugeOutput); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/UnkillableChildProcessException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/UnkillableChildProcessException.java new file mode 100644 index 00000000000..1da27dd853e --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/UnkillableChildProcessException.java @@ -0,0 +1,21 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.hosted.node.admin.task.util.process; + +import java.time.Duration; + +/** + * @author hakonhall + */ +@SuppressWarnings("serial") +public class UnkillableChildProcessException extends ChildProcessException { + public UnkillableChildProcessException(Duration waitForSigTerm, + Duration waitForSigKill, + String commandLine, + String possiblyHugeOutput) { + super("did not terminate even after SIGTERM, +" + waitForSigTerm + + ", SIGKILL, and +" + waitForSigKill, + commandLine, + possiblyHugeOutput); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/time/TestTimer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/time/TestTimer.java new file mode 100644 index 00000000000..beadeeed4a3 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/time/TestTimer.java @@ -0,0 +1,29 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.time; + +import com.yahoo.jdisc.Timer; + +import java.time.Duration; + +/** + * @author hakonhall + */ +public class TestTimer implements Timer { + private Duration durationSinceEpoch = Duration.ZERO; + + public void setMillis(long millisSinceEpoch) { + durationSinceEpoch = Duration.ofMillis(millisSinceEpoch); + } + + public void advanceMillis(long millis) { advance(Duration.ofMillis(millis)); } + public void advanceSeconds(long seconds) { advance(Duration.ofSeconds(seconds)); } + public void advanceMinutes(long minutes) { advance(Duration.ofMinutes(minutes)); } + public void advance(Duration duration) { + durationSinceEpoch = durationSinceEpoch.plus(duration); + } + + @Override + public long currentTimeMillis() { + return durationSinceEpoch.toMillis(); + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSyncTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSyncTest.java index 44868e17464..dbc2cc9a5d5 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSyncTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSyncTest.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.node.admin.task.util.file; +import com.yahoo.vespa.hosted.node.admin.component.TestTaskContext; import com.yahoo.vespa.test.file.TestFileSystem; import org.junit.Test; diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/MakeDirectoryTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/MakeDirectoryTest.java index 05662de3b95..a83f3bbe7d4 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/MakeDirectoryTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/MakeDirectoryTest.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.node.admin.task.util.file; +import com.yahoo.vespa.hosted.node.admin.component.TestTaskContext; import com.yahoo.vespa.test.file.TestFileSystem; import org.junit.Test; diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess2ImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess2ImplTest.java new file mode 100644 index 00000000000..1a88af8ad0f --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess2ImplTest.java @@ -0,0 +1,147 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.process; + +import com.yahoo.jdisc.Timer; +import com.yahoo.vespa.test.file.TestFileSystem; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.FileSystem; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.time.Instant; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyLong; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * @author hakonhall + */ +public class ChildProcess2ImplTest { + private final FileSystem fileSystem = TestFileSystem.create(); + private final Timer timer = mock(Timer.class); + private final CommandLine commandLine = mock(CommandLine.class); + private final ProcessApi2 processApi = mock(ProcessApi2.class); + private Path temporaryFile; + + @Before + public void setUp() throws IOException { + temporaryFile = Files.createTempFile(fileSystem.getPath("/"), "", ""); + } + + @Test + public void testSuccess() throws Exception { + when(commandLine.getTimeout()).thenReturn(Duration.ofHours(1)); + when(commandLine.getMaxOutputBytes()).thenReturn(10L); + when(commandLine.getOutputEncoding()).thenReturn(StandardCharsets.UTF_8); + when(commandLine.getSigTermGracePeriod()).thenReturn(Duration.ofMinutes(2)); + when(commandLine.getSigKillGracePeriod()).thenReturn(Duration.ofMinutes(3)); + when(commandLine.toString()).thenReturn("program arg"); + + when(timer.currentTime()).thenReturn( + Instant.ofEpochMilli(1), + Instant.ofEpochMilli(2)); + + when(processApi.waitFor(anyLong(), any())).thenReturn(true); + + try (ChildProcess2Impl child = + new ChildProcess2Impl(commandLine, processApi, temporaryFile, timer)) { + child.waitForTermination(); + } + } + + @Test + public void testTimeout() throws Exception { + when(commandLine.getTimeout()).thenReturn(Duration.ofSeconds(1)); + when(commandLine.getMaxOutputBytes()).thenReturn(10L); + when(commandLine.getOutputEncoding()).thenReturn(StandardCharsets.UTF_8); + when(commandLine.getSigTermGracePeriod()).thenReturn(Duration.ofMinutes(2)); + when(commandLine.getSigKillGracePeriod()).thenReturn(Duration.ofMinutes(3)); + when(commandLine.toString()).thenReturn("program arg"); + + when(timer.currentTime()).thenReturn( + Instant.ofEpochSecond(0), + Instant.ofEpochSecond(2)); + + when(processApi.waitFor(anyLong(), any())).thenReturn(true); + + try (ChildProcess2Impl child = + new ChildProcess2Impl(commandLine, processApi, temporaryFile, timer)) { + try { + child.waitForTermination(); + fail(); + } catch (TimeoutChildProcessException e) { + assertEquals( + "Command 'program arg' timed out after PT1S: stdout/stderr: ''", + e.getMessage()); + } + } + } + + @Test + public void testMaxOutputBytes() throws Exception { + when(commandLine.getTimeout()).thenReturn(Duration.ofSeconds(1)); + when(commandLine.getMaxOutputBytes()).thenReturn(10L); + when(commandLine.getOutputEncoding()).thenReturn(StandardCharsets.UTF_8); + when(commandLine.getSigTermGracePeriod()).thenReturn(Duration.ofMinutes(2)); + when(commandLine.getSigKillGracePeriod()).thenReturn(Duration.ofMinutes(3)); + when(commandLine.toString()).thenReturn("program arg"); + + when(timer.currentTime()).thenReturn( + Instant.ofEpochMilli(0), + Instant.ofEpochMilli(1)); + + when(processApi.waitFor(anyLong(), any())).thenReturn(true); + + Files.write(temporaryFile, "1234567890123".getBytes(StandardCharsets.UTF_8)); + + try (ChildProcess2Impl child = + new ChildProcess2Impl(commandLine, processApi, temporaryFile, timer)) { + try { + child.waitForTermination(); + fail(); + } catch (LargeOutputChildProcessException e) { + assertEquals( + "Command 'program arg' output more than 13 bytes: stdout/stderr: '1234567890123'", + e.getMessage()); + } + } + } + + @Test + public void testUnkillable() throws Exception { + when(commandLine.getTimeout()).thenReturn(Duration.ofSeconds(1)); + when(commandLine.getMaxOutputBytes()).thenReturn(10L); + when(commandLine.getOutputEncoding()).thenReturn(StandardCharsets.UTF_8); + when(commandLine.getSigTermGracePeriod()).thenReturn(Duration.ofMinutes(2)); + when(commandLine.getSigKillGracePeriod()).thenReturn(Duration.ofMinutes(3)); + when(commandLine.toString()).thenReturn("program arg"); + + when(timer.currentTime()).thenReturn( + Instant.ofEpochMilli(0), + Instant.ofEpochMilli(1)); + + when(processApi.waitFor(anyLong(), any())).thenReturn(false); + + Files.write(temporaryFile, "1234567890123".getBytes(StandardCharsets.UTF_8)); + + try (ChildProcess2Impl child = + new ChildProcess2Impl(commandLine, processApi, temporaryFile, timer)) { + try { + child.waitForTermination(); + fail(); + } catch (UnkillableChildProcessException e) { + assertEquals( + "Command 'program arg' did not terminate even after SIGTERM, +PT2M, SIGKILL, and +PT3M: stdout/stderr: '1234567890123'", + e.getMessage()); + } + } + } +}
\ No newline at end of file diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandLineTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandLineTest.java new file mode 100644 index 00000000000..dd6d9c8226d --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandLineTest.java @@ -0,0 +1,115 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.process; + +import com.yahoo.vespa.hosted.node.admin.component.TestTaskContext; +import org.junit.After; +import org.junit.Test; + +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.List; +import java.util.function.Predicate; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +public class CommandLineTest { + private final TestTerminal terminal = new TestTerminal(); + private final TestTaskContext context = new TestTaskContext(); + private final CommandLine commandLine = terminal.newCommandLine(context); + + @After + public void tearDown() { + terminal.verifyAllCommandsExecuted(); + } + + @Test + public void testStrings() { + terminal.expectCommand( + "/bin/bash \"with space\" \"speci&l\" \"\" \"double\\\"quote\" 2>&1", + 0, + ""); + commandLine.add("/bin/bash", "with space", "speci&l", "", "double\"quote").execute(); + assertEquals("bash", commandLine.programName()); + } + + @Test + public void testBasicExecute() { + terminal.expectCommand("foo bar 2>&1", 0, "line1\nline2\n\n"); + CommandResult result = commandLine.add("foo", "bar").execute(); + assertEquals(0, result.getExitCode()); + assertEquals("line1\nline2", result.getOutput()); + assertEquals("line1\nline2\n\n", result.getUntrimmedOutput()); + assertEquals(Arrays.asList("line1", "line2"), result.getOutputLines()); + assertEquals(1, context.getSystemModificationLog().size()); + assertEquals("Executing command: foo bar 2>&1", context.getSystemModificationLog().get(0)); + + List<CommandLine> commandLines = terminal.getTestProcessFactory().getMutableCommandLines(); + assertEquals(1, commandLines.size()); + assertTrue(commandLine == commandLines.get(0)); + } + + @Test + public void verifyDefaults() { + assertEquals(CommandLine.DEFAULT_TIMEOUT, commandLine.getTimeout()); + assertEquals(CommandLine.DEFAULT_MAX_OUTPUT_BYTES, commandLine.getMaxOutputBytes()); + assertEquals(CommandLine.DEFAULT_SIGTERM_GRACE_PERIOD, commandLine.getSigTermGracePeriod()); + assertEquals(CommandLine.DEFAULT_SIGKILL_GRACE_PERIOD, commandLine.getSigKillGracePeriod()); + assertEquals(0, commandLine.getArguments().size()); + assertEquals(StandardCharsets.UTF_8, commandLine.getOutputEncoding()); + assertTrue(commandLine.getRedirectStderrToStdoutInsteadOfDiscard()); + Predicate<Integer> defaultExitCodePredicate = commandLine.getSuccessfulExitCodePredicate(); + assertTrue(defaultExitCodePredicate.test(0)); + assertFalse(defaultExitCodePredicate.test(1)); + } + + @Test + public void executeSilently() { + terminal.ignoreCommand(""); + commandLine.add("foo", "bar").executeSilently(); + assertEquals(0, context.getSystemModificationLog().size()); + commandLine.recordSilentExecutionAsSystemModification(); + assertEquals(1, context.getSystemModificationLog().size()); + assertEquals("Executed command: foo bar 2>&1", context.getSystemModificationLog().get(0)); + } + + @Test(expected = NegativeArraySizeException.class) + public void processFactorySpawnFails() { + terminal.interceptCommand( + commandLine.toString(), + command -> { throw new NegativeArraySizeException(); }); + commandLine.add("foo").execute(); + } + + @Test + public void waitingForTerminationExceptionStillClosesChild() { + TestChildProcess2 child = new TestChildProcess2(0, ""); + child.throwInWaitForTermination(new NegativeArraySizeException()); + terminal.interceptCommand(commandLine.toString(), command -> child); + assertFalse(child.closeCalled()); + try { + commandLine.add("foo").execute(); + fail(); + } catch (NegativeArraySizeException e) { + // OK + } + + assertTrue(child.closeCalled()); + } + + @Test + public void programFails() { + TestChildProcess2 child = new TestChildProcess2(0, ""); + terminal.expectCommand("foo 2>&1", 1, ""); + try { + commandLine.add("foo").execute(); + fail(); + } catch (ChildProcessFailureException e) { + assertEquals( + "Command 'foo 2>&1' terminated with exit code 1: stdout/stderr: ''", + e.getMessage()); + } + } +}
\ No newline at end of file diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessFactoryImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessFactoryImplTest.java new file mode 100644 index 00000000000..5a32b4c68b1 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessFactoryImplTest.java @@ -0,0 +1,48 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.process; + +import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; +import com.yahoo.vespa.hosted.node.admin.task.util.time.TestTimer; +import org.junit.Test; +import org.mockito.ArgumentCaptor; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class ProcessFactoryImplTest { + private final ProcessStarter starter = mock(ProcessStarter.class); + private final TestTimer timer = new TestTimer(); + private final ProcessFactoryImpl processFactory = new ProcessFactoryImpl(starter, timer); + + @Test + public void testSpawn() { + CommandLine commandLine = mock(CommandLine.class); + when(commandLine.getArguments()).thenReturn(Arrays.asList("program")); + when(commandLine.getRedirectStderrToStdoutInsteadOfDiscard()).thenReturn(true); + when(commandLine.programName()).thenReturn("program"); + Path outputPath; + try (ChildProcess2Impl child = processFactory.spawn(commandLine)) { + outputPath = child.getOutputPath(); + assertTrue(Files.exists(outputPath)); + assertEquals("rw-------", new UnixPath(outputPath).getPermissions()); + ArgumentCaptor<ProcessBuilder> processBuilderCaptor = + ArgumentCaptor.forClass(ProcessBuilder.class); + verify(starter).start(processBuilderCaptor.capture()); + ProcessBuilder processBuilder = processBuilderCaptor.getValue(); + assertTrue(processBuilder.redirectErrorStream()); + ProcessBuilder.Redirect redirect = processBuilder.redirectOutput(); + assertEquals(ProcessBuilder.Redirect.Type.WRITE, redirect.type()); + assertEquals(outputPath.toFile(), redirect.file()); + } + + assertFalse(Files.exists(outputPath)); + } +}
\ No newline at end of file diff --git a/predicate-search-core/src/main/java/com/yahoo/document/predicate/Predicate.java b/predicate-search-core/src/main/java/com/yahoo/document/predicate/Predicate.java index 5845366b325..3c2a6ec2a2b 100644 --- a/predicate-search-core/src/main/java/com/yahoo/document/predicate/Predicate.java +++ b/predicate-search-core/src/main/java/com/yahoo/document/predicate/Predicate.java @@ -11,7 +11,7 @@ import org.antlr.runtime.RecognitionException; import java.nio.charset.StandardCharsets; /** - * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + * @author Simon Thoresen Hult */ public abstract class Predicate implements Cloneable { diff --git a/searchcore/src/apps/proton/proton.cpp b/searchcore/src/apps/proton/proton.cpp index 5cdc8e04eb4..70cd8d8a7ad 100644 --- a/searchcore/src/apps/proton/proton.cpp +++ b/searchcore/src/apps/proton/proton.cpp @@ -208,6 +208,11 @@ App::Main() } sigBusHandler.reset(new search::SigBusHandler(stateFile.get())); ioErrorHandler.reset(new search::IOErrorHandler(stateFile.get())); + if ( ! params.serviceidentity.empty()) { + proton.getMetricManager().init(params.serviceidentity, proton.getThreadPool()); + } else { + proton.getMetricManager().init(params.identity, proton.getThreadPool()); + } if (!downPersistence) { proton.init(configSnapshot); } @@ -218,8 +223,6 @@ App::Main() spiProton->setupConfig(params.subscribeTimeout); spiProton->createNode(); EV_STARTED("servicelayer"); - } else { - proton.getMetricManager().init(params.identity, proton.getThreadPool()); } EV_STARTED("proton"); while (!(SIG::INT.check() || SIG::TERM.check() || (spiProton && spiProton->getNode().attemptedStopped()))) { diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp index 65a47e1ded8..49f8d50d9d4 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp @@ -165,7 +165,7 @@ Proton::Proton(const config::ConfigUri & configUri, _configUri(configUri), _mutex(), _metricsHook(*this), - _metricsEngine(), + _metricsEngine(std::make_unique<MetricsEngine>()), _fileHeaderContext(*this, progName), _tls(), _diskMemUsageSampler(), @@ -235,7 +235,6 @@ Proton::init(const BootstrapConfig::SP & configSnapshot) (protonConfig.basedir, diskMemUsageSamplerConfig(protonConfig, hwInfo)); - _metricsEngine.reset(new MetricsEngine()); _metricsEngine->addMetricsHook(_metricsHook); _fileHeaderContext.setClusterName(protonConfig.clustername, protonConfig.basedir); _tls.reset(new TLS(_configUri.createWithNewId(protonConfig.tlsconfigid), _fileHeaderContext)); diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/FeatureNames.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/FeatureNames.java deleted file mode 100644 index 80028519f67..00000000000 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/FeatureNames.java +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchlib.rankingexpression.evaluation; - -import java.util.Arrays; -import java.util.List; -import java.util.regex.Pattern; -import java.util.stream.Collectors; - -/** - * Utility methods for working with rank feature names - * - * @author bratseth - */ -public class FeatureNames { - - private static final Pattern identifierRegexp = Pattern.compile("[A-Za-z0-9_][A-Za-z0-9_-]*"); - - /** - * Returns the given feature in canonical form. - * A feature name consists of a feature shortname, followed by zero or more arguments enclosed in quotes - * and an optional output prefixed by a dot: shortname[(argument-ist)][.output] - * Arguments may be identifiers or any strings single or double quoted. - * - * Argument string values may not contain comma, single quote nor double quote characters. - * - * <i>The canonical form use no quotes for arguments which are identifiers, and double quotes otherwise.</i> - */ - public static String canonicalize(String feature) { - int startParenthesis = feature.indexOf('('); - int endParenthesis = feature.lastIndexOf(')'); - if (startParenthesis < 1) return feature; // No arguments - if (endParenthesis < startParenthesis) - throw new IllegalArgumentException("A feature name must be on the form shortname[(argument-ist)][.output], " + - "but was '" + feature + "'"); - String argumentString = feature.substring(startParenthesis + 1, endParenthesis); - List<String> canonicalizedArguments = - Arrays.stream(argumentString.split(",")) - .map(FeatureNames::canonicalizeArgument) - .collect(Collectors.toList()); - return feature.substring(0, startParenthesis + 1) + - canonicalizedArguments.stream().collect(Collectors.joining(",")) + - feature.substring(endParenthesis); - } - - /** Canomicalizes a single argument */ - private static String canonicalizeArgument(String argument) { - if (argument.startsWith("'")) { - if ( ! argument.endsWith("'")) - throw new IllegalArgumentException("Feature arguments starting by a single quote " + - "must end by a single quote, but was \"" + argument + "\""); - argument = argument.substring(1, argument.length() - 1); - } - if (argument.startsWith("\"")) { - if ( ! argument.endsWith("\"")) - throw new IllegalArgumentException("Feature arguments starting by a double quote " + - "must end by a double quote, but was '" + argument + "'"); - argument = argument.substring(1, argument.length() - 1); - } - if (identifierRegexp.matcher(argument).matches()) - return argument; - else - return "\"" + argument + "\""; - } - - public static String asConstantFeature(String constantName) { - return canonicalize("constant(\"" + constantName + "\")"); - } - - public static String asAttributeFeature(String attributeName) { - return canonicalize("attribute(\"" + attributeName + "\")"); - } - - public static String asQueryFeature(String propertyName) { - return canonicalize("query(\"" + propertyName + "\")"); - } - -} diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/MapContext.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/MapContext.java index 333af529cb9..a81d0c89f8f 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/MapContext.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/MapContext.java @@ -27,7 +27,7 @@ public class MapContext extends Context { * All the Values of the map will be frozen. */ public MapContext(Map<String,Value> bindings) { - bindings.forEach((k, v) -> this.bindings.put(FeatureNames.canonicalize(k), v.freeze())); + bindings.forEach((k, v) -> this.bindings.put(k, v.freeze())); } /** @@ -43,7 +43,7 @@ public class MapContext extends Context { /** Returns the type of the given value key, or null if it is not bound. */ @Override public TensorType getType(String key) { - Value value = bindings.get(FeatureNames.canonicalize(key)); + Value value = bindings.get(key); if (value == null) return null; return value.type(); } @@ -51,7 +51,7 @@ public class MapContext extends Context { /** Returns the value of a key. 0 is returned if the given key is not bound in this. */ @Override public Value get(String key) { - return bindings.getOrDefault(FeatureNames.canonicalize(key), DoubleValue.zero); + return bindings.getOrDefault(key, DoubleValue.zero); } /** @@ -59,7 +59,7 @@ public class MapContext extends Context { */ @Override public void put(String key,Value value) { - bindings.put(FeatureNames.canonicalize(key), value.freeze()); + bindings.put(key, value.freeze()); } /** Returns an immutable view of the bindings of this. */ diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/TypeMapContext.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/TypeMapContext.java index 0335ead4420..ff2088263d8 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/TypeMapContext.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/TypeMapContext.java @@ -18,12 +18,12 @@ public class TypeMapContext implements TypeContext { private final Map<String, TensorType> featureTypes = new HashMap<>(); public void setType(String name, TensorType type) { - featureTypes.put(FeatureNames.canonicalize(name), type); + featureTypes.put(name, type); } @Override public TensorType getType(String name) { - return featureTypes.get(FeatureNames.canonicalize(name)); + return featureTypes.get(name); } /** Returns an unmodifiable map of the bindings in this */ diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/OperationMapper.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/OperationMapper.java index b8f8e288257..55782c36d18 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/OperationMapper.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/OperationMapper.java @@ -219,7 +219,7 @@ class OperationMapper { private static Optional<TypedTensorFunction> placeholderWithDefault(TensorFlowImporter.Parameters params) { String name = toVespaName(params.node().getInput(0)); Tensor defaultValue = getConstantTensor(params, params.node().getInput(0)); - params.result().constant(name, defaultValue); + params.result().largeConstant(name, defaultValue); params.result().macro(name, new RankingExpression(name, new ReferenceNode("constant(\"" + name + "\")"))); // The default value will be provided by the macro. Users can override macro to change value. TypedTensorFunction output = new TypedTensorFunction(defaultValue.type(), new VariableTensor(name)); @@ -544,7 +544,11 @@ class OperationMapper { private static Optional<TypedTensorFunction> createConstant(TensorFlowImporter.Parameters params, Tensor constant) { String name = toVespaName(params.node().getName()); - params.result().constant(name, constant); + if (constant.type().rank() == 0 || constant.size() <= 1) { + params.result().smallConstant(name, constant); + } else { + params.result().largeConstant(name, constant); + } TypedTensorFunction output = new TypedTensorFunction(constant.type(), new TensorFunctionNode.TensorFunctionExpressionNode( new ReferenceNode("constant(\"" + name + "\")"))); @@ -553,8 +557,11 @@ class OperationMapper { private static Tensor getConstantTensor(TensorFlowImporter.Parameters params, String name) { String vespaName = toVespaName(name); - if (params.result().constants().containsKey(vespaName)) { - return params.result().constants().get(vespaName); + if (params.result().smallConstants().containsKey(vespaName)) { + return params.result().smallConstants().get(vespaName); + } + if (params.result().largeConstants().containsKey(vespaName)) { + return params.result().largeConstants().get(vespaName); } Session.Runner fetched = params.model().session().runner().fetch(name); List<org.tensorflow.Tensor<?>> importedTensors = fetched.run(); diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/TensorFlowModel.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/TensorFlowModel.java index 530f4793b62..351aa417f9c 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/TensorFlowModel.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/TensorFlowModel.java @@ -24,13 +24,15 @@ public class TensorFlowModel { private final Map<String, Signature> signatures = new HashMap<>(); private final Map<String, TensorType> arguments = new HashMap<>(); - private final Map<String, Tensor> constants = new HashMap<>(); + private final Map<String, Tensor> smallConstants = new HashMap<>(); + private final Map<String, Tensor> largeConstants = new HashMap<>(); private final Map<String, RankingExpression> expressions = new HashMap<>(); private final Map<String, RankingExpression> macros = new HashMap<>(); private final Map<String, TensorType> requiredMacros = new HashMap<>(); void argument(String name, TensorType argumentType) { arguments.put(name, argumentType); } - void constant(String name, Tensor constant) { constants.put(name, constant); } + void smallConstant(String name, Tensor constant) { smallConstants.put(name, constant); } + void largeConstant(String name, Tensor constant) { largeConstants.put(name, constant); } void expression(String name, RankingExpression expression) { expressions.put(name, expression); } void macro(String name, RankingExpression expression) { macros.put(name, expression); } void requiredMacro(String name, TensorType type) { requiredMacros.put(name, type); } @@ -43,8 +45,19 @@ public class TensorFlowModel { /** Returns an immutable map of the arguments ("Placeholders") of this */ public Map<String, TensorType> arguments() { return Collections.unmodifiableMap(arguments); } - /** Returns an immutable map of the constants of this */ - public Map<String, Tensor> constants() { return Collections.unmodifiableMap(constants); } + /** + * Returns an immutable map of the small constants of this. + * These should have sizes up to a few kb at most, and correspond to constant + * values given in the TensorFlow source. + */ + public Map<String, Tensor> smallConstants() { return Collections.unmodifiableMap(smallConstants); } + + /** + * Returns an immutable map of the large constants of this. + * These can have sizes in gigabytes and must be distributed to nodes separately from configuration, + * and correspond to Variable files stored separately in TensorFlow. + */ + public Map<String, Tensor> largeConstants() { return Collections.unmodifiableMap(largeConstants); } /** * Returns an immutable map of the expressions of this - corresponding to TensorFlow nodes diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/transform/ConstantDereferencer.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/transform/ConstantDereferencer.java index 216b677f6ff..9cd4f4dced9 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/transform/ConstantDereferencer.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/transform/ConstantDereferencer.java @@ -12,7 +12,7 @@ import java.util.ArrayList; import java.util.List; /** - * Replaces "features" which found in the given constants by their constant value + * Replaces constant reference pseudofeatures in expressions by their constant value * * @author bratseth */ diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/FeatureNamesTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/FeatureNamesTestCase.java deleted file mode 100644 index cf390171fa8..00000000000 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/FeatureNamesTestCase.java +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchlib.rankingexpression.evaluation; - -import org.junit.Test; - -import static org.junit.Assert.assertEquals; - -/** - * Tests rank feature names. - * - * @author bratseth - */ -public class FeatureNamesTestCase { - - @Test - public void testCanonicalization() { - assertEquals("foo", FeatureNames.canonicalize("foo")); - assertEquals("foo.out", FeatureNames.canonicalize("foo.out")); - assertEquals("foo(bar)", FeatureNames.canonicalize("foo(bar)")); - assertEquals("foo(bar)", FeatureNames.canonicalize("foo('bar')")); - assertEquals("foo(bar)", FeatureNames.canonicalize("foo(\"bar\")")); - assertEquals("foo(bar).out", FeatureNames.canonicalize("foo(bar).out")); - assertEquals("foo(bar).out", FeatureNames.canonicalize("foo('bar').out")); - assertEquals("foo(bar).out", FeatureNames.canonicalize("foo(\"bar\").out")); - assertEquals("foo(\"ba.r\")", FeatureNames.canonicalize("foo(ba.r)")); - assertEquals("foo(\"ba.r\")", FeatureNames.canonicalize("foo('ba.r')")); - assertEquals("foo(\"ba.r\")", FeatureNames.canonicalize("foo(\"ba.r\")")); - assertEquals("foo(bar1,\"b.ar2\",\"ba/r3\").out", - FeatureNames.canonicalize("foo(bar1,b.ar2,ba/r3).out")); - assertEquals("foo(bar1,\"b.ar2\",\"ba/r3\").out", - FeatureNames.canonicalize("foo(bar1,'b.ar2',\"ba/r3\").out")); - } - - @Test - public void testConstantFeature() { - assertEquals("constant(\"foo/bar\")", FeatureNames.asConstantFeature("foo/bar")); - } - - @Test - public void testAttributeFeature() { - assertEquals("attribute(foo)", FeatureNames.asAttributeFeature("foo")); - } - - @Test - public void testQueryFeature() { - assertEquals("query(\"foo.bar\")", FeatureNames.asQueryFeature("foo.bar")); - } - -} diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/TypeResolutionTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/TypeResolutionTestCase.java index 5cac2215a00..c882c887c8d 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/TypeResolutionTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/TypeResolutionTestCase.java @@ -19,11 +19,11 @@ public class TypeResolutionTestCase { @Test public void testTypeResolution() { TypeMapContext context = new TypeMapContext(); - context.setType("query('x1')", TensorType.fromSpec("tensor(x[])")); - context.setType("query('x2')", TensorType.fromSpec("tensor(x[10])")); - context.setType("query('y1')", TensorType.fromSpec("tensor(y[])")); - context.setType("query('xy1')", TensorType.fromSpec("tensor(x[10],y[])")); - context.setType("query('xy2')", TensorType.fromSpec("tensor(x[],y[10])")); + context.setType("query(x1)", TensorType.fromSpec("tensor(x[])")); + context.setType("query(x2)", TensorType.fromSpec("tensor(x[10])")); + context.setType("query(y1)", TensorType.fromSpec("tensor(y[])")); + context.setType("query(xy1)", TensorType.fromSpec("tensor(x[10],y[])")); + context.setType("query(xy2)", TensorType.fromSpec("tensor(x[],y[10])")); assertType("tensor(x[])", "query(x1)", context); assertType("tensor(x[])", "if (1>0, query(x1), query(x2))", context); @@ -46,7 +46,8 @@ public class TypeResolutionTestCase { fail("Expected type incompatibility exception"); } catch (IllegalArgumentException expected) { - assertEquals("An if expression must produce compatible types in both alternatives, but the 'true' type is tensor(x[]) while the 'false' type is tensor(y[])", + assertEquals("An if expression must produce compatible types in both alternatives, " + + "but the 'true' type is tensor(x[]) while the 'false' type is tensor(y[])", expected.getMessage()); } catch (ParseException e) { diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/MnistSoftmaxImportTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/MnistSoftmaxImportTestCase.java index 01dd15d5fa0..ad5abd4c03d 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/MnistSoftmaxImportTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/MnistSoftmaxImportTestCase.java @@ -20,15 +20,15 @@ public class MnistSoftmaxImportTestCase { TestableTensorFlowModel model = new TestableTensorFlowModel("src/test/files/integration/tensorflow/mnist_softmax/saved"); // Check constants - assertEquals(2, model.get().constants().size()); + assertEquals(2, model.get().largeConstants().size()); - Tensor constant0 = model.get().constants().get("Variable"); + Tensor constant0 = model.get().largeConstants().get("Variable"); assertNotNull(constant0); assertEquals(new TensorType.Builder().indexed("d0", 784).indexed("d1", 10).build(), constant0.type()); assertEquals(7840, constant0.size()); - Tensor constant1 = model.get().constants().get("Variable_1"); + Tensor constant1 = model.get().largeConstants().get("Variable_1"); assertNotNull(constant1); assertEquals(new TensorType.Builder().indexed("d0", 10).build(), constant1.type()); diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/TestableTensorFlowModel.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/TestableTensorFlowModel.java index 2c621fd2e92..ae7714b271a 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/TestableTensorFlowModel.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/TestableTensorFlowModel.java @@ -57,7 +57,8 @@ public class TestableTensorFlowModel { private Context contextFrom(TensorFlowModel result) { MapContext context = new MapContext(); - result.constants().forEach((name, tensor) -> context.put("constant(\"" + name + "\")", new TensorValue(tensor))); + result.largeConstants().forEach((name, tensor) -> context.put("constant(\"" + name + "\")", new TensorValue(tensor))); + result.smallConstants().forEach((name, tensor) -> context.put("constant(\"" + name + "\")", new TensorValue(tensor))); return context; } diff --git a/searchlib/src/vespa/searchlib/docstore/documentstore.cpp b/searchlib/src/vespa/searchlib/docstore/documentstore.cpp index 42920a093eb..6345e414b99 100644 --- a/searchlib/src/vespa/searchlib/docstore/documentstore.cpp +++ b/searchlib/src/vespa/searchlib/docstore/documentstore.cpp @@ -88,6 +88,8 @@ public: * value along with compression config. */ void set(vespalib::DataBuffer &&buf, ssize_t len, const CompressionConfig &compression); + // Keep buffer uncompressed + void set(vespalib::DataBuffer &&buf, ssize_t len); /** * Decompress value into temporary buffer and deserialize document from @@ -125,6 +127,12 @@ private: CompressionConfig _compression; }; + +void +Value::set(vespalib::DataBuffer &&buf, ssize_t len) { + set(std::move(buf), len, CompressionConfig()); +} + void Value::set(vespalib::DataBuffer &&buf, ssize_t len, const CompressionConfig &compression) { //Underlying buffer must be identical to allow swap. @@ -436,7 +444,7 @@ DocumentStore::WrapVisitor<Visitor>::visit(uint32_t lid, buf.writeBytes(buffer, sz); ssize_t len = sz; if (len > 0) { - value.set(std::move(buf), len, _compression); + value.set(std::move(buf), len); } if (! value.empty()) { document::Document::UP doc(value.deserializeDocument(_repo)); diff --git a/storage/src/vespa/storage/storageserver/storagenode.cpp b/storage/src/vespa/storage/storageserver/storagenode.cpp index 6be8f0a5ec8..aa5475df823 100644 --- a/storage/src/vespa/storage/storageserver/storagenode.cpp +++ b/storage/src/vespa/storage/storageserver/storagenode.cpp @@ -215,7 +215,9 @@ StorageNode::initialize() // and the like. Note that at this time, all metrics should hopefully // have been created, such that we don't need to pay the extra cost of // reinitializing metric manager often. - _context.getComponentRegister().getMetricManager().init(_configUri, _context.getThreadPool()); + if ( ! _context.getComponentRegister().getMetricManager().isInitialized() ) { + _context.getComponentRegister().getMetricManager().init(_configUri, _context.getThreadPool()); + } if (_chain) { LOG(debug, "Storage chain configured. Calling open()"); diff --git a/streamingvisitors/src/vespa/searchvisitor/hitcollector.h b/streamingvisitors/src/vespa/searchvisitor/hitcollector.h index ebc2f15e23c..2f71c5d7a71 100644 --- a/streamingvisitors/src/vespa/searchvisitor/hitcollector.h +++ b/streamingvisitors/src/vespa/searchvisitor/hitcollector.h @@ -92,7 +92,7 @@ public: /** * Adds a hit to this hit collector. * Make sure that the hits are added in increasing local docId order. - * If you add a NULL document you should not use getDocSum() or fillSearchResult(), + * If you add a nullptr document you should not use getDocSum() or fillSearchResult(), * as these functions expect valid documents. * * @param doc The document that is a hit. Must be kept alive on the outside. @@ -104,7 +104,7 @@ public: /** * Adds a hit to this hit collector. * Make sure that the hits are added in increasing local docId order. - * If you add a NULL document you should not use getDocSum() or fillSearchResult(), + * If you add a nullptr document you should not use getDocSum() or fillSearchResult(), * as these functions expect valid documents. * * @param doc The document that is a hit. Must be kept alive on the outside. diff --git a/streamingvisitors/src/vespa/searchvisitor/indexenvironment.cpp b/streamingvisitors/src/vespa/searchvisitor/indexenvironment.cpp index a4ecf655c3d..53e1910b564 100644 --- a/streamingvisitors/src/vespa/searchvisitor/indexenvironment.cpp +++ b/streamingvisitors/src/vespa/searchvisitor/indexenvironment.cpp @@ -17,20 +17,33 @@ IndexEnvironment::IndexEnvironment(const ITableManager & tableManager) : { } -IndexEnvironment::~IndexEnvironment() {} +IndexEnvironment::~IndexEnvironment() = default; bool IndexEnvironment::addField(const vespalib::string & name, bool isAttribute) { - if (getFieldByName(name) != NULL) { + if (getFieldByName(name) != nullptr) { return false; } - FieldInfo info(isAttribute ? FieldType::ATTRIBUTE : FieldType::INDEX, FieldInfo::CollectionType::SINGLE, name, _fields.size()); + FieldInfo info(isAttribute ? FieldType::ATTRIBUTE : FieldType::INDEX, + FieldInfo::CollectionType::SINGLE, name, _fields.size()); info.addAttribute(); // we are able to produce needed attributes at query time _fields.push_back(info); _fieldNames[info.name()] = info.id(); return true; } +void +IndexEnvironment::hintAttributeAccess(const string & name) const { + if (name.empty()) { + return; + } + if (_motivation == RANK) { + _rankAttributes.insert(name); + } else { + _dumpAttributes.insert(name); + } +} + } // namespace storage diff --git a/streamingvisitors/src/vespa/searchvisitor/indexenvironment.h b/streamingvisitors/src/vespa/searchvisitor/indexenvironment.h index a11e5300f03..832d99b4dde 100644 --- a/streamingvisitors/src/vespa/searchvisitor/indexenvironment.h +++ b/streamingvisitors/src/vespa/searchvisitor/indexenvironment.h @@ -34,57 +34,42 @@ public: IndexEnvironment(const search::fef::ITableManager & tableManager); ~IndexEnvironment(); - // inherit documentation - virtual const search::fef::Properties & getProperties() const override { return _properties; } + const search::fef::Properties & getProperties() const override { return _properties; } - // inherit documentation - virtual uint32_t getNumFields() const override { return _fields.size(); } + uint32_t getNumFields() const override { return _fields.size(); } - // inherit documentation - virtual const search::fef::FieldInfo * getField(uint32_t id) const override { + const search::fef::FieldInfo * getField(uint32_t id) const override { if (id >= _fields.size()) { - return NULL; + return nullptr; } return &_fields[id]; } - // inherit documentation - virtual const search::fef::FieldInfo * getFieldByName(const string & name) const override { - StringInt32Map::const_iterator itr = _fieldNames.find(name); + const search::fef::FieldInfo * getFieldByName(const string & name) const override { + auto itr = _fieldNames.find(name); if (itr == _fieldNames.end()) { - return NULL; + return nullptr; } return getField(itr->second); } - // inherit documentation - virtual const search::fef::ITableManager & getTableManager() const override { return *_tableManager; } + const search::fef::ITableManager & getTableManager() const override { + return *_tableManager; + } - virtual FeatureMotivation getFeatureMotivation() const override { + FeatureMotivation getFeatureMotivation() const override { return _motivation; } - // inherit documentation - virtual void hintFeatureMotivation(FeatureMotivation motivation) const override { + void hintFeatureMotivation(FeatureMotivation motivation) const override { _motivation = motivation; } - // inherit documentation - virtual void hintFieldAccess(uint32_t) const override {} + void hintFieldAccess(uint32_t) const override {} - // inherit documentation - virtual void hintAttributeAccess(const string & name) const override { - if (name.empty()) { - return; - } - if (_motivation == RANK) { - _rankAttributes.insert(name); - } else { - _dumpAttributes.insert(name); - } - } + void hintAttributeAccess(const string & name) const override; - virtual vespalib::eval::ConstantValue::UP getConstantValue(const vespalib::string &) const override { + vespalib::eval::ConstantValue::UP getConstantValue(const vespalib::string &) const override { return vespalib::eval::ConstantValue::UP(); } diff --git a/streamingvisitors/src/vespa/searchvisitor/queryenvironment.h b/streamingvisitors/src/vespa/searchvisitor/queryenvironment.h index 33b746178d5..b9391ac838c 100644 --- a/streamingvisitors/src/vespa/searchvisitor/queryenvironment.h +++ b/streamingvisitors/src/vespa/searchvisitor/queryenvironment.h @@ -31,7 +31,7 @@ public: QueryEnvironment(const vespalib::string & location, const IndexEnvironment & indexEnv, const search::fef::Properties & properties, - const search::IAttributeManager * attrMgr = NULL); + const search::IAttributeManager * attrMgr = nullptr); ~QueryEnvironment(); // inherit documentation @@ -43,7 +43,7 @@ public: // inherit documentation virtual const search::fef::ITermData *getTerm(uint32_t idx) const override { if (idx >= _queryTerms.size()) { - return NULL; + return nullptr; } return _queryTerms[idx]; } diff --git a/streamingvisitors/src/vespa/searchvisitor/querywrapper.cpp b/streamingvisitors/src/vespa/searchvisitor/querywrapper.cpp index 01f29ef8121..a4541eb0440 100644 --- a/streamingvisitors/src/vespa/searchvisitor/querywrapper.cpp +++ b/streamingvisitors/src/vespa/searchvisitor/querywrapper.cpp @@ -27,7 +27,7 @@ QueryWrapper::PhraseList::findPhrase(QueryTerm * term, size_t & index) } } } - return NULL; + return nullptr; } QueryWrapper::QueryWrapper(Query & query) : diff --git a/streamingvisitors/src/vespa/searchvisitor/querywrapper.h b/streamingvisitors/src/vespa/searchvisitor/querywrapper.h index eb8d15f8408..94ba63ef569 100644 --- a/streamingvisitors/src/vespa/searchvisitor/querywrapper.h +++ b/streamingvisitors/src/vespa/searchvisitor/querywrapper.h @@ -30,8 +30,8 @@ public: public: Term() : - _term(NULL), - _parent(NULL), + _term(nullptr), + _parent(nullptr), _index(0) { } @@ -44,9 +44,9 @@ public: search::QueryTerm * getTerm() { return _term; } search::PhraseQueryNode * getParent() { return _parent; } size_t getIndex() const { return _index; } - bool isPhraseTerm() const { return _parent != NULL; } + bool isPhraseTerm() const { return _parent != nullptr; } bool isFirstPhraseTerm() const { return isPhraseTerm() && getIndex() == 0; } - size_t getPosAdjust() const { return _parent != NULL ? _parent->width() - 1 : 0; } + size_t getPosAdjust() const { return _parent != nullptr ? _parent->width() - 1 : 0; } }; typedef std::vector<Term> TermList; diff --git a/streamingvisitors/src/vespa/searchvisitor/rankmanager.cpp b/streamingvisitors/src/vespa/searchvisitor/rankmanager.cpp index b2623adca52..4ed4e9462d3 100644 --- a/streamingvisitors/src/vespa/searchvisitor/rankmanager.cpp +++ b/streamingvisitors/src/vespa/searchvisitor/rankmanager.cpp @@ -5,6 +5,7 @@ #include <vespa/searchlib/fef/functiontablefactory.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/util/exception.h> +#include <vespa/vsm/common/document.h> #include <vespa/log/log.h> LOG_SETUP(".searchvisitor.rankmanager"); @@ -18,6 +19,7 @@ using search::fef::Properties; using search::fef::RankSetup; using vsm::VsmfieldsHandle; using vsm::VSMAdapter; +using vsm::FieldIdTList; namespace storage { @@ -49,33 +51,50 @@ RankManager::Snapshot::detectFields(const VsmfieldsHandle & fields) } } +namespace { + +FieldIdTList +buildFieldSet(const VsmfieldsConfig::Documenttype::Index & ci, const search::fef::IIndexEnvironment & indexEnv, + const VsmfieldsConfig::Documenttype::IndexVector & indexes) +{ + LOG(spam, "Index %s with %zd fields", ci.name.c_str(), ci.field.size()); + FieldIdTList ifm; + for (const VsmfieldsConfig::Documenttype::Index::Field & cf : ci.field) { + LOG(spam, "Parsing field %s", cf.name.c_str()); + auto foundIndex = std::find_if(indexes.begin(), indexes.end(), + [&cf](const auto & v) { return v.name == cf.name;}); + if ((foundIndex != indexes.end()) && (cf.name != ci.name)) { + FieldIdTList sub = buildFieldSet(*foundIndex, indexEnv, indexes); + ifm.insert(ifm.end(), sub.begin(), sub.end()); + } else { + const FieldInfo * info = indexEnv.getFieldByName(cf.name); + if (info != nullptr) { + LOG(debug, "Adding field '%s' to view in index '%s' (field id '%u')", + cf.name.c_str(), ci.name.c_str(), info->id()); + ifm.push_back(info->id()); + } else { + LOG(warning, "Field '%s' is not registred in the index environment. " + "Cannot add to index view.", cf.name.c_str()); + } + } + } + return ifm; +} + +} + void RankManager::Snapshot::buildFieldMappings(const VsmfieldsHandle & fields) { - for (uint32_t i = 0; i < fields->documenttype.size(); ++i) { - const char * dname = fields->documenttype[i].name.c_str(); - LOG(debug, "Looking through indexes for documenttype '%s'", dname); - for (uint32_t j = 0; j < fields->documenttype[i].index.size(); ++j) { - const char * iname = fields->documenttype[i].index[j].name.c_str(); - LOG(debug, "Looking through fields for index '%s'", iname); - View view; - for (uint32_t k = 0; k < fields->documenttype[i].index[j].field.size(); ++k) { - const char * fname = fields->documenttype[i].index[j].field[k].name.c_str(); - const FieldInfo * info = _protoEnv.getFieldByName(vespalib::string(fname)); - if (info != NULL) { - LOG(debug, "Adding field '%s' to view in index '%s' (field id '%u')", - fname, iname, info->id()); - view.push_back(info->id()); - } else { - LOG(warning, "Field '%s' is not registred in the index environment. " - "Cannot add to index view.", fname); - } - } - if (_views.find(iname) == _views.end()) { + for(const VsmfieldsConfig::Documenttype & di : fields->documenttype) { + LOG(debug, "Looking through indexes for documenttype '%s'", di.name.c_str()); + for(const VsmfieldsConfig::Documenttype::Index & ci : di.index) { + FieldIdTList view = buildFieldSet(ci, _protoEnv, di.index); + if (_views.find(ci.name) == _views.end()) { std::sort(view.begin(), view.end()); // lowest field id first - _views[iname] = view; + _views[ci.name] = view; } else { - LOG(warning, "We already have a view for index '%s'. Drop the new view.", iname); + LOG(warning, "We already have a view for index '%s'. Drop the new view.", ci.name.c_str()); } } } @@ -129,6 +148,8 @@ RankManager::Snapshot::Snapshot() : _tableManager.addFactory(search::fef::ITableFactory::SP(new search::fef::FunctionTableFactory(256))); } +RankManager::Snapshot::~Snapshot() = default; + bool RankManager::Snapshot::setup(const RankManager & rm, const std::vector<NamedPropertySet> & properties) { @@ -186,16 +207,12 @@ RankManager::RankManager(VSMAdapter * const vsmAdapter) : search::features::setup_search_features(_blueprintFactory); } -RankManager::~RankManager() -{ -} +RankManager::~RankManager() = default; void RankManager::configure(const vsm::VSMConfigSnapshot & snap) { notify(snap); } - - -} // namespace storage - + +} diff --git a/streamingvisitors/src/vespa/searchvisitor/rankmanager.h b/streamingvisitors/src/vespa/searchvisitor/rankmanager.h index e4d7b26d0b9..573833a93f8 100644 --- a/streamingvisitors/src/vespa/searchvisitor/rankmanager.h +++ b/streamingvisitors/src/vespa/searchvisitor/rankmanager.h @@ -2,12 +2,12 @@ #pragma once +#include "indexenvironment.h" #include <vespa/config-rank-profiles.h> #include <vespa/searchlib/fef/blueprintfactory.h> #include <vespa/searchlib/fef/ranksetup.h> #include <vespa/searchlib/fef/tablemanager.h> #include <vespa/vsm/vsm/vsm-adapter.h> -#include "indexenvironment.h" namespace storage { @@ -50,6 +50,7 @@ public: public: typedef std::shared_ptr<Snapshot> SP; Snapshot(); + ~Snapshot(); const std::vector<NamedPropertySet> & getProperties() const { return _properties; } bool setup(const RankManager & manager, const vespa::config::search::RankProfilesConfig & cfg); bool setup(const RankManager & manager, const std::vector<NamedPropertySet> & properties); @@ -64,7 +65,7 @@ public: if (itr != _views.end()) { return &itr->second; } - return NULL; + return nullptr; } }; diff --git a/streamingvisitors/src/vespa/searchvisitor/rankprocessor.cpp b/streamingvisitors/src/vespa/searchvisitor/rankprocessor.cpp index dc9cfdc7efd..dd52c020042 100644 --- a/streamingvisitors/src/vespa/searchvisitor/rankprocessor.cpp +++ b/streamingvisitors/src/vespa/searchvisitor/rankprocessor.cpp @@ -68,7 +68,7 @@ RankProcessor::initQueryEnvironment() vespalib::string expandedIndexName = vsm::FieldSearchSpecMap::stripNonFields(terms[i].getTerm()->index()); const RankManager::View *view = _rankManagerSnapshot->getView(expandedIndexName); - if (view != NULL) { + if (view != nullptr) { RankManager::View::const_iterator iter = view->begin(); RankManager::View::const_iterator endp = view->end(); for (; iter != endp; ++iter) { diff --git a/streamingvisitors/src/vespa/searchvisitor/searchenvironment.cpp b/streamingvisitors/src/vespa/searchvisitor/searchenvironment.cpp index 7ebb71f1855..7de96f0c4ab 100644 --- a/streamingvisitors/src/vespa/searchvisitor/searchenvironment.cpp +++ b/streamingvisitors/src/vespa/searchvisitor/searchenvironment.cpp @@ -65,7 +65,7 @@ SearchEnvironment::Env & SearchEnvironment::getEnv(const vespalib::string & searchCluster) { config::ConfigUri searchClusterUri(_configUri.createWithNewId(searchCluster)); - if (_localEnvMap == NULL) { + if (_localEnvMap == nullptr) { EnvMapUP envMap = std::make_unique<EnvMap>(); _localEnvMap = envMap.get(); vespalib::LockGuard guard(_lock); diff --git a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp index e3483e8beac..f42dfe5af98 100644 --- a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp +++ b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp @@ -105,7 +105,7 @@ SearchVisitor::SummaryGenerator::SummaryGenerator() : _callback(), _docsumState(_callback), _docsumFilter(), - _docsumWriter(NULL), + _docsumWriter(nullptr), _rawBuf(4096) { } @@ -116,7 +116,7 @@ SearchVisitor::SummaryGenerator::~SummaryGenerator() { } vespalib::ConstBufferRef SearchVisitor::SummaryGenerator::fillSummary(AttributeVector::DocId lid, const HitsAggregationResult::SummaryClassType & summaryClass) { - if (_docsumWriter != NULL) { + if (_docsumWriter != nullptr) { _rawBuf.reset(); _docsumState._args.setResultClassName(summaryClass); uint32_t docsumLen = _docsumWriter->WriteDocsum(lid, &_docsumState, _docsumFilter.get(), &_rawBuf); @@ -168,7 +168,7 @@ SearchVisitor::SearchVisitor(StorageComponent& component, Visitor(component), _env(static_cast<SearchEnvironment &>(vEnv)), _params(params), - _vsmAdapter(NULL), + _vsmAdapter(nullptr), _docSearchedCount(0), _hitCount(0), _hitsRejectedCount(0), @@ -185,7 +185,7 @@ SearchVisitor::SearchVisitor(StorageComponent& component, _groupingList(), _attributeFields(), _sortList(), - _docsumWriter(NULL), + _docsumWriter(nullptr), _searchBuffer(new vsm::SearcherBuf()), _tmpSortBuffer(256), _documentIdAttributeBacking(new search::SingleStringExtAttribute("[docid]") ), @@ -213,7 +213,7 @@ void SearchVisitor::init(const Parameters & params) size_t wantedSummaryCount(10); if (params.get("summarycount", valueRef) ) { vespalib::string tmp(valueRef.data(), valueRef.size()); - wantedSummaryCount = strtoul(tmp.c_str(), NULL, 0); + wantedSummaryCount = strtoul(tmp.c_str(), nullptr, 0); LOG(debug, "Received summary count: %ld", wantedSummaryCount); } _queryResult->getSearchResult().setWantedHitCount(wantedSummaryCount); @@ -226,8 +226,8 @@ void SearchVisitor::init(const Parameters & params) if (params.get("queryflags", valueRef) ) { vespalib::string tmp(valueRef.data(), valueRef.size()); - LOG(debug, "Received query flags: 0x%lx", strtoul(tmp.c_str(), NULL, 0)); - uint32_t queryFlags = strtoul(tmp.c_str(), NULL, 0); + LOG(debug, "Received query flags: 0x%lx", strtoul(tmp.c_str(), nullptr, 0)); + uint32_t queryFlags = strtoul(tmp.c_str(), nullptr, 0); _rankController.setDumpFeatures((queryFlags & search::fs4transport::QFLAG_DUMP_FEATURES) != 0); LOG(debug, "QFLAG_DUMP_FEATURES: %s", _rankController.getDumpFeatures() ? "true" : "false"); } @@ -432,7 +432,7 @@ SearchVisitor::RankController::processHintedAttributes(const IndexEnvironment & for (const vespalib::string & name : attributes) { LOG(debug, "Process attribute access hint (%s): '%s'", rank ? "rank" : "dump", name.c_str()); const search::fef::FieldInfo * fieldInfo = indexEnv.getFieldByName(name); - if (fieldInfo != NULL) { + if (fieldInfo != nullptr) { bool found = false; uint32_t fid = fieldInfo->id(); for (size_t j = 0; !found && (j < attributeFields.size()); ++j) { @@ -457,8 +457,8 @@ SearchVisitor::RankController::processHintedAttributes(const IndexEnvironment & SearchVisitor::RankController::RankController() : _rankProfile("default"), - _rankManagerSnapshot(NULL), - _rankSetup(NULL), + _rankManagerSnapshot(nullptr), + _rankSetup(nullptr), _queryProperties(), _hasRanking(false), _rankProcessor(), @@ -694,7 +694,7 @@ SearchVisitor::setupDocsumObjects() _vsmAdapter->getDocsumTools()->getDocsumWriter()->InitState(_attrMan, ds); _summaryGenerator.setDocsumWriter(*_vsmAdapter->getDocsumTools()->getDocsumWriter()); for (const IAttributeVector * v : ds->_attributes) { - if (v != NULL) { + if (v != nullptr) { vespalib::string name(v->getName()); vsm::FieldIdT fid = _fieldSearchSpecMap.nameIdMap().fieldNo(name); if ( fid != StringFieldIdTMap::npos ) { @@ -872,7 +872,7 @@ SearchVisitor::handleDocuments(const document::BucketId&, HitCounter& hitCounter) { (void) hitCounter; - if (_vsmAdapter == NULL) { + if (_vsmAdapter == nullptr) { init(_params); } if ( ! _rankController.valid() ) { @@ -889,7 +889,7 @@ SearchVisitor::handleDocuments(const document::BucketId&, StorageDocument::UP document(new StorageDocument(entry->releaseDocument(), _fieldPathMap, highestFieldNo)); try { - if (defaultDocType != NULL + if (defaultDocType != nullptr && !compatibleDocumentTypes(*defaultDocType, document->docDoc().getType())) { LOG(debug, "Skipping document of type '%s' when handling only documents of type '%s'", @@ -1001,7 +1001,7 @@ SearchVisitor::fillAttributeVectors(const vespalib::string & documentId, const S AttributeVector & attrV = const_cast<AttributeVector & >(*finfoGuard); AttributeVector::DocId docId(0); attrV.addDoc(docId); - if (subDoc.getFieldValue() != NULL) { + if (subDoc.getFieldValue() != nullptr) { LOG(debug, "value = '%s'", subDoc.getFieldValue()->toString().c_str()); if (isPosition) { LOG(spam, "Position"); @@ -1053,7 +1053,7 @@ void SearchVisitor::completedBucket(const document::BucketId&, HitCounter&) void SearchVisitor::completedVisitingInternal(HitCounter& hitCounter) { - if (_vsmAdapter == NULL) { + if (_vsmAdapter == nullptr) { init(_params); } LOG(debug, "Completed visiting"); @@ -1120,7 +1120,7 @@ SearchVisitor::generateDocumentSummaries() vdslib::SearchResult & searchResult(_queryResult->getSearchResult()); vdslib::DocumentSummary & documentSummary(_queryResult->getDocumentSummary()); for (size_t i(0), m(searchResult.getHitCount()); (i < m) && (i < searchResult.getWantedHitCount()); i++ ) { - const char * docId(NULL); + const char * docId(nullptr); vdslib::SearchResult::RankType rank(0); uint32_t lid = searchResult.getHit(i, docId, rank); vespalib::ConstBufferRef docsum = _summaryGenerator.fillSummary(lid, _summaryClass); diff --git a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h index 5f497732802..ceadd8a9c79 100644 --- a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h +++ b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h @@ -60,7 +60,7 @@ private: AttrInfo(vsm::FieldIdT fid, search::AttributeGuard::UP attr) : _field(fid), _ascending(true), - _converter(NULL), + _converter(nullptr), _attr(std::move(attr)) { } /** @@ -135,7 +135,7 @@ private: public: RankController(); ~RankController(); - bool valid() const { return _rankProcessor.get() != NULL; } + bool valid() const { return _rankProcessor.get() != nullptr; } void setRankProfile(const vespalib::string &rankProfile) { _rankProfile = rankProfile; } const vespalib::string &getRankProfile() const { return _rankProfile; } void setRankManagerSnapshot(const RankManager::Snapshot::SP & snapshot) { _rankManagerSnapshot = snapshot; } diff --git a/vagrant/Vagrantfile b/vagrant/Vagrantfile index aa7688570ff..41f22f22c80 100644 --- a/vagrant/Vagrantfile +++ b/vagrant/Vagrantfile @@ -4,8 +4,8 @@ # For a complete reference, please see the online documentation at https://docs.vagrantup.com. Vagrant.configure("2") do |config| - config.vm.box = "TODO" - config.vm.box_url = "TODO" + config.vm.box = "centos7-desktop" + config.vm.box_url = "/Users/balder/vagrant/centos7-desktop-17.0915.1.box" config.ssh.forward_agent = true @@ -16,8 +16,8 @@ Vagrant.configure("2") do |config| vb.gui = true vb.name = "vespa-dev" - vb.memory = "8192" - vb.cpus = 4 + vb.memory = "10240" + vb.cpus = 8 end # Install required and nice-to-have packages diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/BucketSpaceEnumerator.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/BucketSpaceEnumerator.java index 24692859266..6fbec41aba1 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/BucketSpaceEnumerator.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/BucketSpaceEnumerator.java @@ -9,7 +9,7 @@ import java.util.Map; import java.util.stream.Collectors; /** - * TODO description + * Class that based on BucketspacesConfig builds a mapping from document type to which bucket space it belongs to. */ class BucketSpaceEnumerator { diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandlerImpl.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandlerImpl.java index d6ae4a9285a..af8650a8e7c 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandlerImpl.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandlerImpl.java @@ -323,7 +323,7 @@ public class OperationHandlerImpl implements OperationHandler { if (!targetBucketSpace.isPresent()) { throw new RestApiException(Response.createErrorResponse(400, String.format( "Document type '%s' in cluster '%s' is not mapped to a known bucket space", docType, clusterDef.getName()), - RestUri.apiErrorCodes.UNKNOWN_BUCKET_SPACE)); // TODO own code + RestUri.apiErrorCodes.UNKNOWN_BUCKET_SPACE)); } return new BucketSpaceRoute(clusterDefToRoute(clusterDef), targetBucketSpace.get()); } diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/RestUri.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/RestUri.java index 15d1b54adbe..e3423eec2c8 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/RestUri.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/RestUri.java @@ -34,14 +34,15 @@ public class RestUri { URL_PARSING(-6), INVALID_CREATE_VALUE(-7), TOO_MANY_PARALLEL_REQUESTS(-8), - MISSING_CLUSTER(-9), UNKNOWN_BUCKET_SPACE(-9), INTERNAL_EXCEPTION(-9), + MISSING_CLUSTER(-9), INTERNAL_EXCEPTION(-9), DOCUMENT_CONDITION_NOT_MET(-10), DOCUMENT_EXCPETION(-11), PARSER_ERROR(-11), GROUP_AND_EXPRESSION_ERROR(-12), TIME_OUT(-13), INTERRUPTED(-14), - UNSPECIFIED(-15); + UNSPECIFIED(-15), + UNKNOWN_BUCKET_SPACE(-16); public final long value; apiErrorCodes(long value) { diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerRemove.java b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerRemove.java index c24398388d5..36ab8090e95 100755 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerRemove.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerRemove.java @@ -7,6 +7,7 @@ import com.yahoo.cloud.config.ClusterListConfig; import com.yahoo.cloud.config.SlobroksConfig; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.protect.Error; import com.yahoo.document.DocumentId; import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.feedapi.FeedContext; @@ -78,7 +79,7 @@ public class VespaFeedHandlerRemove extends VespaFeedHandlerBase { long millis = getTimeoutMillis(request); boolean completed = sender.waitForPending(millis); if ( ! completed) - response.addError("Timed out after "+millis+" ms waiting for responses"); + response.addError(Error.TIMEOUT, "Timed out after "+millis+" ms waiting for responses"); return response; } diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerRemoveLocation.java b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerRemoveLocation.java index 2f0526a21ef..04ca6798b4c 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerRemoveLocation.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerRemoveLocation.java @@ -7,6 +7,7 @@ import com.yahoo.cloud.config.ClusterListConfig; import com.yahoo.cloud.config.SlobroksConfig; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.protect.Error; import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.documentapi.messagebus.protocol.RemoveLocationMessage; import com.yahoo.feedapi.FeedContext; @@ -78,7 +79,7 @@ public class VespaFeedHandlerRemoveLocation extends VespaFeedHandlerBase { long millis = getTimeoutMillis(request); boolean completed = sender.waitForPending(millis); if ( ! completed) - response.addError("Timed out after "+millis+" ms waiting for responses"); + response.addError(Error.TIMEOUT, "Timed out after "+millis+" ms waiting for responses"); return response; } diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/OperationHandlerImplTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/OperationHandlerImplTest.java index 3a6782f5830..06054bd2dbb 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/OperationHandlerImplTest.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/OperationHandlerImplTest.java @@ -212,7 +212,7 @@ public class OperationHandlerImplTest { String errorMsg = renderRestApiExceptionAsString(e); // FIXME isn't this really more of a case of unknown document type..? assertThat(errorMsg, is("{\"errors\":[{\"description\":" + - "\"UNKNOWN_BUCKET_SPACE Document type 'document-type' in cluster 'foo' is not mapped to a known bucket space\",\"id\":-9}]}")); + "\"UNKNOWN_BUCKET_SPACE Document type 'document-type' in cluster 'foo' is not mapped to a known bucket space\",\"id\":-16}]}")); } } diff --git a/vespaclient-core/src/main/java/com/yahoo/feedapi/SharedSender.java b/vespaclient-core/src/main/java/com/yahoo/feedapi/SharedSender.java index 0380fa2a7c4..b9aeb2e6e69 100755 --- a/vespaclient-core/src/main/java/com/yahoo/feedapi/SharedSender.java +++ b/vespaclient-core/src/main/java/com/yahoo/feedapi/SharedSender.java @@ -4,11 +4,12 @@ package com.yahoo.feedapi; import com.yahoo.concurrent.SystemTimer; import com.yahoo.jdisc.Metric; import com.yahoo.log.LogLevel; -import com.yahoo.messagebus.*; +import com.yahoo.messagebus.EmptyReply; +import com.yahoo.messagebus.Message; +import com.yahoo.messagebus.Reply; +import com.yahoo.messagebus.ReplyHandler; import com.yahoo.clientmetrics.RouteMetricSet; -import java.util.HashMap; -import java.util.Map; import java.util.logging.Logger; /** @@ -157,14 +158,14 @@ public class SharedSender implements ReplyHandler { log.log(LogLevel.SPAM, "Received reply for file " + owner.toString() + " count was " + owner.getPending().val()); } if (owner.isAborted()) { - log.log(LogLevel.WARNING, "Received reply for file " + owner.toString() + " which is aborted"); + log.log(LogLevel.DEBUG, "Received reply for file " + owner.toString() + " which is aborted"); owner.getPending().clear(); return; } if (owner.handleReply(r)) { owner.getPending().dec(); } else { - log.log(LogLevel.WARNING, "Received reply for file " + owner.toString() + " which wants to abort"); + log.log(LogLevel.DEBUG, "Received reply for file " + owner.toString() + " which wants to abort"); owner.getPending().clear(); } } else { diff --git a/vespaclient-core/src/main/java/com/yahoo/feedhandler/FeedResponse.java b/vespaclient-core/src/main/java/com/yahoo/feedhandler/FeedResponse.java index c235fdfce72..3b84f83fafa 100755 --- a/vespaclient-core/src/main/java/com/yahoo/feedhandler/FeedResponse.java +++ b/vespaclient-core/src/main/java/com/yahoo/feedhandler/FeedResponse.java @@ -9,8 +9,8 @@ import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage; import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentMessage; import com.yahoo.documentapi.messagebus.protocol.UpdateDocumentMessage; import com.yahoo.feedapi.SharedSender; -import com.yahoo.log.LogLevel; import com.yahoo.messagebus.Error; +import com.yahoo.messagebus.ErrorCode; import com.yahoo.messagebus.Message; import com.yahoo.messagebus.Reply; import com.yahoo.search.result.ErrorMessage; @@ -122,7 +122,7 @@ public final class FeedResponse extends HttpResponse implements SharedSender.Res String str = out.toString(); log.finest(str); - addError(str); + addError(convertErrorCode(err.getCode()), str); } if (abortOnError) { isAborted = true; @@ -149,6 +149,11 @@ public final class FeedResponse extends HttpResponse implements SharedSender.Res errors.add(error); return this; } + public FeedResponse addError(com.yahoo.container.protect.Error code, String error) { + errorMessages.add(new ErrorMessage(code.code, error)); + errors.add(error); + return this; + } public List<String> getErrorList() { return errors; @@ -162,6 +167,27 @@ public final class FeedResponse extends HttpResponse implements SharedSender.Res return errors.anyMatch(e -> e.getCode() != DocumentProtocol.ERROR_TEST_AND_SET_CONDITION_FAILED); } + private static com.yahoo.container.protect.Error convertErrorCode(int error) { + // We should try to enumerate these error a bit finer. + // Like busy, no space etc. + if (error == DocumentProtocol.ERROR_NO_SPACE) { + return com.yahoo.container.protect.Error.INSUFFICIENT_STORAGE; + } else if (isTransientError(error)) { + return com.yahoo.container.protect.Error.INTERNAL_SERVER_ERROR; + } if (isFatalError(error)) { + return com.yahoo.container.protect.Error.INTERNAL_SERVER_ERROR; + } + return com.yahoo.container.protect.Error.INTERNAL_SERVER_ERROR; + } + + private static boolean isFatalError(int error) { + return (error >= ErrorCode.FATAL_ERROR) && (error < ErrorCode.ERROR_LIMIT); + } + + private static boolean isTransientError(int error) { + return (error >= ErrorCode.TRANSIENT_ERROR) && (error < ErrorCode.FATAL_ERROR); + } + public boolean isSuccess() { return errors.isEmpty(); } diff --git a/vespaclient-core/src/main/java/com/yahoo/feedhandler/VespaFeedHandler.java b/vespaclient-core/src/main/java/com/yahoo/feedhandler/VespaFeedHandler.java index 4b7d913c0f4..8180bfd84ea 100755 --- a/vespaclient-core/src/main/java/com/yahoo/feedhandler/VespaFeedHandler.java +++ b/vespaclient-core/src/main/java/com/yahoo/feedhandler/VespaFeedHandler.java @@ -8,6 +8,7 @@ import com.yahoo.cloud.config.SlobroksConfig; import com.yahoo.container.jdisc.EmptyResponse; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.protect.Error; import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.feedapi.DocprocMessageProcessor; import com.yahoo.feedapi.FeedContext; @@ -106,7 +107,7 @@ public final class VespaFeedHandler extends VespaFeedHandlerBase { long millis = getTimeoutMillis(request); boolean completed = sender.waitForPending(millis); if (!completed) { - response.addError("Timed out after " + millis + " ms waiting for responses"); + response.addError(Error.TIMEOUT, "Timed out after " + millis + " ms waiting for responses"); } response.done(); return response; diff --git a/vsm/src/vespa/vsm/vsm/fieldsearchspec.cpp b/vsm/src/vespa/vsm/vsm/fieldsearchspec.cpp index dd208b1ca48..8855923610c 100644 --- a/vsm/src/vespa/vsm/vsm/fieldsearchspec.cpp +++ b/vsm/src/vespa/vsm/vsm/fieldsearchspec.cpp @@ -21,6 +21,24 @@ using search::ConstQueryTermList; namespace vsm { +namespace { + +void setMatchType(FieldSearcherContainer & searcher, vespalib::stringref arg1) { + if (arg1 == "prefix") { + searcher->setMatchType(FieldSearcher::PREFIX); + } else if (arg1 == "substring") { + searcher->setMatchType(FieldSearcher::SUBSTRING); + } else if (arg1 == "suffix") { + searcher->setMatchType(FieldSearcher::SUFFIX); + } else if (arg1 == "exact") { + searcher->setMatchType(FieldSearcher::EXACT); + } else if (arg1 == "word") { + searcher->setMatchType(FieldSearcher::EXACT); + } +} + +} + FieldSearchSpec::FieldSearchSpec() : _id(0), _name(), @@ -31,10 +49,11 @@ FieldSearchSpec::FieldSearchSpec() : _reconfigured(false) { } -FieldSearchSpec::~FieldSearchSpec() {} +FieldSearchSpec::~FieldSearchSpec() = default; FieldSearchSpec::FieldSearchSpec(const FieldIdT & fid, const vespalib::string & fname, - VsmfieldsConfig::Fieldspec::Searchmethod searchDef, const vespalib::string & arg1, size_t maxLength_) : + VsmfieldsConfig::Fieldspec::Searchmethod searchDef, + const vespalib::string & arg1, size_t maxLength_) : _id(fid), _name(fname), _maxLength(maxLength_), @@ -79,17 +98,7 @@ FieldSearchSpec::FieldSearchSpec(const FieldIdT & fid, const vespalib::string & break; } if (_searcher.valid()) { - if (arg1 == "prefix") { - _searcher->setMatchType(FieldSearcher::PREFIX); - } else if (arg1 == "substring") { - _searcher->setMatchType(FieldSearcher::SUBSTRING); - } else if (arg1 == "suffix") { - _searcher->setMatchType(FieldSearcher::SUFFIX); - } else if (arg1 == "exact") { - _searcher->setMatchType(FieldSearcher::EXACT); - } else if (arg1 == "word") { - _searcher->setMatchType(FieldSearcher::EXACT); - } + setMatchType(_searcher, arg1); _searcher->maxFieldLength(maxLength()); } } @@ -112,17 +121,7 @@ FieldSearchSpec::reconfig(const search::QueryTerm & term) { _searcher = UTF8FlexibleStringFieldSearcher(id()); // preserve the basic match property of the searcher - if (_arg1 == "prefix") { - _searcher->setMatchType(FieldSearcher::PREFIX); - } else if (_arg1 == "substring") { - _searcher->setMatchType(FieldSearcher::SUBSTRING); - } else if (_arg1 == "suffix") { - _searcher->setMatchType(FieldSearcher::SUFFIX); - } else if (_arg1 == "exact") { - _searcher->setMatchType(FieldSearcher::EXACT); - } else if (_arg1 == "word") { - _searcher->setMatchType(FieldSearcher::EXACT); - } + setMatchType(_searcher, _arg1); LOG(debug, "Reconfigured to use UTF8FlexibleStringFieldSearcher (%s) for field '%s' with id '%d'", _searcher->prefix() ? "prefix" : "regular", name().c_str(), id()); _reconfigured = true; @@ -136,8 +135,7 @@ FieldSearchSpec::reconfig(const search::QueryTerm & term) vespalib::asciistream & operator <<(vespalib::asciistream & os, const FieldSearchSpec & f) { os << f._id << ' ' << f._name << ' '; - if (f._searcher.valid()) { - } else { + if ( ! f._searcher.valid()) { os << " No searcher defined.\n"; } return os; @@ -176,15 +174,15 @@ bool FieldSearchSpecMap::buildFieldsInQuery(const Query & query, StringFieldIdTM ConstQueryTermList qtl; query.getLeafs(qtl); - for (ConstQueryTermList::const_iterator it = qtl.begin(), mt = qtl.end(); it != mt; it++) { - for (DocumentTypeIndexFieldMapT::const_iterator dt(documentTypeMap().begin()), dmt(documentTypeMap().end()); dt != dmt; dt++) { - const IndexFieldMapT & fim = dt->second; - vespalib::string rawIndex((*it)->index()); + for (const auto & term : qtl) { + for (const auto & dtm : documentTypeMap()) { + const IndexFieldMapT & fim = dtm.second; + vespalib::string rawIndex(term->index()); vespalib::string index(stripNonFields(rawIndex)); IndexFieldMapT::const_iterator fIt = fim.find(index); if (fIt != fim.end()) { - for(FieldIdTList::const_iterator ifIt = fIt->second.begin(), ifMt = fIt->second.end(); ifIt != ifMt; ifIt++) { - const FieldSearchSpec & spec = specMap().find(*ifIt)->second; + for(FieldIdT fid : fIt->second) { + const FieldSearchSpec & spec = specMap().find(fid)->second; LOG(debug, "buildFieldsInQuery = rawIndex='%s', index='%s'", rawIndex.c_str(), index.c_str()); if ((rawIndex != index) && (spec.name().find(index) == 0)) { vespalib::string modIndex(rawIndex); @@ -195,7 +193,7 @@ bool FieldSearchSpecMap::buildFieldsInQuery(const Query & query, StringFieldIdTM } } } else { - LOG(warning, "No valid indexes registered for index %s", (*it)->index().c_str()); + LOG(warning, "No valid indexes registered for index %s", term->index().c_str()); retval = false; } } @@ -203,28 +201,49 @@ bool FieldSearchSpecMap::buildFieldsInQuery(const Query & query, StringFieldIdTM return retval; } -void FieldSearchSpecMap::buildFieldsInQuery(const std::vector<vespalib::string> & otherFieldsNeeded, StringFieldIdTMap & fieldsInQuery) const +void FieldSearchSpecMap::buildFromConfig(const std::vector<vespalib::string> & otherFieldsNeeded) { - for (size_t i(0), m(otherFieldsNeeded.size()); i < m; i++) { - fieldsInQuery.add(otherFieldsNeeded[i], _nameIdMap.fieldNo(otherFieldsNeeded[i])); + for(size_t i(0), m(otherFieldsNeeded.size()); i < m; i++) { + LOG(debug, "otherFieldsNeeded[%zd] = '%s'", i, otherFieldsNeeded[i].c_str()); + _nameIdMap.add(otherFieldsNeeded[i]); } } +namespace { -void FieldSearchSpecMap::buildFromConfig(const std::vector<vespalib::string> & otherFieldsNeeded) +FieldIdTList +buildFieldSet(const VsmfieldsConfig::Documenttype::Index & ci, const FieldSearchSpecMapT & specMap, + const VsmfieldsConfig::Documenttype::IndexVector & indexes) { - for(size_t i(0), m(otherFieldsNeeded.size()); i < m; i++) { - LOG(debug, "otherFieldsNeeded[%zd] = '%s'", i, otherFieldsNeeded[i].c_str()); - _nameIdMap.add(otherFieldsNeeded[i]); + LOG(spam, "Index %s with %zd fields", ci.name.c_str(), ci.field.size()); + FieldIdTList ifm; + for (const VsmfieldsConfig::Documenttype::Index::Field & cf : ci.field) { + LOG(spam, "Parsing field %s", cf.name.c_str()); + auto foundIndex = std::find_if(indexes.begin(), indexes.end(), + [&cf](const auto & v) { return v.name == cf.name;}); + if ((foundIndex != indexes.end()) && (cf.name != ci.name)) { + FieldIdTList sub = buildFieldSet(*foundIndex, specMap, indexes); + ifm.insert(ifm.end(), sub.begin(), sub.end()); + } else { + auto foundField = std::find_if(specMap.begin(), specMap.end(), + [&cf](const auto & v) { return v.second.name() == cf.name;} ); + if (foundField != specMap.end()) { + ifm.push_back(foundField->second.id()); + } else { + LOG(warning, "Field %s not defined. Ignoring....", cf.name.c_str()); + } + } } + return ifm; +} + } bool FieldSearchSpecMap::buildFromConfig(const VsmfieldsHandle & conf) { bool retval(true); LOG(spam, "Parsing %zd fields", conf->fieldspec.size()); - for(size_t i=0, m = conf->fieldspec.size(); i < m; i++) { - const VsmfieldsConfig::Fieldspec & cfs = conf->fieldspec[i]; + for(const VsmfieldsConfig::Fieldspec & cfs : conf->fieldspec) { LOG(spam, "Parsing %s", cfs.name.c_str()); FieldIdT fieldId = specMap().size(); FieldSearchSpec fss(fieldId, cfs.name, cfs.searchmethod, cfs.arg1.c_str(), cfs.maxlength); @@ -234,26 +253,11 @@ bool FieldSearchSpecMap::buildFromConfig(const VsmfieldsHandle & conf) } LOG(spam, "Parsing %zd document types", conf->documenttype.size()); - for(size_t d=0, dm = conf->documenttype.size(); d < dm; d++) { - const VsmfieldsConfig::Documenttype & di = conf->documenttype[d]; + for(const VsmfieldsConfig::Documenttype & di : conf->documenttype) { IndexFieldMapT indexMapp; LOG(spam, "Parsing document type %s with %zd indexes", di.name.c_str(), di.index.size()); - for(size_t i=0, m = di.index.size(); i < m; i++) { - const VsmfieldsConfig::Documenttype::Index & ci = di.index[i]; - LOG(spam, "Index %s with %zd fields", ci.name.c_str(), ci.field.size()); - FieldIdTList ifm; - for (size_t j=0, n=ci.field.size(); j < n; j++) { - const VsmfieldsConfig::Documenttype::Index::Field & cf = ci.field[j]; - LOG(spam, "Parsing field %s", cf.name.c_str()); - FieldSearchSpecMapT::const_iterator fIt, mIt; - for (fIt=specMap().begin(), mIt=specMap().end(); (fIt != mIt) && (fIt->second.name() != cf.name); fIt++); - if (fIt != mIt) { - ifm.push_back(fIt->second.id()); - } else { - LOG(warning, "Field %s not defined. Ignoring....", cf.name.c_str()); - } - } - indexMapp[ci.name] = ifm; + for(const VsmfieldsConfig::Documenttype::Index & ci : di.index) { + indexMapp[ci.name] = buildFieldSet(ci, specMap(), di.index); } _documentTypeMap[di.name] = indexMapp; } @@ -266,16 +270,13 @@ FieldSearchSpecMap::reconfigFromQuery(const search::Query & query) ConstQueryTermList qtl; query.getLeafs(qtl); - for (ConstQueryTermList::const_iterator ita = qtl.begin(); ita != qtl.end(); ++ita) { - for (DocumentTypeIndexFieldMapT::const_iterator itb = documentTypeMap().begin(); - itb != documentTypeMap().end(); ++itb) - { - const IndexFieldMapT & ifm = itb->second; - IndexFieldMapT::const_iterator itc = ifm.find((*ita)->index()); - if (itc != ifm.end()) { - for (FieldIdTList::const_iterator itd = itc->second.begin(); itd != itc->second.end(); ++itd) { - FieldSearchSpec & spec = _specMap.find(*itd)->second; - spec.reconfig(**ita); + for (const auto & termA : qtl) { + for (const auto & ifm : documentTypeMap()) { + IndexFieldMapT::const_iterator itc = ifm.second.find(termA->index()); + if (itc != ifm.second.end()) { + for (FieldIdT fid : itc->second) { + FieldSearchSpec & spec = _specMap.find(fid)->second; + spec.reconfig(*termA); } } } @@ -290,8 +291,8 @@ bool lesserField(const FieldSearcherContainer & a, const FieldSearcherContainer void FieldSearchSpecMap::buildSearcherMap(const StringFieldIdTMapT & fieldsInQuery, FieldIdTSearcherMap & fieldSearcherMap) { fieldSearcherMap.clear(); - for (StringFieldIdTMapT::const_iterator it = fieldsInQuery.begin(), mt = fieldsInQuery.end(); it != mt; it++) { - FieldIdT fId = it->second; + for (const auto & entry : fieldsInQuery) { + FieldIdT fId = entry.second; const FieldSearchSpec & spec = specMap().find(fId)->second; fieldSearcherMap.push_back(spec.searcher()); } @@ -302,19 +303,21 @@ void FieldSearchSpecMap::buildSearcherMap(const StringFieldIdTMapT & fieldsInQue vespalib::asciistream & operator <<(vespalib::asciistream & os, const FieldSearchSpecMap & df) { os << "DocumentTypeMap = \n"; - for (DocumentTypeIndexFieldMapT::const_iterator difIt=df.documentTypeMap().begin(), difMt=df.documentTypeMap().end(); difIt != difMt; difIt++) { - os << "DocType = " << difIt->first << "\n"; + for (const auto & dtm : df.documentTypeMap()) { + os << "DocType = " << dtm.first << "\n"; os << "IndexMap = \n"; - for (IndexFieldMapT::const_iterator ifIt=difIt->second.begin(), ifMt=difIt->second.end(); ifIt != ifMt; ifIt++) { - os << ifIt->first << ": "; - for (FieldIdTList::const_iterator it=ifIt->second.begin(), mt=ifIt->second.end(); it != mt; it++) - os << *it << ' '; + for (const auto &index : dtm.second) { + os << index.first << ": "; + for (FieldIdT fid : index.second) { + os << fid << ' '; + } os << '\n'; } } os << "SpecMap = \n"; - for (FieldSearchSpecMapT::const_iterator it=df.specMap().begin(), mt=df.specMap().end(); it != mt; it++) - os << it->first << " = " << it->second << '\n'; + for (const auto & entry : df.specMap()) { + os << entry.first << " = " << entry.second << '\n'; + } os << "NameIdMap = \n" << df.nameIdMap(); return os; } |