diff options
Diffstat (limited to 'config-model/src/main/java')
44 files changed, 620 insertions, 437 deletions
diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java index a77dd65c608..3fd2d009412 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java @@ -432,7 +432,7 @@ public class DeployState implements ConfigDefinitionStore { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); QueryProfiles queryProfiles = new QueryProfilesBuilder().build(applicationPackage, logger); SemanticRules semanticRules = new SemanticRuleBuilder().build(applicationPackage); - SearchDocumentModel searchDocumentModel = createSearchDocumentModel(rankProfileRegistry, logger, queryProfiles, validationParameters); + SearchDocumentModel searchDocumentModel = createSearchDocumentModel(rankProfileRegistry, queryProfiles, validationParameters); return new DeployState(applicationPackage, searchDocumentModel, rankProfileRegistry, @@ -458,16 +458,15 @@ public class DeployState implements ConfigDefinitionStore { } private SearchDocumentModel createSearchDocumentModel(RankProfileRegistry rankProfileRegistry, - DeployLogger logger, QueryProfiles queryProfiles, ValidationParameters validationParameters) { Collection<NamedReader> readers = applicationPackage.getSearchDefinitions(); Map<String, String> names = new LinkedHashMap<>(); - SearchBuilder builder = new SearchBuilder(applicationPackage, rankProfileRegistry, queryProfiles.getRegistry()); + SearchBuilder builder = new SearchBuilder(applicationPackage, logger, rankProfileRegistry, queryProfiles.getRegistry()); for (NamedReader reader : readers) { try { String readerName = reader.getName(); - String topLevelName = builder.importReader(reader, readerName, logger); + String topLevelName = builder.importReader(reader, readerName); String sdName = stripSuffix(readerName, ApplicationPackage.SD_NAME_SUFFIX); names.put(topLevelName, sdName); if ( ! sdName.equals(topLevelName)) { @@ -483,7 +482,7 @@ public class DeployState implements ConfigDefinitionStore { closeIgnoreException(reader.getReader()); } } - builder.build(! validationParameters.ignoreValidationErrors(), logger); + builder.build(! validationParameters.ignoreValidationErrors()); return SearchDocumentModel.fromBuilderAndNames(builder, names); } diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index e6bf3a835c6..66e728957f3 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -52,6 +52,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private double feedConcurrency = 0.5; private boolean useBucketExecutorForPruneRemoved; private boolean enableFeedBlockInDistributor = true; + private boolean useExternalRankExpression = false; private int clusterControllerMaxHeapSizeInMb = 128; private int metricsProxyMaxHeapSizeInMb = 256; private int maxActivationInhibitedOutOfSyncGroups = 0; @@ -98,7 +99,12 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public String jvmOmitStackTraceInFastThrowOption(ClusterSpec.Type type) { return jvmOmitStackTraceInFastThrowOption; } @Override public int numDistributorStripes() { return numDistributorStripes; } @Override public boolean allowDisableMtls() { return allowDisableMtls; } + @Override public boolean useExternalRankExpressions() { return useExternalRankExpression; } + public TestProperties useExternalRankExpression(boolean value) { + useExternalRankExpression = value; + return this; + } public TestProperties setFeedConcurrency(double feedConcurrency) { this.feedConcurrency = feedConcurrency; return this; diff --git a/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java b/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java index 602e0c80d4b..2784c111019 100644 --- a/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java +++ b/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java @@ -43,7 +43,7 @@ public abstract class AbstractConfigProducer<CHILD extends AbstractConfigProduce private final String subId; private String configId = null; - private List<Service> descendantServices = new ArrayList<>(); + private final List<Service> descendantServices = new ArrayList<>(); private AbstractConfigProducer parent = null; @@ -59,8 +59,8 @@ public abstract class AbstractConfigProducer<CHILD extends AbstractConfigProduce * Creates a new AbstractConfigProducer with the given parent and subId. * This constructor will add the resulting producer to the children of parent. * - * @param parent The parent of this ConfigProducer - * @param subId The fragment of the config id for the producer + * @param parent the parent of this ConfigProducer + * @param subId the fragment of the config id for the producer */ public AbstractConfigProducer(AbstractConfigProducer parent, String subId) { this(subId); @@ -69,7 +69,13 @@ public abstract class AbstractConfigProducer<CHILD extends AbstractConfigProduce } } - protected final void setParent(AbstractConfigProducer parent) { + /** Removes this from the config model */ + protected void remove() { + if (parent != null) + parent.removeChild(this); + } + + protected final void setParent(AbstractConfigProducer<?> parent) { this.parent = parent; computeConfigId(); } @@ -92,7 +98,7 @@ public abstract class AbstractConfigProducer<CHILD extends AbstractConfigProduce /** * Adds a child to this config producer. * - * @param child The child config producer to add. + * @param child the child config producer to add */ protected void addChild(CHILD child) { if (child == null) { diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java index 2594c64b951..7200c3211ba 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java @@ -6,6 +6,7 @@ import com.yahoo.config.application.api.ComponentInfo; import com.yahoo.config.application.api.UnparsedConfigDefinition; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.component.Version; +import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.InstanceName; @@ -116,6 +117,7 @@ public class MockApplicationPackage implements ApplicationPackage { public List<NamedReader> getSearchDefinitions() { ArrayList<NamedReader> readers = new ArrayList<>(); SearchBuilder searchBuilder = new SearchBuilder(this, + new BaseDeployLogger(), new RankProfileRegistry(), queryProfileRegistry); for (String sd : schemas) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java b/config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java new file mode 100644 index 00000000000..77ce2dd41b5 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java @@ -0,0 +1,72 @@ +package com.yahoo.searchdefinition; + +import com.yahoo.config.FileReference; +import com.yahoo.path.Path; +import com.yahoo.vespa.model.AbstractService; +import com.yahoo.vespa.model.utils.FileSender; + +import java.util.Collection; +import java.util.Objects; + +public class DistributableResource { + public enum PathType { FILE, URI }; + + /** The search definition-unique name of this constant */ + private final String name; + private String path = null; + private String fileReference = ""; + private PathType pathType = PathType.FILE; + + public PathType getPathType() { + return pathType; + } + + public DistributableResource(String name) { + this(name, null); + } + public DistributableResource(String name, String path) { + this.name = name; + this.path = path; + } + + public void setFileName(String fileName) { + Objects.requireNonNull(fileName, "Filename cannot be null"); + this.path = fileName; + this.pathType = PathType.FILE; + } + + public void setUri(String uri) { + Objects.requireNonNull(uri, "uri cannot be null"); + this.path = uri; + this.pathType = PathType.URI; + } + + protected void setFileReference(String fileReference) { this.fileReference = fileReference; } + /** Initiate sending of this constant to some services over file distribution */ + public void sendTo(Collection<? extends AbstractService> services) { + FileReference reference = (pathType == PathType.FILE) + ? FileSender.sendFileToServices(path, services) + : FileSender.sendUriToServices(path, services); + this.fileReference = reference.value(); + } + + public String getName() { return name; } + public String getFileName() { return path; } + public Path getFilePath() { return Path.fromString(path); } + public String getUri() { return path; } + public String getFileReference() { return fileReference; } + + public void validate() { + if (path == null || path.isEmpty()) + throw new IllegalArgumentException("Distributable resource must have a file or uri."); + } + + public String toString() { + StringBuilder b = new StringBuilder(); + b.append("resource '").append(name) + .append(pathType == PathType.FILE ? "' from file '" : " from uri ").append(path) + .append("' with ref '").append(fileReference) + .append("'"); + return b.toString(); + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentOnlySearch.java b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentOnlySearch.java index 5940f908be8..6e17bd0f0bb 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentOnlySearch.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentOnlySearch.java @@ -1,6 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.application.api.DeployLogger; import com.yahoo.searchdefinition.document.SDDocumentType; /** @@ -11,8 +13,8 @@ import com.yahoo.searchdefinition.document.SDDocumentType; */ public class DocumentOnlySearch extends Search { - public DocumentOnlySearch() { - // empty + public DocumentOnlySearch(ApplicationPackage applicationPackage, DeployLogger deployLogger) { + super(applicationPackage, deployLogger); } @Override diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/ImmutableSearch.java b/config-model/src/main/java/com/yahoo/searchdefinition/ImmutableSearch.java index 6ac73ad45a9..efd9ed5e88f 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/ImmutableSearch.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/ImmutableSearch.java @@ -28,6 +28,7 @@ public interface ImmutableSearch { Reader getRankingExpression(String fileName); ApplicationPackage applicationPackage(); RankingConstants rankingConstants(); + RankExpressionFiles rankExpressionFiles(); OnnxModels onnxModels(); Stream<ImmutableSDField> allImportedFields(); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/OnnxModel.java b/config-model/src/main/java/com/yahoo/searchdefinition/OnnxModel.java index 3e5726d6d94..3c42987512b 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/OnnxModel.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/OnnxModel.java @@ -1,14 +1,9 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition; -import com.yahoo.config.FileReference; -import com.yahoo.path.Path; import com.yahoo.tensor.TensorType; -import com.yahoo.vespa.model.AbstractService; import com.yahoo.vespa.model.ml.OnnxModelInfo; -import com.yahoo.vespa.model.utils.FileSender; -import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -19,42 +14,26 @@ import java.util.Objects; * * @author lesters */ -public class OnnxModel { +public class OnnxModel extends DistributableResource { - public enum PathType {FILE, URI}; - - private final String name; - private PathType pathType = PathType.FILE; - private String path = null; - private String fileReference = ""; private OnnxModelInfo modelInfo = null; private Map<String, String> inputMap = new HashMap<>(); private Map<String, String> outputMap = new HashMap<>(); public OnnxModel(String name) { - this.name = name; + super(name); } public OnnxModel(String name, String fileName) { - this(name); - this.path = fileName; + super(name, fileName); validate(); } - public void setFileName(String fileName) { - Objects.requireNonNull(fileName, "Filename cannot be null"); - this.path = fileName; - this.pathType = PathType.FILE; - } - + @Override public void setUri(String uri) { throw new IllegalArgumentException("URI for ONNX models are not currently supported"); } - public PathType getPathType() { - return pathType; - } - public void addInputNameMapping(String onnxName, String vespaName) { addInputNameMapping(onnxName, vespaName, true); } @@ -90,20 +69,6 @@ public class OnnxModel { this.modelInfo = modelInfo; } - /** Initiate sending of this constant to some services over file distribution */ - public void sendTo(Collection<? extends AbstractService> services) { - FileReference reference = (pathType == OnnxModel.PathType.FILE) - ? FileSender.sendFileToServices(path, services) - : FileSender.sendUriToServices(path, services); - this.fileReference = reference.value(); - } - - public String getName() { return name; } - public String getFileName() { return path; } - public Path getFilePath() { return Path.fromString(path); } - public String getUri() { return path; } - public String getFileReference() { return fileReference; } - public Map<String, String> getInputMap() { return Collections.unmodifiableMap(inputMap); } public Map<String, String> getOutputMap() { return Collections.unmodifiableMap(outputMap); } @@ -114,19 +79,4 @@ public class OnnxModel { TensorType getTensorType(String onnxName, Map<String, TensorType> inputTypes) { return modelInfo != null ? modelInfo.getTensorType(onnxName, inputTypes) : TensorType.empty; } - - public void validate() { - if (path == null || path.isEmpty()) - throw new IllegalArgumentException("ONNX models must have a file or uri."); - } - - public String toString() { - StringBuilder b = new StringBuilder(); - b.append("onnx-model '").append(name) - .append(pathType == PathType.FILE ? "' from file '" : " from uri ").append(path) - .append("' with ref '").append(fileReference) - .append("'"); - return b.toString(); - } - } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankExpressionFile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankExpressionFile.java new file mode 100644 index 00000000000..56385efeb0b --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankExpressionFile.java @@ -0,0 +1,35 @@ +package com.yahoo.searchdefinition; + +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.vespa.model.AbstractService; +import com.yahoo.vespa.model.utils.FileSender; + +import java.util.Collection; + +public class RankExpressionFile extends DistributableResource { + + public RankExpressionFile(String name, String path) { + super(name, path); + validate(); + } + + @Override + public void sendTo(Collection<? extends AbstractService> services) { + /* + * TODO This is a very dirty hack due to using both SEARCH_DEFINITIONS_DIR and SCHEMA_DIR + * and doing so inconsistently, combined with using both fields from application package on disk and in zookeeper. + * The mess is spread out nicely, but ZookeeperClient, and writeSearchDefinitions and ZkApplicationPackage and FilesApplicationPackage + * should be consolidated + */ + try { + setFileReference(FileSender.sendFileToServices(ApplicationPackage.SCHEMAS_DIR + "/" + getFileName(), services).value()); + } catch (IllegalArgumentException e1) { + try { + setFileReference(FileSender.sendFileToServices(ApplicationPackage.SEARCH_DEFINITIONS_DIR + "/" + getFileName(), services).value()); + } catch (IllegalArgumentException e2) { + throw new IllegalArgumentException("Failed to find expression file '" + getFileName() + "' in '" + + ApplicationPackage.SEARCH_DEFINITIONS_DIR + "' or '" + ApplicationPackage.SCHEMAS_DIR + "'.", e2); + } + } + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankExpressionFiles.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankExpressionFiles.java new file mode 100644 index 00000000000..ebc91e0693f --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankExpressionFiles.java @@ -0,0 +1,35 @@ +package com.yahoo.searchdefinition; + +import com.yahoo.vespa.model.AbstractService; + +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +public class RankExpressionFiles { + private final Map<String, RankExpressionFile> expressions = new HashMap<>(); + + public void add(RankExpressionFile expression) { + expression.validate(); + String name = expression.getName(); + if (expressions.containsKey(name)) + throw new IllegalArgumentException("Rank expression file '" + name + "' defined twice"); + expressions.put(name, expression); + } + + /** Returns the ranking constant with the given name, or null if not present */ + public RankExpressionFile get(String name) { + return expressions.get(name); + } + + /** Returns a read-only map of the ranking constants in this indexed by name */ + public Map<String, RankExpressionFile> asMap() { + return Collections.unmodifiableMap(expressions); + } + + /** Initiate sending of these constants to some services over file distribution */ + public void sendTo(Collection<? extends AbstractService> services) { + expressions.values().forEach(constant -> constant.sendTo(services)); + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java index 95b291cf744..b0a7d2aaca2 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java @@ -52,6 +52,8 @@ import java.util.stream.Stream; */ public class RankProfile implements Cloneable { + public final static String FIRST_PHASE = "firstphase"; + public final static String SECOND_PHASE = "secondphase"; /** The search definition-unique name of this rank profile */ private final String name; @@ -76,6 +78,8 @@ public class RankProfile implements Cloneable { /** The ranking expression to be used for second phase */ private RankingExpression secondPhaseRanking = null; + private Set<String> externalFileExpressions = new HashSet<>(); + /** Number of hits to be reranked in second phase, -1 means use default */ private int rerankCount = -1; @@ -165,6 +169,10 @@ public class RankProfile implements Cloneable { return search != null ? search.rankingConstants() : model.rankingConstants(); } + public RankExpressionFiles rankExpressionFiles() { + return search != null ? search.rankExpressionFiles() : model.rankExpressionFiles(); + } + public Map<String, OnnxModel> onnxModels() { return search != null ? search.onnxModels().asMap() : onnxModels.asMap(); } @@ -316,7 +324,7 @@ public class RankProfile implements Cloneable { public void addConstant(String name, Value value) { if (value instanceof TensorValue) { - TensorType type = ((TensorValue)value).type(); + TensorType type = value.type(); if (type.dimensions().stream().anyMatch(d -> d.isIndexed() && d.size().isEmpty())) throw new IllegalArgumentException("Illegal type of constant " + name + " type " + type + ": Dense tensor dimensions must have a size"); @@ -370,9 +378,44 @@ public class RankProfile implements Cloneable { this.firstPhaseRanking = rankingExpression; } + public String getUniqueExpressionName(String name) { + return getName() + "_" + name; + } + public String getFirstPhaseFile() { + String name = FIRST_PHASE; + if (externalFileExpressions.contains(name)) { + return rankExpressionFiles().get(getUniqueExpressionName(name)).getFileName(); + } + if ((firstPhaseRanking == null) && (getInherited() != null)) { + return getInherited().getFirstPhaseFile(); + } + return null; + } + + public String getSecondPhaseFile() { + String name = SECOND_PHASE; + if (externalFileExpressions.contains(name)) { + return rankExpressionFiles().get(getUniqueExpressionName(name)).getFileName(); + } + if ((secondPhaseRanking == null) && (getInherited() != null)) { + return getInherited().getSecondPhaseFile(); + } + return null; + } + + public String getExpressionFile(String name) { + if (externalFileExpressions.contains(name)) { + return rankExpressionFiles().get(getUniqueExpressionName(name)).getFileName(); + } + if (getInherited() != null) { + return getInherited().getExpressionFile(name); + } + return null; + } + public void setFirstPhaseRanking(String expression) { try { - this.firstPhaseRanking = parseRankingExpression("firstphase", expression); + this.firstPhaseRanking = parseRankingExpression(FIRST_PHASE, expression); } catch (ParseException e) { throw new IllegalArgumentException("Illegal first phase ranking function", e); @@ -389,13 +432,9 @@ public class RankProfile implements Cloneable { return null; } - public void setSecondPhaseRanking(RankingExpression rankingExpression) { - this.secondPhaseRanking = rankingExpression; - } - public void setSecondPhaseRanking(String expression) { try { - this.secondPhaseRanking = parseRankingExpression("secondphase", expression); + this.secondPhaseRanking = parseRankingExpression(SECOND_PHASE, expression); } catch (ParseException e) { throw new IllegalArgumentException("Illegal second phase ranking function", e); @@ -436,7 +475,6 @@ public class RankProfile implements Cloneable { * the final (with inheritance included) summary features of the given parent. * The profile must be the profile which is directly inherited by this. * - * @param parentProfile */ public void setInheritedSummaryFeatures(String parentProfile) { if ( ! parentProfile.equals(inheritedName)) @@ -496,12 +534,7 @@ public class RankProfile implements Cloneable { private void addRankProperty(RankProperty rankProperty) { // Just the usual multimap semantics here - List<RankProperty> properties = rankProperties.get(rankProperty.getName()); - if (properties == null) { - properties = new ArrayList<>(1); - rankProperties.put(rankProperty.getName(), properties); - } - properties.add(rankProperty); + rankProperties.computeIfAbsent(rankProperty.getName(), (String key) -> new ArrayList<>(1)).add(rankProperty); } @Override @@ -675,18 +708,27 @@ public class RankProfile implements Cloneable { } } - private Reader openRankingExpressionReader(String expName, String expression) { - if ( ! expression.startsWith("file:")) return new StringReader(expression); - + private static String extractFileName(String expression) { String fileName = expression.substring("file:".length()).trim(); if ( ! fileName.endsWith(ApplicationPackage.RANKEXPRESSION_NAME_SUFFIX)) fileName = fileName + ApplicationPackage.RANKEXPRESSION_NAME_SUFFIX; + return fileName; + } + + private Reader openRankingExpressionReader(String expName, String expression) { + if ( ! expression.startsWith("file:")) return new StringReader(expression); + + String fileName = extractFileName(expression); File file = new File(fileName); - if ( ! (file.isAbsolute()) && file.getPath().contains("/")) // See ticket 4102122 + if ( ! file.isAbsolute() && file.getPath().contains("/")) // See ticket 4102122 throw new IllegalArgumentException("In " + getName() +", " + expName + ", ranking references file '" + file + "' in subdirectory, which is not supported."); + /* TODO balder: Disabled until end-2-end verified + rankExpressionFiles().add(new RankExpressionFile(getUniqueExpressionName(expName), fileName)); + externalFileExpressions.add(expName); + */ return search.getRankingExpression(fileName); } @@ -918,12 +960,12 @@ public class RankProfile implements Cloneable { */ public static class RankSetting implements Serializable { - private String fieldName; + private final String fieldName; - private Type type; + private final Type type; /** The rank value */ - private Object value; + private final Object value; public enum Type { @@ -932,10 +974,10 @@ public class RankProfile implements Cloneable { WEIGHT("weight"), PREFERBITVECTOR("preferbitvector",true); - private String name; + private final String name; /** True if this setting really pertains to an index, not a field within an index */ - private boolean isIndexLevel; + private final boolean isIndexLevel; Type(String name) { this(name,false); @@ -1008,8 +1050,8 @@ public class RankProfile implements Cloneable { /** A rank property. Rank properties are Value Objects */ public static class RankProperty implements Serializable { - private String name; - private String value; + private final String name; + private final String value; public RankProperty(String name, String value) { this.name = name; @@ -1084,7 +1126,6 @@ public class RankProfile implements Cloneable { public void setMinGroups(int value) { minGroups = value; } public void setCutoffFactor(double value) { cutoffFactor = value; } public void setCutoffStrategy(Diversity.CutoffStrategy strategy) { cutoffStrategy = strategy; } - public void setCutoffStrategy(String strategy) { cutoffStrategy = Diversity.CutoffStrategy.valueOf(strategy); } public String getAttribute() { return attribute; } public int getMinGroups() { return minGroups; } public double getCutoffFactor() { return cutoffFactor; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstant.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstant.java index b41cf582204..8e376f13615 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstant.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstant.java @@ -1,13 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition; -import com.yahoo.config.FileReference; import com.yahoo.tensor.TensorType; -import com.yahoo.vespa.model.AbstractService; -import com.yahoo.vespa.model.utils.FileSender; - -import java.util.Collection; -import java.util.Objects; /** * A global ranking constant distributed using file distribution. @@ -17,81 +11,38 @@ import java.util.Objects; * @author arnej * @author bratseth */ -public class RankingConstant { - - public enum PathType {FILE, URI}; - - /** The search definition-unique name of this constant */ - private final String name; +public class RankingConstant extends DistributableResource { private TensorType tensorType = null; - private String path = null; - private String fileReference = ""; - - public PathType getPathType() { - return pathType; - } - - private PathType pathType = PathType.FILE; public RankingConstant(String name) { - this.name = name; + super(name); } public RankingConstant(String name, TensorType type, String fileName) { - this(name); + super(name, fileName); this.tensorType = type; - this.path = fileName; validate(); } - public void setFileName(String fileName) { - Objects.requireNonNull(fileName, "Filename cannot be null"); - this.path = fileName; - this.pathType = PathType.FILE; - } - - public void setUri(String uri) { - Objects.requireNonNull(uri, "uri cannot be null"); - this.path = uri; - this.pathType = PathType.URI; - } - public void setType(TensorType type) { this.tensorType = type; } - /** Initiate sending of this constant to some services over file distribution */ - public void sendTo(Collection<? extends AbstractService> services) { - FileReference reference = (pathType == RankingConstant.PathType.FILE) - ? FileSender.sendFileToServices(path, services) - : FileSender.sendUriToServices(path, services); - this.fileReference = reference.value(); - } - - public String getName() { return name; } - public String getFileName() { return path; } - public String getUri() { return path; } - public String getFileReference() { return fileReference; } public TensorType getTensorType() { return tensorType; } public String getType() { return tensorType.toString(); } public void validate() { - if (path == null || path.isEmpty()) - throw new IllegalArgumentException("Ranking constants must have a file or uri."); + super.validate(); if (tensorType == null) - throw new IllegalArgumentException("Ranking constant '" + name + "' must have a type."); + throw new IllegalArgumentException("Ranking constant '" + getName() + "' must have a type."); if (tensorType.dimensions().stream().anyMatch(d -> d.isIndexed() && d.size().isEmpty())) - throw new IllegalArgumentException("Illegal type in field " + name + " type " + tensorType + + throw new IllegalArgumentException("Illegal type in field " + getName() + " type " + tensorType + ": Dense tensor dimensions must have a size"); } public String toString() { - StringBuilder b = new StringBuilder(); - b.append("constant '").append(name) - .append(pathType == PathType.FILE ? "' from file '" : " from uri ").append(path) - .append("' with ref '").append(fileReference) - .append("' of type '").append(tensorType) - .append("'"); + StringBuilder b = new StringBuilder(super.toString()) + .append("' of type '").append(tensorType).append("'"); return b.toString(); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstants.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstants.java index adefa5566ab..82381aa63fc 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstants.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstants.java @@ -40,5 +40,4 @@ public class RankingConstants { public void sendTo(Collection<? extends AbstractService> services) { constants.values().forEach(constant -> constant.sendTo(services)); } - } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/Search.java b/config-model/src/main/java/com/yahoo/searchdefinition/Search.java index 9b7434dccab..b07740620f8 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/Search.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/Search.java @@ -2,6 +2,8 @@ package com.yahoo.searchdefinition; import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.document.Field; import com.yahoo.searchdefinition.derived.SummaryClass; import com.yahoo.searchdefinition.document.Attribute; @@ -15,6 +17,7 @@ import com.yahoo.searchdefinition.document.TemporaryImportedFields; import com.yahoo.searchdefinition.document.annotation.SDAnnotationType; import com.yahoo.vespa.documentmodel.DocumentSummary; import com.yahoo.vespa.documentmodel.SummaryField; +import com.yahoo.vespa.model.AbstractService; import java.io.Reader; import java.util.ArrayList; @@ -27,6 +30,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.TreeMap; +import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Stream; @@ -42,7 +46,6 @@ import java.util.stream.Stream; // Ensure that after the processing step, all implicit instances of the above types are explicitly represented public class Search implements ImmutableSearch { - private static final Logger log = Logger.getLogger(Search.class.getName()); private static final String SD_DOC_FIELD_NAME = "sddocname"; private static final List<String> RESERVED_NAMES = Arrays.asList( "index", "index_url", "summary", "attribute", "select_input", "host", SummaryClass.DOCUMENT_ID_FIELD, @@ -59,7 +62,7 @@ public class Search implements ImmutableSearch { private String name; /** True if this doesn't define a search, just a document type */ - private boolean documentsOnly = false; + private final boolean documentsOnly; private boolean rawAsBase64 = false; @@ -78,6 +81,9 @@ public class Search implements ImmutableSearch { /** The explicitly defined summaries of this search definition. _Must_ preserve order. */ private final Map<String, DocumentSummary> summaries = new LinkedHashMap<>(); + /** External rank expression files of this */ + private final RankExpressionFiles rankExpressionFiles = new RankExpressionFiles(); + /** Ranking constants of this */ private final RankingConstants rankingConstants = new RankingConstants(); @@ -88,26 +94,33 @@ public class Search implements ImmutableSearch { private Optional<ImportedFields> importedFields = Optional.empty(); private final ApplicationPackage applicationPackage; + private final DeployLogger deployLogger; - /** - * Creates a search definition which just holds a set of documents which should not (here, directly) be searchable - */ - protected Search() { - applicationPackage = null; - documentsOnly = true; + /** Testin only */ + public Search(String name) { + this(name, null, new BaseDeployLogger()); } - /** * Creates a proper search definition * * @param name of the the searchdefinition * @param applicationPackage the application containing this */ - public Search(String name, ApplicationPackage applicationPackage) { - this.applicationPackage = applicationPackage; + public Search(String name, ApplicationPackage applicationPackage, DeployLogger deployLogger) { + this(applicationPackage, deployLogger, false); this.name = name; } + protected Search(ApplicationPackage applicationPackage, DeployLogger deployLogger) { + this(applicationPackage, deployLogger, true); + } + + private Search(ApplicationPackage applicationPackage, DeployLogger deployLogger, boolean documentsOnly) { + this.applicationPackage = applicationPackage; + this.deployLogger = deployLogger; + this.documentsOnly = documentsOnly; + } + protected void setName(String name) { this.name = name; } @@ -172,11 +185,20 @@ public class Search implements ImmutableSearch { } @Override + public RankExpressionFiles rankExpressionFiles() { return rankExpressionFiles; } + + @Override public RankingConstants rankingConstants() { return rankingConstants; } @Override public OnnxModels onnxModels() { return onnxModels; } + public void sendTo(Collection<? extends AbstractService> services) { + rankingConstants.sendTo(services); + rankExpressionFiles.sendTo(services); + onnxModels.sendTo(services); + } + public Optional<TemporaryImportedFields> temporaryImportedFields() { return temporaryImportedFields; } @@ -310,7 +332,7 @@ public class Search implements ImmutableSearch { */ public void addExtraField(SDField field) { if (fields.containsKey(field.getName())) { - log.warning("Duplicate field " + field.getName() + " in search definition " + getName()); + deployLogger.logApplicationPackage(Level.WARNING, "Duplicate field " + field.getName() + " in search definition " + getName()); } else { field.setIsExtraField(true); fields.put(field.getName(), field); @@ -422,7 +444,7 @@ public class Search implements ImmutableSearch { if (current.getRankType() != null && !consolidated.getRankType().equals(current.getRankType())) { - log.warning("Conflicting rank type settings for " + + deployLogger.logApplicationPackage(Level.WARNING, "Conflicting rank type settings for " + first.getName() + " in " + this + ", using " + consolidated.getRankType()); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java index 852fdc17726..fc110504f4d 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java @@ -41,53 +41,72 @@ import java.util.List; public class SearchBuilder { private final DocumentTypeManager docTypeMgr = new DocumentTypeManager(); - private List<Search> searchList = new LinkedList<>(); - private ApplicationPackage app; - private boolean isBuilt = false; - private DocumentModel model = new DocumentModel(); + private final DocumentModel model = new DocumentModel(); + private final ApplicationPackage app; private final RankProfileRegistry rankProfileRegistry; private final QueryProfileRegistry queryProfileRegistry; + private final DeployLogger deployLogger; + private List<Search> searchList = new LinkedList<>(); + private boolean isBuilt = false; /** True to build the document aspect only, skipping instantiation of rank profiles */ private final boolean documentsOnly; /** For testing only */ public SearchBuilder() { - this(MockApplicationPackage.createEmpty(), new RankProfileRegistry(), new QueryProfileRegistry()); + this(new RankProfileRegistry(), new QueryProfileRegistry()); + } + + /** For testing only */ + public SearchBuilder(DeployLogger deployLogger) { + this(MockApplicationPackage.createEmpty(), deployLogger); + } + + /** For testing only */ + public SearchBuilder(DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry) { + this(MockApplicationPackage.createEmpty(), deployLogger, rankProfileRegistry); } /** Used for generating documents for typed access to document fields in Java */ public SearchBuilder(boolean documentsOnly) { - this(MockApplicationPackage.createEmpty(), new RankProfileRegistry(), new QueryProfileRegistry(), documentsOnly); + this(MockApplicationPackage.createEmpty(), new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfileRegistry(), documentsOnly); } /** For testing only */ - public SearchBuilder(ApplicationPackage app) { - this(app, new RankProfileRegistry(), new QueryProfileRegistry()); + public SearchBuilder(ApplicationPackage app, DeployLogger deployLogger) { + this(app, deployLogger, new RankProfileRegistry(), new QueryProfileRegistry()); + } + + /** For testing only */ + public SearchBuilder(ApplicationPackage app, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry) { + this(app, deployLogger, rankProfileRegistry, new QueryProfileRegistry()); } /** For testing only */ public SearchBuilder(RankProfileRegistry rankProfileRegistry) { - this(MockApplicationPackage.createEmpty(), rankProfileRegistry, new QueryProfileRegistry()); + this(rankProfileRegistry, new QueryProfileRegistry()); } /** For testing only */ public SearchBuilder(RankProfileRegistry rankProfileRegistry, QueryProfileRegistry queryProfileRegistry) { - this(MockApplicationPackage.createEmpty(), rankProfileRegistry, queryProfileRegistry); + this(MockApplicationPackage.createEmpty(), new BaseDeployLogger(), rankProfileRegistry, queryProfileRegistry); } public SearchBuilder(ApplicationPackage app, + DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfileRegistry queryProfileRegistry) { - this(app, rankProfileRegistry, queryProfileRegistry, false); + this(app, deployLogger, rankProfileRegistry, queryProfileRegistry, false); } - public SearchBuilder(ApplicationPackage app, - RankProfileRegistry rankProfileRegistry, - QueryProfileRegistry queryProfileRegistry, - boolean documentsOnly) { + private SearchBuilder(ApplicationPackage app, + DeployLogger deployLogger, + RankProfileRegistry rankProfileRegistry, + QueryProfileRegistry queryProfileRegistry, + boolean documentsOnly) { this.app = app; this.rankProfileRegistry = rankProfileRegistry; this.queryProfileRegistry = queryProfileRegistry; + this.deployLogger = deployLogger; this.documentsOnly = documentsOnly; } @@ -95,29 +114,17 @@ public class SearchBuilder { * Import search definition. * * @param fileName The name of the file to import. - * @param deployLogger Logger for deploy messages. * @return The name of the imported object. * @throws IOException Thrown if the file can not be read for some reason. * @throws ParseException Thrown if the file does not contain a valid search definition. ``` */ - public String importFile(String fileName, DeployLogger deployLogger) throws IOException, ParseException { + public String importFile(String fileName) throws IOException, ParseException { File file = new File(fileName); - return importString(IOUtils.readFile(file), file.getAbsoluteFile().getParent(), deployLogger); + return importString(IOUtils.readFile(file), file.getAbsoluteFile().getParent()); } - /** - * Import search definition. - * - * @param fileName The name of the file to import. - * @return The name of the imported object. - * @throws IOException Thrown if the file can not be read for some reason. - * @throws ParseException Thrown if the file does not contain a valid search definition. - */ - public String importFile(String fileName) throws IOException, ParseException { - return importFile(fileName, new BaseDeployLogger()); - } - public String importFile(Path file) throws IOException, ParseException { - return importFile(file.toString(), new BaseDeployLogger()); + private String importFile(Path file) throws IOException, ParseException { + return importFile(file.toString()); } /** @@ -129,17 +136,8 @@ public class SearchBuilder { * @return The name of the imported object. * @throws ParseException Thrown if the file does not contain a valid search definition. */ - public String importReader(NamedReader reader, String searchDefDir, DeployLogger deployLogger) throws IOException, ParseException { - return importString(IOUtils.readAll(reader), searchDefDir, deployLogger); - } - - /** - * See #{@link #importReader} - * - * Convenience, should only be used for testing as logs will be swallowed. - */ public String importReader(NamedReader reader, String searchDefDir) throws IOException, ParseException { - return importString(IOUtils.readAll(reader), searchDefDir, new BaseDeployLogger()); + return importString(IOUtils.readAll(reader), searchDefDir); } /** @@ -150,21 +148,10 @@ public class SearchBuilder { * @throws ParseException thrown if the file does not contain a valid search definition. */ public String importString(String str) throws ParseException { - return importString(str, null, new BaseDeployLogger()); - } - - /** - * Import search definition. - * - * @param str the string to parse. - * @return the name of the imported object. - * @throws ParseException thrown if the file does not contain a valid search definition. - */ - public String importString(String str, DeployLogger logger) throws ParseException { - return importString(str, null, logger); + return importString(str, null); } - private String importString(String str, String searchDefDir, DeployLogger deployLogger) throws ParseException { + private String importString(String str, String searchDefDir) throws ParseException { SimpleCharStream stream = new SimpleCharStream(str); try { return importRawSearch(new SDParser(stream, deployLogger, app, rankProfileRegistry, documentsOnly) @@ -208,7 +195,7 @@ public class SearchBuilder { * @throws IllegalStateException Thrown if this method has already been called. */ public void build() { - build(true, new BaseDeployLogger()); + build(true); } /** @@ -216,9 +203,8 @@ public class SearchBuilder { * #getSearch(String)} method. * * @throws IllegalStateException Thrown if this method has already been called. - * @param deployLogger The logger to use during build */ - public void build(boolean validate, DeployLogger deployLogger) { + public void build(boolean validate) { if (isBuilt) throw new IllegalStateException("Model already built"); List<Search> built = new ArrayList<>(); @@ -325,9 +311,9 @@ public class SearchBuilder { } public static SearchBuilder createFromString(String sd, DeployLogger logger) throws ParseException { - SearchBuilder builder = new SearchBuilder(MockApplicationPackage.createEmpty()); - builder.importString(sd, logger); - builder.build(true, logger); + SearchBuilder builder = new SearchBuilder(logger); + builder.importString(sd); + builder.build(true); return builder; } @@ -386,12 +372,13 @@ public class SearchBuilder { QueryProfileRegistry queryprofileRegistry) throws IOException, ParseException { SearchBuilder builder = new SearchBuilder(MockApplicationPackage.createEmpty(), + deployLogger, rankProfileRegistry, queryprofileRegistry); for (String fileName : fileNames) { builder.importFile(fileName); } - builder.build(true, deployLogger); + builder.build(true); return builder; } @@ -413,15 +400,16 @@ public class SearchBuilder { public static SearchBuilder createFromDirectory(String dir, RankProfileRegistry rankProfileRegistry, QueryProfileRegistry queryProfileRegistry, - DeployLogger logger, + DeployLogger deployLogger, ApplicationPackage applicationPackage) throws IOException, ParseException { SearchBuilder builder = new SearchBuilder(applicationPackage, + deployLogger, rankProfileRegistry, queryProfileRegistry); for (Iterator<Path> i = Files.list(new File(dir).toPath()).filter(p -> p.getFileName().toString().endsWith(".sd")).iterator(); i.hasNext(); ) { builder.importFile(i.next()); } - builder.build(true, logger); + builder.build(true); return builder; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java index 4d6ce783947..52d99c523ea 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java @@ -85,7 +85,8 @@ public class DerivedConfiguration { summaries = new Summaries(search, deployLogger); summaryMap = new SummaryMap(search); juniperrc = new Juniperrc(search); - rankProfileList = new RankProfileList(search, search.rankingConstants(), attributeFields, rankProfileRegistry, queryProfiles, importedModels, deployProperties); + rankProfileList = new RankProfileList(search, search.rankingConstants(), search.rankExpressionFiles(), attributeFields, + rankProfileRegistry, queryProfiles, importedModels, deployProperties); indexingScript = new IndexingScript(search); indexInfo = new IndexInfo(search); indexSchema = new IndexSchema(search); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java index 42fa1df802b..d414b9ed79f 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java @@ -6,6 +6,8 @@ import com.yahoo.config.model.api.ModelContext; import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.OnnxModel; import com.yahoo.searchdefinition.OnnxModels; +import com.yahoo.searchdefinition.RankExpressionFile; +import com.yahoo.searchdefinition.RankExpressionFiles; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.RankingConstant; import com.yahoo.searchdefinition.RankingConstants; @@ -14,6 +16,7 @@ import com.yahoo.searchdefinition.RankProfile; import com.yahoo.searchdefinition.Search; import com.yahoo.vespa.config.search.core.OnnxModelsConfig; import com.yahoo.vespa.config.search.core.RankingConstantsConfig; +import com.yahoo.vespa.config.search.core.RankingExpressionsConfig; import com.yahoo.vespa.model.AbstractService; import java.util.Collection; @@ -25,21 +28,21 @@ import java.util.logging.Logger; * * @author bratseth */ -public class RankProfileList extends Derived implements RankProfilesConfig.Producer, - RankingConstantsConfig.Producer, - OnnxModelsConfig.Producer { +public class RankProfileList extends Derived implements RankProfilesConfig.Producer { private static final Logger log = Logger.getLogger(RankProfileList.class.getName()); private final Map<String, RawRankProfile> rankProfiles = new java.util.LinkedHashMap<>(); private final RankingConstants rankingConstants; + private final RankExpressionFiles rankExpressionFiles; private final OnnxModels onnxModels; public static RankProfileList empty = new RankProfileList(); private RankProfileList() { - this.rankingConstants = new RankingConstants(); - this.onnxModels = new OnnxModels(); + rankingConstants = new RankingConstants(); + rankExpressionFiles = new RankExpressionFiles(); + onnxModels = new OnnxModels(); } /** @@ -50,6 +53,7 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ */ public RankProfileList(Search search, RankingConstants rankingConstants, + RankExpressionFiles rankExpressionFiles, AttributeFields attributeFields, RankProfileRegistry rankProfileRegistry, QueryProfileRegistry queryProfiles, @@ -57,7 +61,8 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ ModelContext.Properties deployProperties) { setName(search == null ? "default" : search.getName()); this.rankingConstants = rankingConstants; - this.onnxModels = search == null ? new OnnxModels() : search.onnxModels(); // as ONNX models come from parsing rank expressions + this.rankExpressionFiles = rankExpressionFiles; + onnxModels = search == null ? new OnnxModels() : search.onnxModels(); // as ONNX models come from parsing rank expressions deriveRankProfiles(rankProfileRegistry, queryProfiles, importedModels, search, attributeFields, deployProperties); } @@ -93,11 +98,9 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ return rankProfiles.get(name); } - public void sendConstantsTo(Collection<? extends AbstractService> services) { + public void sendTo(Collection<? extends AbstractService> services) { rankingConstants.sendTo(services); - } - - public void sendOnnxModelsTo(Collection<? extends AbstractService> services) { + rankExpressionFiles.sendTo(services); onnxModels.sendTo(services); } @@ -111,7 +114,10 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ } } - @Override + public void getConfig(RankingExpressionsConfig.Builder builder) { + rankExpressionFiles.asMap().values().forEach((expr) -> builder.expression.add(new RankingExpressionsConfig.Expression.Builder().name(expr.getName()).fileref(expr.getFileReference()))); + } + public void getConfig(RankingConstantsConfig.Builder builder) { for (RankingConstant constant : rankingConstants.asMap().values()) { if ("".equals(constant.getFileReference())) @@ -124,7 +130,6 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ } } - @Override public void getConfig(OnnxModelsConfig.Builder builder) { for (OnnxModel model : onnxModels.asMap().values()) { if ("".equals(model.getFileReference())) diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java index c656a426b61..41ac1e17d93 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java @@ -125,8 +125,8 @@ public class RawRankProfile implements RankProfilesConfig.Producer { */ private Map<String, FieldRankSettings> fieldRankSettings = new java.util.LinkedHashMap<>(); + private final RankProfile rankProfile; private RankingExpression firstPhaseRanking = null; - private RankingExpression secondPhaseRanking = null; private Set<ReferenceNode> summaryFeatures = new LinkedHashSet<>(); @@ -159,6 +159,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { private final Map<String, String> attributeTypes; private final Map<String, String> queryFeatureTypes; + private final boolean useExternalExpressionFiles; private Set<String> filterFields = new java.util.LinkedHashSet<>(); @@ -166,11 +167,13 @@ public class RawRankProfile implements RankProfilesConfig.Producer { * Creates a raw rank profile from the given rank profile */ Deriver(RankProfile rankProfile, QueryProfileRegistry queryProfiles, ImportedMlModels importedModels, - AttributeFields attributeFields, ModelContext.Properties deployProperties) + AttributeFields attributeFields, ModelContext.Properties deployProperties) { + this.rankProfile = rankProfile; RankProfile compiled = rankProfile.compile(queryProfiles, importedModels); attributeTypes = compiled.getAttributeTypes(); queryFeatureTypes = compiled.getQueryFeatureTypes(); + useExternalExpressionFiles = deployProperties.featureFlags().useExternalRankExpressions(); deriveRankingFeatures(compiled, deployProperties); deriveRankTypeSetting(compiled, attributeFields); deriveFilterFields(compiled); @@ -230,13 +233,13 @@ public class RawRankProfile implements RankProfilesConfig.Producer { Map<String, String> functionProperties) { SerializationContext context = new SerializationContext(functionExpressions, null, functionProperties); for (Map.Entry<String, RankProfile.RankingExpressionFunction> e : functions.entrySet()) { + if (useExternalExpressionFiles && rankProfile.getExpressionFile(e.getKey()) != null) continue; String propertyName = RankingExpression.propertyName(e.getKey()); - if (context.serializedFunctions().containsKey(propertyName)) { - continue; - } + if (context.serializedFunctions().containsKey(propertyName)) continue; + String expressionString = e.getValue().function().getBody().getRoot().toString(new StringBuilder(), context, null, null).toString(); - context.addFunctionSerialization(RankingExpression.propertyName(e.getKey()), expressionString); + context.addFunctionSerialization(propertyName, expressionString); for (Map.Entry<String, TensorType> argumentType : e.getValue().function().argumentTypes().entrySet()) context.addArgumentTypeSerialization(e.getKey(), argumentType.getKey(), argumentType.getValue()); if (e.getValue().function().returnType().isPresent()) @@ -333,7 +336,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { public List<Pair<String, String>> derive() { List<Pair<String, String>> properties = new ArrayList<>(); for (RankProfile.RankProperty property : rankProperties) { - if ("rankingExpression(firstphase).rankingScript".equals(property.getName())) { + if (("rankingExpression(" + RankProfile.FIRST_PHASE + ").rankingScript").equals(property.getName())) { // Could have been set by function expansion. Set expressions, then skip this property. try { firstPhaseRanking = new RankingExpression(property.getValue()); @@ -341,7 +344,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { throw new IllegalArgumentException("Could not parse first phase expression", e); } } - else if ("rankingExpression(secondphase).rankingScript".equals(property.getName())) { + else if (("rankingExpression(" + RankProfile.SECOND_PHASE + ").rankingScript").equals(property.getName())) { try { secondPhaseRanking = new RankingExpression(property.getValue()); } catch (ParseException e) { @@ -352,8 +355,8 @@ public class RawRankProfile implements RankProfilesConfig.Producer { properties.add(new Pair<>(property.getName(), property.getValue())); } } - properties.addAll(deriveRankingPhaseRankProperties(firstPhaseRanking, "firstphase")); - properties.addAll(deriveRankingPhaseRankProperties(secondPhaseRanking, "secondphase")); + properties.addAll(deriveRankingPhaseRankProperties(firstPhaseRanking, rankProfile.getFirstPhaseFile(), RankProfile.FIRST_PHASE)); + properties.addAll(deriveRankingPhaseRankProperties(secondPhaseRanking, rankProfile.getSecondPhaseFile(), RankProfile.SECOND_PHASE)); for (FieldRankSettings settings : fieldRankSettings.values()) { properties.addAll(settings.deriveRankProperties()); } @@ -420,7 +423,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { return properties; } - private List<Pair<String, String>> deriveRankingPhaseRankProperties(RankingExpression expression, String phase) { + private List<Pair<String, String>> deriveRankingPhaseRankProperties(RankingExpression expression, String fileName, String phase) { List<Pair<String, String>> properties = new ArrayList<>(); if (expression == null) return properties; @@ -428,7 +431,9 @@ public class RawRankProfile implements RankProfilesConfig.Producer { if ("".equals(name)) name = phase; - if (expression.getRoot() instanceof ReferenceNode) { + if (useExternalExpressionFiles && (fileName != null)) { + properties.add(new Pair<>("vespa.rank." + phase, "rankingExpression(" + rankProfile.getUniqueExpressionName(name) + ")")); + } else if (expression.getRoot() instanceof ReferenceNode) { properties.add(new Pair<>("vespa.rank." + phase, expression.getRoot().toString())); } else { properties.add(new Pair<>("vespa.rank." + phase, "rankingExpression(" + name + ")")); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java b/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java index 6e16b1a120e..8dea1b65079 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java @@ -92,21 +92,30 @@ public abstract class AbstractService extends AbstractConfigProducer<AbstractCon * Preferred constructor when building from XML. Use this if you are building * in doBuild() in an AbstractConfigProducerBuilder. * build() will call initService() in that case, after setting hostalias and baseport. - * @param parent Parent config producer in the model tree. - * @param name Name of this service. + * + * @param parent the parent config producer in the model tree + * @param name the name of this service */ - public AbstractService(AbstractConfigProducer parent, String name) { + public AbstractService(AbstractConfigProducer<?> parent, String name) { super(parent, name); } /** * Only used for testing. Stay away. - * @param name Name of this service. + * + * @param name the name of this service. */ public AbstractService(String name) { super(name); } + @Override + public void remove() { + super.remove(); + if (hostResource != null) + hostResource.deallocateService(this); + } + /** * Distribute affinity on a collection of services. Services that are located on the same host * will be assigned a specific cpu socket on that host. diff --git a/config-model/src/main/java/com/yahoo/vespa/model/HostPorts.java b/config-model/src/main/java/com/yahoo/vespa/model/HostPorts.java index a80982fe75b..b6a9d2bf97a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/HostPorts.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/HostPorts.java @@ -6,9 +6,12 @@ import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.provision.NetworkPorts; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; +import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; +import java.util.ListIterator; import java.util.Map; import java.util.Optional; import java.util.logging.Level; @@ -36,6 +39,7 @@ public class HostPorts { private PortFinder portFinder = new PortFinder(Collections.emptyList()); + private boolean flushed = false; private Optional<NetworkPorts> networkPortsList = Optional.empty(); public HostPorts(String hostname) { @@ -143,8 +147,16 @@ public class HostPorts { return allocator.result(); } + void deallocatePorts(NetworkPortRequestor service) { + if (flushed) + throw new IllegalStateException("Cannot deallocate ports after calling flushPortReservations()"); + portDB.entrySet().removeIf(entry -> entry.getValue().getServiceName().equals(service.getServiceName())); + allocatedPorts--; + } + public void flushPortReservations() { this.networkPortsList = Optional.of(new NetworkPorts(portFinder.allocations())); + this.flushed = true; } /** diff --git a/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java b/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java index 3bc07db9507..ef041d06978 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java @@ -79,6 +79,11 @@ public class HostResource implements Comparable<HostResource> { return ports; } + void deallocateService(AbstractService service) { + hostPorts.deallocatePorts(service); + services.remove(service.getServiceName()); + } + /** * Returns the service with the given "sentinel name" on this Host, * or null if the name does not match any service. diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java index 4dc38c09ab1..ab00e9d295f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java @@ -33,6 +33,7 @@ import com.yahoo.container.QrConfig; import com.yahoo.path.Path; import com.yahoo.searchdefinition.OnnxModel; import com.yahoo.searchdefinition.OnnxModels; +import com.yahoo.searchdefinition.RankExpressionFiles; import com.yahoo.searchdefinition.RankProfile; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.RankingConstants; @@ -129,6 +130,9 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri /** The global ranking constants of this model */ private final RankingConstants rankingConstants = new RankingConstants(); + /** External rank expression files of this */ + private final RankExpressionFiles rankExpressionFiles = new RankExpressionFiles(); + /** The validation overrides of this. This is never null. */ private final ValidationOverrides validationOverrides; @@ -181,15 +185,17 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri createGlobalRankProfiles(deployState.getDeployLogger(), deployState.getImportedModels(), deployState.rankProfileRegistry(), deployState.getQueryProfiles()); - this.rankProfileList = new RankProfileList(null, // null search -> global - rankingConstants, AttributeFields.empty, - deployState.rankProfileRegistry(), - deployState.getQueryProfiles().getRegistry(), - deployState.getImportedModels(), - deployState.getProperties()); + rankProfileList = new RankProfileList(null, // null search -> global + rankingConstants, + rankExpressionFiles, + AttributeFields.empty, + deployState.rankProfileRegistry(), + deployState.getQueryProfiles().getRegistry(), + deployState.getImportedModels(), + deployState.getProperties()); HostSystem hostSystem = root.hostSystem(); - if (complete) { // create a a completed, frozen model + if (complete) { // create a completed, frozen model configModelRepo.readConfigModels(deployState, this, builder, root, configModelRegistry); addServiceClusters(deployState, builder); setupRouting(deployState); @@ -260,6 +266,8 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri /** Returns the global ranking constants of this */ public RankingConstants rankingConstants() { return rankingConstants; } + public RankExpressionFiles rankExpressionFiles() { return rankExpressionFiles; } + /** Creates a mutable model with no services instantiated */ public static VespaModel createIncomplete(DeployState deployState) throws IOException, SAXException { return new VespaModel(new NullConfigModelRegistry(), deployState, false, diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java index d1b9bbb4e58..e080ce43730 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java @@ -148,15 +148,23 @@ public class Admin extends AbstractConfigProducer<Admin> implements Serializable public void setClusterControllers(ClusterControllerContainerCluster clusterControllers, DeployLogger deployLogger) { this.clusterControllers = clusterControllers; - if (isHostedVespa) + if (isHostedVespa) { + // Prefer to put Slobroks on the admin cluster running cluster controllers to avoid unnecessary + // movement of the slobroks when there are changes to the content cluster nodes + removeSlobroks(); addSlobroks(createSlobroksOn(clusterControllers, deployLogger)); + } + } + + private void removeSlobroks() { + slobroks.forEach(Slobrok::remove); + slobroks.clear(); } private List<Slobrok> createSlobroksOn(ClusterControllerContainerCluster clusterControllers, DeployLogger deployLogger) { List<Slobrok> slobroks = new ArrayList<>(); - int index = this.slobroks.size(); for (ClusterControllerContainer clusterController : clusterControllers.getContainers()) { - Slobrok slobrok = new Slobrok(this, index++); + Slobrok slobrok = new Slobrok(this, clusterController.index()); slobrok.setHostResource(clusterController.getHostResource()); slobroks.add(slobrok); slobrok.initService(deployLogger); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Slobrok.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Slobrok.java index eed886b707f..00b6d872ddd 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/Slobrok.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Slobrok.java @@ -9,9 +9,10 @@ import com.yahoo.vespa.model.PortAllocBridge; /** * Represents a Slobrok service. * - * @author gjoranv + * @author gjoranv */ public class Slobrok extends AbstractService implements StateserverConfig.Producer { + private static final long serialVersionUID = 1L; public final static int BASEPORT = 19099; @@ -25,7 +26,7 @@ public class Slobrok extends AbstractService implements StateserverConfig.Produc * @param parent the parent ConfigProducer. * @param index unique index for all slobroks */ - public Slobrok(AbstractConfigProducer parent, int index) { + public Slobrok(AbstractConfigProducer<?> parent, int index) { super(parent, "slobrok." + index); portsMeta.on(0).tag("rpc").tag("admin").tag("status"); portsMeta.on(1).tag("http").tag("state"); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java index 7049213ddab..d87c6596fa4 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java @@ -8,6 +8,8 @@ import com.yahoo.log.InvalidLogFormatException; import com.yahoo.log.LogMessage; import com.yahoo.path.Path; import com.yahoo.searchdefinition.OnnxModel; +import com.yahoo.searchdefinition.RankExpressionFile; +import com.yahoo.vespa.config.search.core.RankingExpressionsConfig; import com.yahoo.vespa.defaults.Defaults; import com.yahoo.yolean.Exceptions; import com.yahoo.system.ProcessExecuter; @@ -38,7 +40,6 @@ import java.util.logging.Logger; import java.util.logging.Level; import java.util.ArrayList; import java.util.List; -import java.util.Map; /** * Validate rank setup for all search clusters (rank-profiles, index-schema, attributes configs), validating done @@ -116,50 +117,61 @@ public class RankSetupValidator extends Validator { } private void writeConfigs(String dir, AbstractConfigProducer<?> producer) throws IOException { - RankProfilesConfig.Builder rpcb = new RankProfilesConfig.Builder(); - ((RankProfilesConfig.Producer) producer).getConfig(rpcb); - RankProfilesConfig rpc = new RankProfilesConfig(rpcb); - writeConfig(dir, RankProfilesConfig.getDefName() + ".cfg", rpc); - - IndexschemaConfig.Builder iscb = new IndexschemaConfig.Builder(); - ((IndexschemaConfig.Producer) producer).getConfig(iscb); - IndexschemaConfig isc = new IndexschemaConfig(iscb); - writeConfig(dir, IndexschemaConfig.getDefName() + ".cfg", isc); - - AttributesConfig.Builder acb = new AttributesConfig.Builder(); - ((AttributesConfig.Producer) producer).getConfig(acb); - AttributesConfig ac = new AttributesConfig(acb); - writeConfig(dir, AttributesConfig.getDefName() + ".cfg", ac); - - RankingConstantsConfig.Builder rccb = new RankingConstantsConfig.Builder(); - ((RankingConstantsConfig.Producer) producer).getConfig(rccb); - RankingConstantsConfig rcc = new RankingConstantsConfig(rccb); - writeConfig(dir, RankingConstantsConfig.getDefName() + ".cfg", rcc); - - OnnxModelsConfig.Builder omcb = new OnnxModelsConfig.Builder(); - ((OnnxModelsConfig.Producer) producer).getConfig(omcb); - OnnxModelsConfig omc = new OnnxModelsConfig(omcb); - writeConfig(dir, OnnxModelsConfig.getDefName() + ".cfg", omc); - - ImportedFieldsConfig.Builder ifcb = new ImportedFieldsConfig.Builder(); - ((ImportedFieldsConfig.Producer) producer).getConfig(ifcb); - ImportedFieldsConfig ifc = new ImportedFieldsConfig(ifcb); - writeConfig(dir, ImportedFieldsConfig.getDefName() + ".cfg", ifc); + RankProfilesConfig.Builder rpcb = new RankProfilesConfig.Builder(); + ((RankProfilesConfig.Producer) producer).getConfig(rpcb); + RankProfilesConfig rpc = new RankProfilesConfig(rpcb); + writeConfig(dir, RankProfilesConfig.getDefName() + ".cfg", rpc); + + IndexschemaConfig.Builder iscb = new IndexschemaConfig.Builder(); + ((IndexschemaConfig.Producer) producer).getConfig(iscb); + IndexschemaConfig isc = new IndexschemaConfig(iscb); + writeConfig(dir, IndexschemaConfig.getDefName() + ".cfg", isc); + + AttributesConfig.Builder acb = new AttributesConfig.Builder(); + ((AttributesConfig.Producer) producer).getConfig(acb); + AttributesConfig ac = new AttributesConfig(acb); + writeConfig(dir, AttributesConfig.getDefName() + ".cfg", ac); + + RankingConstantsConfig.Builder rccb = new RankingConstantsConfig.Builder(); + ((RankingConstantsConfig.Producer) producer).getConfig(rccb); + RankingConstantsConfig rcc = new RankingConstantsConfig(rccb); + writeConfig(dir, RankingConstantsConfig.getDefName() + ".cfg", rcc); + + RankingExpressionsConfig.Builder recb = new RankingExpressionsConfig.Builder(); + ((RankingExpressionsConfig.Producer) producer).getConfig(recb); + RankingExpressionsConfig rec = new RankingExpressionsConfig(recb); + writeConfig(dir, RankingExpressionsConfig.getDefName() + ".cfg", rec); + + OnnxModelsConfig.Builder omcb = new OnnxModelsConfig.Builder(); + ((OnnxModelsConfig.Producer) producer).getConfig(omcb); + OnnxModelsConfig omc = new OnnxModelsConfig(omcb); + writeConfig(dir, OnnxModelsConfig.getDefName() + ".cfg", omc); + + ImportedFieldsConfig.Builder ifcb = new ImportedFieldsConfig.Builder(); + ((ImportedFieldsConfig.Producer) producer).getConfig(ifcb); + ImportedFieldsConfig ifc = new ImportedFieldsConfig(ifcb); + writeConfig(dir, ImportedFieldsConfig.getDefName() + ".cfg", ifc); } private void writeExtraVerifyRanksetupConfig(String dir, DocumentDatabase db) throws IOException { String configName = "verify-ranksetup.cfg"; String configContent = ""; + List<String> config = new ArrayList<>(); // Assist verify-ranksetup in finding the actual ONNX model files - Map<String, OnnxModel> models = db.getDerivedConfiguration().getSearch().onnxModels().asMap(); - if (models.values().size() > 0) { - List<String> config = new ArrayList<>(models.values().size() * 2); - for (OnnxModel model : models.values()) { - String modelPath = getFileRepositoryPath(model.getFilePath(), model.getFileReference()); - config.add(String.format("file[%d].ref \"%s\"", config.size() / 2, model.getFileReference())); - config.add(String.format("file[%d].path \"%s\"", config.size() / 2, modelPath)); - } + for (OnnxModel model : db.getDerivedConfiguration().getSearch().onnxModels().asMap().values()) { + String modelPath = getFileRepositoryPath(model.getFilePath(), model.getFileReference()); + config.add(String.format("file[%d].ref \"%s\"", config.size() / 2, model.getFileReference())); + config.add(String.format("file[%d].path \"%s\"", config.size() / 2, modelPath)); + } + + for (RankExpressionFile expr : db.getDerivedConfiguration().getSearch().rankExpressionFiles().asMap().values()) { + String modelPath = getFileRepositoryPath(expr.getFilePath(), expr.getFileReference()); + config.add(String.format("file[%d].ref \"%s\"", config.size() / 2, expr.getFileReference())); + config.add(String.format("file[%d].path \"%s\"", config.size() / 2, modelPath)); + } + + if ( ! config.isEmpty() ) { configContent = StringUtilities.implodeMultiline(config); } IOUtils.writeFile(dir + configName, configContent, false); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java index a0673824907..bbec64d45fb 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java @@ -67,7 +67,7 @@ public class DomAdminV4Builder extends DomAdminBuilderBase { admin, allocateHosts(admin.hostSystem(), "slobroks", nodesSpecification)); } - else { // TODO: Remove + else { // These will be removed later, if an admin cluster (for cluster controllers) is assigned createSlobroks(deployLogger, admin, pickContainerHostsForSlobrok(nodesSpecification.minResources().nodes(), 2)); @@ -153,8 +153,8 @@ public class DomAdminV4Builder extends DomAdminBuilderBase { List<HostResource> picked = sortedContainerHostsFrom(model, count, !retired); // if we can return multiple hosts, include retired nodes which would have been picked before - // (probably - assuming all previous nodes were retired, which is always true for a single cluster - // at the moment (Sept 2015)) to ensure a smoother transition between the old and new topology + // (probably - assuming all previous nodes were retired, which is always true for a single cluster, + // to ensure a smoother transition between the old and new topology // by including both new and old nodes during the retirement period picked.addAll(sortedContainerHostsFrom(model, count, retired)); @@ -166,6 +166,7 @@ public class DomAdminV4Builder extends DomAdminBuilderBase { List<HostResource> hosts = model.getCluster().getContainers().stream() .filter(container -> retired == container.isRetired()) .map(Container::getHostResource) + .sorted(HostResource::comparePrimarilyByIndexTo) .collect(Collectors.toList()); return hosts.subList(0, Math.min(count, hosts.size())); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomFederationSearcherBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomFederationSearcherBuilder.java index 3a4e8a70613..36bf792ee82 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomFederationSearcherBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomFederationSearcherBuilder.java @@ -43,7 +43,6 @@ public class DomFederationSearcherBuilder extends VespaDomBuilder.DomConfigProdu return XML.getChild(searcherSpec, "source-set") != null; } - private List<FederationSearcherModel.TargetSpec> readSources(Element searcherSpec) { List<FederationSearcherModel.TargetSpec> sources = new ArrayList<>(); for (Element source : XML.getChildren(searcherSpec, "source")) { @@ -76,14 +75,14 @@ public class DomFederationSearcherBuilder extends VespaDomBuilder.DomConfigProdu } @Override - protected FederationSearcher doBuild(DeployState deployState, AbstractConfigProducer ancestor, Element searcherElement) { + protected FederationSearcher doBuild(DeployState deployState, AbstractConfigProducer<?> ancestor, Element searcherElement) { FederationSearcherModel model = new FederationSearcherModelBuilder(searcherElement).build(); Optional<Component> targetSelector = buildTargetSelector(deployState, ancestor, searcherElement, model.getComponentId()); return new FederationSearcher(model, targetSelector); } - private Optional<Component> buildTargetSelector(DeployState deployState, AbstractConfigProducer ancestor, Element searcherElement, ComponentId namespace) { + private Optional<Component> buildTargetSelector(DeployState deployState, AbstractConfigProducer<?> ancestor, Element searcherElement, ComponentId namespace) { Element targetSelectorElement = XML.getChild(searcherElement, "target-selector"); if (targetSelectorElement == null) return Optional.empty(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/SearchChainsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/SearchChainsBuilder.java index 0106123666d..9fb19efbf75 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/SearchChainsBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/SearchChainsBuilder.java @@ -16,9 +16,10 @@ import java.util.List; import java.util.Map; /** + * Creates top level search chains(searchchain, provider) from xml. + * * @author Tony Vaagenes * @author gjoranv - * Creates top level search chains(searchchain, provider) from xml. */ public class SearchChainsBuilder extends ChainsBuilder<Searcher<?>, SearchChain> { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index 4e78f44d0fe..317ed0f66c7 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -25,6 +25,7 @@ import com.yahoo.search.config.QrStartConfig; import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.vespa.config.search.core.OnnxModelsConfig; import com.yahoo.vespa.config.search.core.RankingConstantsConfig; +import com.yahoo.vespa.config.search.core.RankingExpressionsConfig; import com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainer; import com.yahoo.vespa.model.container.component.BindingPattern; import com.yahoo.vespa.model.container.component.Component; @@ -58,6 +59,7 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat RankProfilesConfig.Producer, RankingConstantsConfig.Producer, OnnxModelsConfig.Producer, + RankingExpressionsConfig.Producer, ServletPathsConfig.Producer, ContainerMbusConfig.Producer, MetricsProxyApiConfig.Producer, @@ -233,6 +235,10 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat if (modelEvaluation != null) modelEvaluation.getConfig(builder); } + public void getConfig(RankingExpressionsConfig.Builder builder) { + if (modelEvaluation != null) modelEvaluation.getConfig(builder); + } + @Override public void getConfig(ContainerMbusConfig.Builder builder) { if (mbusParams != null) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java index 6ef29269bc1..cd596038137 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java @@ -79,7 +79,6 @@ public abstract class Container extends AbstractService implements private final ComponentGroup<Component<?, ?>> components = new ComponentGroup<>(this, "components"); private final JettyHttpServer defaultHttpServer; - private final boolean enableJdiscHttp2; protected Container(AbstractConfigProducer<?> parent, String name, int index, DeployState deployState) { this(parent, name, false, index, deployState); @@ -100,8 +99,6 @@ public abstract class Container extends AbstractService implements addChild(new SimpleComponent("com.yahoo.container.jdisc.ConfiguredApplication$ApplicationContext")); appendJvmOptions(jvmOmitStackTraceInFastThrowOption(deployState.featureFlags())); - - this.enableJdiscHttp2 = deployState.featureFlags().enableJdiscHttp2(); } protected String jvmOmitStackTraceInFastThrowOption(ModelContext.FeatureFlags featureFlags) { @@ -183,7 +180,7 @@ public abstract class Container extends AbstractService implements } private void initDefaultJettyConnector() { - defaultHttpServer.addConnector(new ConnectorFactory.Builder("SearchServer", getSearchPort()).enableHttp2(enableJdiscHttp2).build()); + defaultHttpServer.addConnector(new ConnectorFactory.Builder("SearchServer", getSearchPort()).build()); } private ContainerServiceType myServiceType = null; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java index 510d2fe3d99..37bfb8821c3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java @@ -7,6 +7,7 @@ import com.yahoo.searchdefinition.derived.RankProfileList; import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.vespa.config.search.core.OnnxModelsConfig; import com.yahoo.vespa.config.search.core.RankingConstantsConfig; +import com.yahoo.vespa.config.search.core.RankingExpressionsConfig; import com.yahoo.vespa.model.container.component.Handler; import com.yahoo.vespa.model.container.component.SystemBindingPattern; @@ -18,10 +19,12 @@ import java.util.Objects; * * @author bratseth */ -public class ContainerModelEvaluation implements RankProfilesConfig.Producer, - RankingConstantsConfig.Producer, - OnnxModelsConfig.Producer -{ + +public class ContainerModelEvaluation implements + RankProfilesConfig.Producer, + RankingConstantsConfig.Producer, + OnnxModelsConfig.Producer, + RankingExpressionsConfig.Producer { private final static String BUNDLE_NAME = "model-evaluation"; private final static String EVALUATOR_NAME = ModelsEvaluator.class.getName(); @@ -38,8 +41,7 @@ public class ContainerModelEvaluation implements RankProfilesConfig.Producer, } public void prepare(List<ApplicationContainer> containers) { - rankProfileList.sendConstantsTo(containers); - rankProfileList.sendOnnxModelsTo(containers); + rankProfileList.sendTo(containers); } @Override @@ -57,6 +59,10 @@ public class ContainerModelEvaluation implements RankProfilesConfig.Producer, rankProfileList.getConfig(builder); } + public void getConfig(RankingExpressionsConfig.Builder builder) { + rankProfileList.getConfig(builder); + } + public static Handler<?> getHandler() { Handler<?> handler = new Handler<>(new ComponentModel(REST_HANDLER_NAME, null, BUNDLE_NAME)); handler.addServerBindings( diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ConnectorFactory.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ConnectorFactory.java index 9b0075c79c3..0d024398de7 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ConnectorFactory.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ConnectorFactory.java @@ -20,7 +20,6 @@ public class ConnectorFactory extends SimpleComponent implements ConnectorConfig private final String name; private final int listenPort; private final SslProvider sslProviderComponent; - private final boolean enableHttp2; private volatile ComponentId defaultRequestFilterChain; private volatile ComponentId defaultResponseFilterChain; @@ -33,7 +32,6 @@ public class ConnectorFactory extends SimpleComponent implements ConnectorConfig this.sslProviderComponent = builder.sslProvider != null ? builder.sslProvider : new DefaultSslProvider(name); this.defaultRequestFilterChain = builder.defaultRequestFilterChain; this.defaultResponseFilterChain = builder.defaultResponseFilterChain; - this.enableHttp2 = builder.enableHttp2 != null ? builder.enableHttp2 : false; addChild(sslProviderComponent); inject(sslProviderComponent); } @@ -42,7 +40,6 @@ public class ConnectorFactory extends SimpleComponent implements ConnectorConfig public void getConfig(ConnectorConfig.Builder connectorBuilder) { connectorBuilder.listenPort(listenPort); connectorBuilder.name(name); - connectorBuilder.http2Enabled(enableHttp2); sslProviderComponent.amendConnectorConfig(connectorBuilder); } @@ -69,7 +66,6 @@ public class ConnectorFactory extends SimpleComponent implements ConnectorConfig private SslProvider sslProvider; private ComponentId defaultRequestFilterChain; private ComponentId defaultResponseFilterChain; - private Boolean enableHttp2; public Builder(String name, int listenPort) { this.name = name; @@ -88,8 +84,6 @@ public class ConnectorFactory extends SimpleComponent implements ConnectorConfig this.defaultResponseFilterChain = filterChain; return this; } - public Builder enableHttp2(boolean enabled) { this.enableHttp2 = enabled; return this; } - public ConnectorFactory build() { return new ConnectorFactory(this); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java index 9d715073f77..766aa46fc01 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java @@ -29,32 +29,32 @@ public class HostedSslConnectorFactory extends ConnectorFactory { * Create connector factory that uses a certificate provided by the config-model / configserver and default hosted Vespa truststore. */ public static HostedSslConnectorFactory withProvidedCertificate( - String serverName, EndpointCertificateSecrets endpointCertificateSecrets, boolean enforceHandshakeClientAuth, boolean enableHttp2) { + String serverName, EndpointCertificateSecrets endpointCertificateSecrets, boolean enforceHandshakeClientAuth) { ConfiguredDirectSslProvider sslProvider = createConfiguredDirectSslProvider( serverName, endpointCertificateSecrets, DEFAULT_HOSTED_TRUSTSTORE, /*tlsCaCertificates*/null, enforceHandshakeClientAuth); - return new HostedSslConnectorFactory(sslProvider, false, enforceHandshakeClientAuth, enableHttp2); + return new HostedSslConnectorFactory(sslProvider, false, enforceHandshakeClientAuth); } /** * Create connector factory that uses a certificate provided by the config-model / configserver and a truststore configured by the application. */ public static HostedSslConnectorFactory withProvidedCertificateAndTruststore( - String serverName, EndpointCertificateSecrets endpointCertificateSecrets, String tlsCaCertificates, boolean enableHttp2) { + String serverName, EndpointCertificateSecrets endpointCertificateSecrets, String tlsCaCertificates) { ConfiguredDirectSslProvider sslProvider = createConfiguredDirectSslProvider( serverName, endpointCertificateSecrets, /*tlsCaCertificatesPath*/null, tlsCaCertificates, false); - return new HostedSslConnectorFactory(sslProvider, true, false, enableHttp2); + return new HostedSslConnectorFactory(sslProvider, true, false); } /** * Create connector factory that uses the default certificate and truststore provided by Vespa (through Vespa-global TLS configuration). */ - public static HostedSslConnectorFactory withDefaultCertificateAndTruststore(String serverName, boolean enableHttp2) { - return new HostedSslConnectorFactory(new DefaultSslProvider(serverName), true, false, enableHttp2); + public static HostedSslConnectorFactory withDefaultCertificateAndTruststore(String serverName) { + return new HostedSslConnectorFactory(new DefaultSslProvider(serverName), true, false); } private HostedSslConnectorFactory(SslProvider sslProvider, boolean enforceClientAuth, - boolean enforceHandshakeClientAuth, boolean enableHttp2) { - super(new Builder("tls4443", 4443).sslProvider(sslProvider).enableHttp2(enableHttp2)); + boolean enforceHandshakeClientAuth) { + super(new Builder("tls4443", 4443).sslProvider(sslProvider)); this.enforceClientAuth = enforceClientAuth; this.enforceHandshakeClientAuth = enforceHandshakeClientAuth; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyConnectorBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyConnectorBuilder.java index 3b616c34a03..22fa5332d5e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyConnectorBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyConnectorBuilder.java @@ -39,7 +39,7 @@ public class JettyConnectorBuilder extends VespaDomBuilder.DomConfigProducerBuil .map(ComponentId::new) .ifPresent(builder::defaultResponseFilterChain); SslProvider sslProviderComponent = getSslConfigComponents(name, serverSpec); - return builder.sslProvider(sslProviderComponent).enableHttp2(deployState.featureFlags().enableJdiscHttp2()).build(); + return builder.sslProvider(sslProviderComponent).build(); } SslProvider getSslConfigComponents(String serverName, Element serverSpec) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/FederationSearcher.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/FederationSearcher.java index ceb48732116..6b4cb003cda 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/FederationSearcher.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/FederationSearcher.java @@ -11,7 +11,12 @@ import com.yahoo.search.federation.FederationConfig; import com.yahoo.search.searchchain.model.federation.FederationSearcherModel.TargetSpec; import com.yahoo.vespa.model.container.component.Component; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; /** * Config producer for the FederationSearcher. @@ -26,8 +31,8 @@ public class FederationSearcher extends Searcher<FederationSearcherModel> implem * Generates config for a single search chain contained in a target. */ private static final class SearchChainConfig { + private final SearchChain searchChain; - //Zero if not applicable final ComponentId providerId; final FederationOptions targetOptions; final List<String> documentTypes; @@ -61,6 +66,7 @@ public class FederationSearcher extends Searcher<FederationSearcherModel> implem * which can be federated to as a single entity. */ private static abstract class Target { + final ComponentId id; final FederationOptions targetOptions; @@ -79,41 +85,36 @@ public class FederationSearcher extends Searcher<FederationSearcherModel> implem } protected abstract void getSearchChainsConfig(FederationConfig.Target.Builder tb); + } private static class SearchChainTarget extends Target { + private final SearchChainConfig searchChainConfig; - public SearchChainTarget(SearchChain searchChain, - FederationOptions targetOptions) { + public SearchChainTarget(SearchChain searchChain, FederationOptions targetOptions) { super(searchChain.getComponentId(), targetOptions); - searchChainConfig = new SearchChainConfig( - searchChain, - null, - targetOptions, - searchChain.getDocumentTypes()); + searchChainConfig = new SearchChainConfig(searchChain, null, targetOptions, searchChain.getDocumentTypes()); } @Override protected void getSearchChainsConfig(FederationConfig.Target.Builder tB) { tB.searchChain(searchChainConfig.getSearchChainConfig()); } + } private static class SourceGroupTarget extends Target { + private final SearchChainConfig leaderConfig; - private final List<SearchChainConfig> participantsConfig = - new ArrayList<>(); + private final List<SearchChainConfig> participantsConfig = new ArrayList<>(); - public SourceGroupTarget(SourceGroup group, - FederationOptions targetOptions) { + public SourceGroupTarget(SourceGroup group, FederationOptions targetOptions) { super(group.getComponentId(), applyDefaultSourceGroupOptions(targetOptions)); leaderConfig = createConfig(group.leader(), targetOptions); - for (Source participant : group.participants()) { - participantsConfig.add( - createConfig(participant, targetOptions)); - } + for (Source participant : group.participants()) + participantsConfig.add(createConfig(participant, targetOptions)); } private static FederationOptions applyDefaultSourceGroupOptions(FederationOptions targetOptions) { @@ -121,64 +122,49 @@ public class FederationSearcher extends Searcher<FederationSearcherModel> implem return targetOptions.inherit(defaultSourceGroupOption); } - private SearchChainConfig createConfig(Source source, - FederationOptions targetOptions) { - return new SearchChainConfig( - source, - source.getParentProvider().getComponentId(), - targetOptions, - source.getDocumentTypes()); + private SearchChainConfig createConfig(Source source, FederationOptions targetOptions) { + return new SearchChainConfig(source, + source.getParentProvider().getComponentId(), + targetOptions, + source.getDocumentTypes()); } @Override protected void getSearchChainsConfig(FederationConfig.Target.Builder tB) { tB.searchChain(leaderConfig.getSearchChainConfig()); - for (SearchChainConfig participant : participantsConfig) { + for (SearchChainConfig participant : participantsConfig) tB.searchChain(participant.getSearchChainConfig()); - } } } private static class TargetResolver { + final ComponentRegistry<SearchChain> searchChainRegistry; final SourceGroupRegistry sourceGroupRegistry; - /** - * @return true if searchChain.id newer than sourceGroup.id - */ - private boolean newerVersion(SearchChain searchChain, - SourceGroup sourceGroup) { - if (searchChain == null || sourceGroup == null) { - return false; - } else { - return newerVersion(searchChain.getComponentId(), sourceGroup.getComponentId()); - } + /** Returns true if searchChain.id newer than sourceGroup.id */ + private boolean newerVersion(SearchChain searchChain, SourceGroup sourceGroup) { + if (searchChain == null || sourceGroup == null) return false; + return newerVersion(searchChain.getComponentId(), sourceGroup.getComponentId()); } - /** - * @return true if a newer than b - */ + /** Returns true if a newer than b */ private boolean newerVersion(ComponentId a, ComponentId b) { return a.compareTo(b) > 0; } - - TargetResolver(ComponentRegistry<SearchChain> searchChainRegistry, - SourceGroupRegistry sourceGroupRegistry) { + TargetResolver(ComponentRegistry<SearchChain> searchChainRegistry, SourceGroupRegistry sourceGroupRegistry) { this.searchChainRegistry = searchChainRegistry; this.sourceGroupRegistry = sourceGroupRegistry; } Target resolve(FederationSearcherModel.TargetSpec specification) { - SearchChain searchChain = searchChainRegistry.getComponent( - specification.sourceSpec); - SourceGroup sourceGroup = sourceGroupRegistry.getComponent( - specification.sourceSpec); + SearchChain searchChain = searchChainRegistry.getComponent(specification.sourceSpec); + SourceGroup sourceGroup = sourceGroupRegistry.getComponent(specification.sourceSpec); if (searchChain == null && sourceGroup == null) { return null; - } else if (sourceGroup == null || - newerVersion(searchChain, sourceGroup)) { + } else if (sourceGroup == null || newerVersion(searchChain, sourceGroup)) { return new SearchChainTarget(searchChain, specification.federationOptions); } else { return new SourceGroupTarget(sourceGroup, specification.federationOptions); @@ -186,26 +172,21 @@ public class FederationSearcher extends Searcher<FederationSearcherModel> implem } } - private final Map<ComponentId, Target> resolvedTargets = - new LinkedHashMap<>(); + private final Map<ComponentId, Target> resolvedTargets = new LinkedHashMap<>(); public FederationSearcher(FederationSearcherModel searcherModel, Optional<Component> targetSelector) { super(searcherModel); this.targetSelector = targetSelector; - if (targetSelector.isPresent()) - addChild(targetSelector.get()); + targetSelector.ifPresent(selector -> addChild(selector)); } @Override public void getConfig(FederationConfig.Builder builder) { - for (Target target : resolvedTargets.values()) { + for (Target target : resolvedTargets.values()) builder.target(target.getTargetConfig()); - } - if (targetSelector.isPresent()) { - builder.targetSelector(targetSelector.get().getGlobalComponentId().stringValue()); - } + targetSelector.ifPresent(selector -> builder.targetSelector(selector.getGlobalComponentId().stringValue())); } @Override @@ -213,10 +194,8 @@ public class FederationSearcher extends Searcher<FederationSearcherModel> implem initialize(getSearchChains().allChains(), getSearchChains().allSourceGroups()); } - void initialize(ComponentRegistry<SearchChain> searchChainRegistry, - SourceGroupRegistry sourceGroupRegistry) { - TargetResolver targetResolver = new TargetResolver( - searchChainRegistry, sourceGroupRegistry); + void initialize(ComponentRegistry<SearchChain> searchChainRegistry, SourceGroupRegistry sourceGroupRegistry) { + TargetResolver targetResolver = new TargetResolver(searchChainRegistry, sourceGroupRegistry); addSourceTargets(targetResolver, model.targets); @@ -229,16 +208,14 @@ public class FederationSearcher extends Searcher<FederationSearcherModel> implem Target target = targetResolver.resolve(targetSpec); if (target == null) { - throw new RuntimeException("Can't find source " + - targetSpec.sourceSpec + - " used as a source for federation '" + - getComponentId() + "'"); + throw new RuntimeException("Can't find source " + targetSpec.sourceSpec + + " used as a source for federation '" + getComponentId() + "'"); } Target duplicate = resolvedTargets.put(target.id, target); if (duplicate != null && !duplicate.targetOptions.equals(target.targetOptions)) { - throw new RuntimeException("Search chain " + target.id + " added twice with different federation options" - + " to the federation searcher " + getComponentId()); + throw new RuntimeException("Search chain " + target.id + " added twice with different federation options" + + " to the federation searcher " + getComponentId()); } } } @@ -248,23 +225,21 @@ public class FederationSearcher extends Searcher<FederationSearcherModel> implem for (GenericTarget genericTarget : defaultTargets(searchChainRegistry.allComponents())) { ComponentSpecification specification = genericTarget.getComponentId().toSpecification(); - //Can't use genericTarget directly, as it might be part of a source group. + // Can't use genericTarget directly, as it might be part of a source group. Target federationTarget = targetResolver.resolve(new TargetSpec(specification, new FederationOptions())); - //Do not replace manually added sources, as they might have manually configured federation options + // Do not replace manually added sources, as they might have manually configured federation options if (!resolvedTargets.containsKey(federationTarget.id)) resolvedTargets.put(federationTarget.id, federationTarget); } } - private static List<GenericTarget> defaultTargets(Collection<SearchChain> allSearchChains) { - Collection<Provider> providers = - CollectionUtil.filter(allSearchChains, Provider.class); + Collection<Provider> providers = CollectionUtil.filter(allSearchChains, Provider.class); List<GenericTarget> defaultTargets = new ArrayList<>(); - for (Provider provider : providers) { + for (Provider provider : providers) defaultTargets.addAll(provider.defaultFederationTargets()); - } return defaultTargets; } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/Provider.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/Provider.java index ee4edf3fd8c..10e0f3e55da 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/Provider.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/Provider.java @@ -7,6 +7,7 @@ import com.yahoo.vespa.model.container.component.ConfigProducerGroup; import java.util.Arrays; import java.util.Collection; +import java.util.List; /** * Base config producer for search chains that communicate with backends. @@ -15,7 +16,7 @@ import java.util.Collection; */ public class Provider extends GenericTarget { - private ConfigProducerGroup<Source> sources; + private final ConfigProducerGroup<Source> sources; public Provider(ChainSpecification specWithoutInnerSearchers, FederationOptions federationOptions) { super(specWithoutInnerSearchers, federationOptions); @@ -37,9 +38,10 @@ public class Provider extends GenericTarget { public Collection<? extends GenericTarget> defaultFederationTargets() { if (sources.getComponents().isEmpty()) { - return Arrays.asList(this); + return List.of(this); } else { return sources.getComponents(); } } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 87406b8bc9a..b477587bcac 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -431,7 +431,6 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { // If the deployment contains certificate/private key reference, setup TLS port HostedSslConnectorFactory connectorFactory; - boolean enableHttp2 = deployState.featureFlags().enableJdiscHttp2(); if (deployState.endpointCertificateSecrets().isPresent()) { boolean authorizeClient = deployState.zone().system().isPublic(); if (authorizeClient && deployState.tlsClientAuthority().isEmpty()) { @@ -445,10 +444,10 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { .orElse(false); connectorFactory = authorizeClient - ? HostedSslConnectorFactory.withProvidedCertificateAndTruststore(serverName, endpointCertificateSecrets, deployState.tlsClientAuthority().get(), enableHttp2) - : HostedSslConnectorFactory.withProvidedCertificate(serverName, endpointCertificateSecrets, enforceHandshakeClientAuth, enableHttp2); + ? HostedSslConnectorFactory.withProvidedCertificateAndTruststore(serverName, endpointCertificateSecrets, deployState.tlsClientAuthority().get()) + : HostedSslConnectorFactory.withProvidedCertificate(serverName, endpointCertificateSecrets, enforceHandshakeClientAuth); } else { - connectorFactory = HostedSslConnectorFactory.withDefaultCertificateAndTruststore(serverName, enableHttp2); + connectorFactory = HostedSslConnectorFactory.withDefaultCertificateAndTruststore(serverName); } cluster.getHttp().getAccessControl().ifPresent(accessControl -> accessControl.configureHostedConnector(connectorFactory)); server.addConnector(connectorFactory); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterResourceLimits.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterResourceLimits.java index c7d56c9b4b5..70f2acd3c7b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterResourceLimits.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterResourceLimits.java @@ -1,6 +1,7 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.content; +import com.yahoo.config.application.api.DeployLogger; import com.yahoo.vespa.model.builder.xml.dom.ModelElement; import com.yahoo.vespa.model.content.cluster.DomResourceLimitsBuilder; @@ -35,28 +36,31 @@ public class ClusterResourceLimits { public static class Builder { private final boolean enableFeedBlockInDistributor; + private final boolean hostedVespa; + private final DeployLogger deployLogger; private ResourceLimits.Builder ctrlBuilder = new ResourceLimits.Builder(); private ResourceLimits.Builder nodeBuilder = new ResourceLimits.Builder(); - public Builder(boolean enableFeedBlockInDistributor) { + public Builder(boolean enableFeedBlockInDistributor, boolean hostedVespa, DeployLogger deployLogger) { this.enableFeedBlockInDistributor = enableFeedBlockInDistributor; + this.hostedVespa = hostedVespa; + this.deployLogger = deployLogger; } public ClusterResourceLimits build(ModelElement clusterElem) { - ModelElement tuningElem = clusterElem.childByPath("tuning"); - if (tuningElem != null) { - ctrlBuilder = DomResourceLimitsBuilder.createBuilder(tuningElem); - } - - ModelElement protonElem = clusterElem.childByPath("engine.proton"); - if (protonElem != null) { - nodeBuilder = DomResourceLimitsBuilder.createBuilder(protonElem); - } + ctrlBuilder = createBuilder(clusterElem.childByPath("tuning")); + nodeBuilder = createBuilder(clusterElem.childByPath("engine.proton")); deriveLimits(); return new ClusterResourceLimits(this); } + private ResourceLimits.Builder createBuilder(ModelElement element) { + return element == null + ? new ResourceLimits.Builder() + : DomResourceLimitsBuilder.createBuilder(element, hostedVespa, deployLogger); + } + public void setClusterControllerBuilder(ResourceLimits.Builder builder) { ctrlBuilder = builder; } @@ -88,7 +92,7 @@ public class ClusterResourceLimits { Optional<Double> contentNodeLimit, Consumer<Double> setter) { // TODO: remove this when feed block in distributor is default enabled. - if (!clusterControllerLimit.isPresent() && !contentNodeLimit.isPresent()) { + if (clusterControllerLimit.isEmpty() && contentNodeLimit.isEmpty()) { setter.accept(0.8); } } @@ -96,7 +100,7 @@ public class ClusterResourceLimits { private void deriveClusterControllerLimit(Optional<Double> clusterControllerLimit, Optional<Double> contentNodeLimit, Consumer<Double> setter) { - if (!clusterControllerLimit.isPresent()) { + if (clusterControllerLimit.isEmpty()) { contentNodeLimit.ifPresent(limit -> // TODO: emit warning when feed block in distributor is default enabled. setter.accept(Double.max(0.0, (limit - 0.01)))); @@ -106,7 +110,7 @@ public class ClusterResourceLimits { private void deriveContentNodeLimit(Optional<Double> contentNodeLimit, Optional<Double> clusterControllerLimit, Consumer<Double> setter) { - if (!contentNodeLimit.isPresent()) { + if (contentNodeLimit.isEmpty()) { clusterControllerLimit.ifPresent(limit -> setter.accept(calcContentNodeLimit(limit))); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index f6a45842bd9..b5cae857ce0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -25,7 +25,6 @@ import com.yahoo.vespa.config.content.core.BucketspacesConfig; import com.yahoo.vespa.config.content.core.StorDistributormanagerConfig; import com.yahoo.vespa.model.HostResource; import com.yahoo.vespa.model.admin.Admin; -import com.yahoo.vespa.model.admin.Configserver; import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerCluster; import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerComponent; import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerConfigurer; @@ -134,7 +133,10 @@ public class ContentCluster extends AbstractConfigProducer implements globallyDistributedDocuments, routingSelection, deployState.zone(), deployState.isHosted()); boolean enableFeedBlockInDistributor = deployState.getProperties().featureFlags().enableFeedBlockInDistributor(); - var resourceLimits = new ClusterResourceLimits.Builder(enableFeedBlockInDistributor).build(contentElement); + var resourceLimits = new ClusterResourceLimits.Builder(enableFeedBlockInDistributor, + stateIsHosted(deployState), + deployState.getDeployLogger()) + .build(contentElement); c.clusterControllerConfig = new ClusterControllerConfig.Builder(getClusterId(contentElement), contentElement, resourceLimits.getClusterControllerLimits()).build(deployState, c, contentElement.getXml()); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomResourceLimitsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomResourceLimitsBuilder.java index 210f062f9b2..9f4852629d0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomResourceLimitsBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomResourceLimitsBuilder.java @@ -1,9 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.content.cluster; +import com.yahoo.config.application.api.DeployLogger; import com.yahoo.vespa.model.builder.xml.dom.ModelElement; import com.yahoo.vespa.model.content.ResourceLimits; +import java.util.logging.Level; + /** * Builder for feed block resource limits. * @@ -11,10 +14,16 @@ import com.yahoo.vespa.model.content.ResourceLimits; */ public class DomResourceLimitsBuilder { - public static ResourceLimits.Builder createBuilder(ModelElement contentXml) { + public static ResourceLimits.Builder createBuilder(ModelElement contentXml, boolean hostedVespa, DeployLogger deployLogger) { ResourceLimits.Builder builder = new ResourceLimits.Builder(); ModelElement resourceLimits = contentXml.child("resource-limits"); - if (resourceLimits == null) { + if (resourceLimits == null) { return builder; } + + if (hostedVespa) { + deployLogger.logApplicationPackage(Level.WARNING, "Element " + resourceLimits + + " is not allowed, default limits will be used"); + // TODO: Throw exception when we are sure nobody is using this + //throw new IllegalArgumentException("Element " + element + " is not allowed to be set, default limits will be used"); return builder; } if (resourceLimits.child("disk") != null) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java index fb9402614f6..5bb57f4ff6c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java @@ -8,7 +8,12 @@ import com.yahoo.config.application.api.FileRegistry; import com.yahoo.vespa.model.ConfigProxy; import com.yahoo.vespa.model.Host; -import java.util.*; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + /** * Sends RPC requests to hosts (tenant hosts and config servers) asking them to start download of files. This is used diff --git a/config-model/src/main/java/com/yahoo/vespa/model/routing/DocumentProtocol.java b/config-model/src/main/java/com/yahoo/vespa/model/routing/DocumentProtocol.java index 15e6c5993b3..fbba377c950 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/routing/DocumentProtocol.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/routing/DocumentProtocol.java @@ -111,6 +111,8 @@ public final class DocumentProtocol implements Protocol, addSelector(cluster.getConfigId(), cluster.getRoutingSelector(), clusterBuilder); if (cluster.getSearch().hasIndexedCluster()) addRoutes(getDirectRouteName(cluster.getConfigId()), getIndexedRouteName(cluster.getConfigId()), clusterBuilder); + else + clusterBuilder.defaultRoute(cluster.getConfigId()); builder.cluster(cluster.getConfigId(), clusterBuilder); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java index 705c48d1c9b..3ca5b9e3a0a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java @@ -39,8 +39,7 @@ public abstract class AbstractSearchCluster extends AbstractConfigProducer<Abstr public void prepareToDistributeFiles(List<SearchNode> backends) { for (SchemaSpec sds : localSDS) { - sds.getSearchDefinition().getSearch().rankingConstants().sendTo(backends); - sds.getSearchDefinition().getSearch().onnxModels().sendTo(backends); + sds.getSearchDefinition().getSearch().sendTo(backends); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/DocumentDatabase.java b/config-model/src/main/java/com/yahoo/vespa/model/search/DocumentDatabase.java index 8d4c1675bbf..9f0ff7db9f8 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/DocumentDatabase.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/DocumentDatabase.java @@ -12,6 +12,7 @@ import com.yahoo.vespa.config.search.SummaryConfig; import com.yahoo.vespa.config.search.SummarymapConfig; import com.yahoo.vespa.config.search.core.OnnxModelsConfig; import com.yahoo.vespa.config.search.core.RankingConstantsConfig; +import com.yahoo.vespa.config.search.core.RankingExpressionsConfig; import com.yahoo.vespa.config.search.summary.JuniperrcConfig; import com.yahoo.vespa.configdefinition.IlscriptsConfig; @@ -26,6 +27,7 @@ public class DocumentDatabase extends AbstractConfigProducer<DocumentDatabase> i AttributesConfig.Producer, RankProfilesConfig.Producer, RankingConstantsConfig.Producer, + RankingExpressionsConfig.Producer, OnnxModelsConfig.Producer, IndexschemaConfig.Producer, JuniperrcConfig.Producer, @@ -75,6 +77,11 @@ public class DocumentDatabase extends AbstractConfigProducer<DocumentDatabase> i } @Override + public void getConfig(RankingExpressionsConfig.Builder builder) { + derivedCfg.getRankProfileList().getConfig(builder); + } + + @Override public void getConfig(RankingConstantsConfig.Builder builder) { derivedCfg.getRankProfileList().getConfig(builder); } |