diff options
Diffstat (limited to 'config-model/src/main/java')
40 files changed, 458 insertions, 268 deletions
diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java index 1d0541b67d1..dd66861f2ce 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java @@ -22,7 +22,7 @@ import com.yahoo.config.model.api.ValidationParameters; import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.config.model.application.provider.MockFileRegistry; import com.yahoo.config.model.provision.HostsXmlProvisioner; -import com.yahoo.config.model.provision.SingleNodeProvisioner; +import com.yahoo.config.model.provision .SingleNodeProvisioner; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Zone; @@ -142,12 +142,8 @@ public class DeployState implements ConfigDefinitionStore { this.semanticRules = semanticRules; // TODO: Remove this by seeing how pagetemplates are propagated this.importedModels = importMlModels(applicationPackage, modelImporters, deployLogger); - ValidationOverrides suppliedValidationOverrides = applicationPackage.getValidationOverrides().map(ValidationOverrides::fromXml) - .orElse(ValidationOverrides.empty); - this.validationOverrides = - zone.environment().isManuallyDeployed() // // Warn but allow in manually deployed zones - ? new ValidationOverrides.AllowAllValidationOverrides(suppliedValidationOverrides, deployLogger) - : suppliedValidationOverrides; + this.validationOverrides = applicationPackage.getValidationOverrides().map(ValidationOverrides::fromXml) + .orElse(ValidationOverrides.empty); this.wantedNodeVespaVersion = wantedNodeVespaVersion; this.now = now; @@ -460,7 +456,7 @@ public class DeployState implements ConfigDefinitionStore { private SearchDocumentModel createSearchDocumentModel(RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles, ValidationParameters validationParameters) { - Collection<NamedReader> readers = applicationPackage.getSearchDefinitions(); + Collection<NamedReader> readers = applicationPackage.getSchemas(); Map<String, String> names = new LinkedHashMap<>(); SearchBuilder builder = new SearchBuilder(applicationPackage, logger, properties, rankProfileRegistry, queryProfiles.getRegistry()); for (NamedReader reader : readers) { @@ -470,14 +466,14 @@ public class DeployState implements ConfigDefinitionStore { String sdName = stripSuffix(readerName, ApplicationPackage.SD_NAME_SUFFIX); names.put(topLevelName, sdName); if ( ! sdName.equals(topLevelName)) { - throw new IllegalArgumentException("Schema definition file name ('" + sdName + "') and name of " + + throw new IllegalArgumentException("Schema file name ('" + sdName + "') and name of " + "top level element ('" + topLevelName + "') are not equal for file '" + readerName + "'"); } } catch (ParseException e) { - throw new IllegalArgumentException("Could not parse sd file '" + reader.getName() + "'", e); + throw new IllegalArgumentException("Could not parse schema file '" + reader.getName() + "'", e); } catch (IOException e) { - throw new IllegalArgumentException("Could not read sd file '" + reader.getName() + "'", e); + throw new IllegalArgumentException("Could not read schema file '" + reader.getName() + "'", e); } finally { closeIgnoreException(reader.getReader()); } diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 304855e545d..fe1bf93f32b 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -42,6 +42,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private double defaultTermwiseLimit = 1.0; private String jvmGCOptions = null; private String sequencerType = "LATENCY"; + private boolean firstTimeDeployment = false; private String responseSequencerType = "ADAPTIVE"; private int responseNumThreads = 2; private Optional<EndpointCertificateSecrets> endpointCertificateSecrets = Optional.empty(); @@ -54,11 +55,13 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private boolean enableFeedBlockInDistributor = true; private boolean useExternalRankExpression = false; private int clusterControllerMaxHeapSizeInMb = 128; - private int metricsProxyMaxHeapSizeInMb = 256; private int maxActivationInhibitedOutOfSyncGroups = 0; private List<TenantSecretStore> tenantSecretStores = Collections.emptyList(); private String jvmOmitStackTraceInFastThrowOption; private int numDistributorStripes = 0; + private int maxConcurrentMergesPerNode = 16; + private int maxMergeQueueSize = 1024; + private int largeRankExpressionLimit = 0x10000; private boolean allowDisableMtls = true; private List<X509Certificate> operatorCertificates = Collections.emptyList(); @@ -75,7 +78,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public String jvmGCOptions(Optional<ClusterSpec.Type> clusterType) { return jvmGCOptions; } @Override public String feedSequencerType() { return sequencerType; } @Override public boolean isBootstrap() { return false; } - @Override public boolean isFirstTimeDeployment() { return false; } + @Override public boolean isFirstTimeDeployment() { return firstTimeDeployment; } @Override public boolean useDedicatedNodeForLogserver() { return useDedicatedNodeForLogserver; } @Override public Optional<EndpointCertificateSecrets> endpointCertificateSecrets() { return endpointCertificateSecrets; } @Override public double defaultTermwiseLimit() { return defaultTermwiseLimit; } @@ -93,7 +96,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public double feedConcurrency() { return feedConcurrency; } @Override public boolean enableFeedBlockInDistributor() { return enableFeedBlockInDistributor; } @Override public int clusterControllerMaxHeapSizeInMb() { return clusterControllerMaxHeapSizeInMb; } - @Override public int metricsProxyMaxHeapSizeInMb(ClusterSpec.Type type) { return metricsProxyMaxHeapSizeInMb; } @Override public int maxActivationInhibitedOutOfSyncGroups() { return maxActivationInhibitedOutOfSyncGroups; } @Override public List<TenantSecretStore> tenantSecretStores() { return tenantSecretStores; } @Override public String jvmOmitStackTraceInFastThrowOption(ClusterSpec.Type type) { return jvmOmitStackTraceInFastThrowOption; } @@ -102,11 +104,18 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public List<X509Certificate> operatorCertificates() { return operatorCertificates; } @Override public boolean useExternalRankExpressions() { return useExternalRankExpression; } @Override public boolean distributeExternalRankExpressions() { return useExternalRankExpression; } + @Override public int largeRankExpressionLimit() { return largeRankExpressionLimit; } + @Override public int maxConcurrentMergesPerNode() { return maxConcurrentMergesPerNode; } + @Override public int maxMergeQueueSize() { return maxMergeQueueSize; } public TestProperties useExternalRankExpression(boolean value) { useExternalRankExpression = value; return this; } + public TestProperties largeRankExpressionLimit(int value) { + largeRankExpressionLimit = value; + return this; + } public TestProperties setFeedConcurrency(double feedConcurrency) { this.feedConcurrency = feedConcurrency; return this; @@ -129,11 +138,24 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea responseSequencerType = type; return this; } + public TestProperties setFirstTimeDeployment(boolean firstTimeDeployment) { + this.firstTimeDeployment = firstTimeDeployment; + return this; + } public TestProperties setResponseNumThreads(int numThreads) { responseNumThreads = numThreads; return this; } + public TestProperties setMaxConcurrentMergesPerNode(int maxConcurrentMergesPerNode) { + this.maxConcurrentMergesPerNode = maxConcurrentMergesPerNode; + return this; + } + public TestProperties setMaxMergeQueueSize(int maxMergeQueueSize) { + this.maxMergeQueueSize = maxMergeQueueSize; + return this; + } + public TestProperties setDefaultTermwiseLimit(double limit) { defaultTermwiseLimit = limit; return this; @@ -209,11 +231,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } - public TestProperties metricsProxyMaxHeapSizeInMb(int heapSize) { - metricsProxyMaxHeapSizeInMb = heapSize; - return this; - } - public TestProperties maxActivationInhibitedOutOfSyncGroups(int nGroups) { maxActivationInhibitedOutOfSyncGroups = nGroups; return this; diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java index e9fe0824f30..411a37bb70a 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java @@ -115,7 +115,7 @@ public class MockApplicationPackage implements ApplicationPackage { } @Override - public List<NamedReader> getSearchDefinitions() { + public List<NamedReader> getSchemas() { ArrayList<NamedReader> readers = new ArrayList<>(); SearchBuilder searchBuilder = new SearchBuilder(this, new BaseDeployLogger(), @@ -134,11 +134,6 @@ public class MockApplicationPackage implements ApplicationPackage { } @Override - public List<NamedReader> searchDefinitionContents() { - return new ArrayList<>(); - } - - @Override public Map<ConfigDefinitionKey, UnparsedConfigDefinition> getAllExistingConfigDefs() { return Collections.emptyMap(); } diff --git a/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java b/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java index 38d831a0b28..da338ad3107 100644 --- a/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java +++ b/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java @@ -3,7 +3,6 @@ package com.yahoo.documentmodel; import com.yahoo.document.DataType; import com.yahoo.document.Document; -import com.yahoo.document.DocumentId; import com.yahoo.document.Field; import com.yahoo.document.StructDataType; import com.yahoo.document.StructuredDataType; @@ -32,34 +31,6 @@ import static java.util.Collections.emptySet; */ public final class NewDocumentType extends StructuredDataType implements DataTypeCollection { - public static final class Name { - - private final String name; - private final int id; - - public Name(String name) { - this(name.hashCode(), name); - } - - public Name(int id, String name) { - this.id = id; - this.name = name; - } - - public String toString() { return name; } - - public final String getName() { return name; } - - public final int getId() { return id; } - - public int hashCode() { return name.hashCode(); } - - public boolean equals(Object other) { - if ( ! (other instanceof Name)) return false; - return name.equals(((Name)other).getName()); - } - } - private final Name name; private final DataTypeRepo dataTypes = new DataTypeRepo(); private final Map<Integer, NewDocumentType> inherits = new LinkedHashMap<>(); @@ -139,7 +110,7 @@ public final class NewDocumentType extends StructuredDataType implements DataTyp } @Override - public Class getValueClass() { + public Class<Document> getValueClass() { return Document.class; } @@ -148,7 +119,8 @@ public final class NewDocumentType extends StructuredDataType implements DataTyp if (!(value instanceof Document)) { return false; } - /** Temporary disabled due to clash with document and covariant return type + /* + Temporary disabled due to clash with document and covariant return type Document doc = (Document) value; if (((NewDocumentType) doc.getDataType()).inherits(this)) { //the value is of this type; or the supertype of the value is of this type, etc.... @@ -162,28 +134,31 @@ public final class NewDocumentType extends StructuredDataType implements DataTyp for (Field f : getFields()) { Field inhF = inherited.getField(f.getName()); if (inhF != null && !inhF.equals(f)) { - throw new IllegalArgumentException("Inherited document '" + inherited.toString() + "' already contains field '" + - inhF.getName() + "'. Can not override with '" + f.getName() + "'."); + throw new IllegalArgumentException("Inherited document '" + inherited + "' already contains field '" + + inhF.getName() + "'. Can not override with '" + f.getName() + "'."); } } for (Field f : inherited.getAllFields()) { for (NewDocumentType side : inherits.values()) { Field sideF = side.getField(f.getName()); if (sideF != null && !sideF.equals(f)) { - throw new IllegalArgumentException("Inherited document '" + side.toString() + "' already contains field '" + - sideF.getName() + "'. Document '" + inherited.toString() + "' also defines field '" + f.getName() + - "'.Multiple inheritance must be disjunctive."); + throw new IllegalArgumentException("Inherited document '" + side + "' already contains field '" + + sideF.getName() + "'. Document '" + inherited + + "' also defines field '" + f.getName() + + "'.Multiple inheritance must be disjunctive."); } } } return true; } + public void inherit(NewDocumentType inherited) { if ( ! inherits.containsKey(inherited.getId())) { verifyInheritance(inherited); inherits.put(inherited.getId(), inherited); } } + public boolean inherits(NewDocumentType superType) { if (getId() == superType.getId()) return true; for (NewDocumentType type : inherits.values()) { @@ -243,7 +218,7 @@ public final class NewDocumentType extends StructuredDataType implements DataTyp @Override public Document createFieldValue() { - return new Document(null, (DocumentId)null); + throw new RuntimeException("Cannot create an instance of " + this); } @Override @@ -375,4 +350,36 @@ public final class NewDocumentType extends StructuredDataType implements DataTyp return importedFieldNames; } + public static final class Name { + + private final String name; + private final int id; + + public Name(String name) { + this(name.hashCode(), name); + } + + public Name(int id, String name) { + this.id = id; + this.name = name; + } + + @Override + public String toString() { return name; } + + public final String getName() { return name; } + + public final int getId() { return id; } + + @Override + public int hashCode() { return name.hashCode(); } + + @Override + public boolean equals(Object other) { + if ( ! (other instanceof Name)) return false; + return name.equals(((Name)other).getName()); + } + + } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java b/config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java index 77ce2dd41b5..ffa9cbe9ba5 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java @@ -5,15 +5,17 @@ import com.yahoo.path.Path; import com.yahoo.vespa.model.AbstractService; import com.yahoo.vespa.model.utils.FileSender; +import java.nio.ByteBuffer; import java.util.Collection; import java.util.Objects; public class DistributableResource { - public enum PathType { FILE, URI }; + public enum PathType { FILE, URI, BLOB }; /** The search definition-unique name of this constant */ private final String name; - private String path = null; + private final ByteBuffer blob; + private String path; private String fileReference = ""; private PathType pathType = PathType.FILE; @@ -22,11 +24,20 @@ public class DistributableResource { } public DistributableResource(String name) { - this(name, null); + this.name = name; + blob = null; } public DistributableResource(String name, String path) { this.name = name; this.path = path; + blob = null; + } + public DistributableResource(String name, ByteBuffer blob) { + Objects.requireNonNull(name, "Blob name cannot be null"); + Objects.requireNonNull(blob, "Blob cannot be null"); + this.name = name; + this.blob = blob; + pathType = PathType.BLOB; } public void setFileName(String fileName) { @@ -41,16 +52,24 @@ public class DistributableResource { this.pathType = PathType.URI; } - protected void setFileReference(String fileReference) { this.fileReference = fileReference; } /** Initiate sending of this constant to some services over file distribution */ public void sendTo(Collection<? extends AbstractService> services) { - FileReference reference = (pathType == PathType.FILE) - ? FileSender.sendFileToServices(path, services) - : FileSender.sendUriToServices(path, services); - this.fileReference = reference.value(); + fileReference = sendToServices(services).value(); + } + private FileReference sendToServices(Collection<? extends AbstractService> services) { + switch (pathType) { + case FILE: + return FileSender.sendFileToServices(path, services); + case URI: + return FileSender.sendUriToServices(path, services); + case BLOB: + return FileSender.sendBlobToServices(blob, services); + } + throw new IllegalArgumentException("Unknown path type " + pathType); } public String getName() { return name; } + public ByteBuffer getBlob() { return blob; } public String getFileName() { return path; } public Path getFilePath() { return Path.fromString(path); } public String getUri() { return path; } @@ -63,10 +82,8 @@ public class DistributableResource { public String toString() { StringBuilder b = new StringBuilder(); - b.append("resource '").append(name) - .append(pathType == PathType.FILE ? "' from file '" : " from uri ").append(path) - .append("' with ref '").append(fileReference) - .append("'"); + b.append("resource '").append(name).append(" of type '").append(pathType) + .append("' with ref '").append(fileReference).append("'"); return b.toString(); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java index fed35382b21..9b752c4179f 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java @@ -209,17 +209,13 @@ public class DocumentModelBuilder { private static DataType resolveTemporariesRecurse(DataType type, DataTypeCollection repo, Collection<NewDocumentType> docs) { if (type instanceof TemporaryStructuredDataType) { - NewDocumentType docType = getDocumentType(docs, type.getId()); - if (docType != null) { - type = docType; - return type; - } - DataType real = repo.getDataType(type.getId()); - if (real == null) { - throw new NullPointerException("Can not find type '" + type.toString() + "', impossible."); - } - type = real; - } else if (type instanceof StructDataType) { + DataType struct = repo.getDataType(type.getId()); + if (struct != null) + type = struct; + else + type = getDocumentType(docs, type.getId()); + } + else if (type instanceof StructDataType) { StructDataType dt = (StructDataType) type; for (com.yahoo.document.Field field : dt.getFields()) { if (field.getDataType() != type) { @@ -227,14 +223,17 @@ public class DocumentModelBuilder { field.setDataType(resolveTemporariesRecurse(field.getDataType(), repo, docs)); } } - } else if (type instanceof MapDataType) { + } + else if (type instanceof MapDataType) { MapDataType t = (MapDataType) type; t.setKeyType(resolveTemporariesRecurse(t.getKeyType(), repo, docs)); t.setValueType(resolveTemporariesRecurse(t.getValueType(), repo, docs)); - } else if (type instanceof CollectionDataType) { + } + else if (type instanceof CollectionDataType) { CollectionDataType t = (CollectionDataType) type; t.setNestedType(resolveTemporariesRecurse(t.getNestedType(), repo, docs)); - } else if (type instanceof ReferenceDataType) { + } + else if (type instanceof ReferenceDataType) { ReferenceDataType t = (ReferenceDataType) type; if (t.getTargetType() instanceof TemporaryStructuredDataType) { DataType targetType = resolveTemporariesRecurse(t.getTargetType(), repo, docs); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/ImmutableSearch.java b/config-model/src/main/java/com/yahoo/searchdefinition/ImmutableSearch.java index 6b40289e17d..24bc081aded 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/ImmutableSearch.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/ImmutableSearch.java @@ -32,7 +32,7 @@ public interface ImmutableSearch { DeployLogger getDeployLogger(); ModelContext.Properties getDeployProperties(); RankingConstants rankingConstants(); - RankExpressionFiles rankExpressionFiles(); + LargeRankExpressions rankExpressionFiles(); OnnxModels onnxModels(); Stream<ImmutableSDField> allImportedFields(); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/LargeRankExpressions.java b/config-model/src/main/java/com/yahoo/searchdefinition/LargeRankExpressions.java new file mode 100644 index 00000000000..6fadcb39d11 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/LargeRankExpressions.java @@ -0,0 +1,38 @@ +package com.yahoo.searchdefinition; + +import com.yahoo.vespa.model.AbstractService; + +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +public class LargeRankExpressions { + private final Map<String, RankExpressionBody> expressions = new HashMap<>(); + + public void add(RankExpressionBody expression) { + expression.validate(); + String name = expression.getName(); + if (expressions.containsKey(name)) { + throw new IllegalArgumentException("Rank expression '" + name + + "' defined twice. Previous blob with " + expressions.get(name).getBlob().remaining() + + " bytes, while current has " + expression.getBlob().remaining() + " bytes"); + } + expressions.put(name, expression); + } + + /** Returns the ranking constant with the given name, or null if not present */ + public RankExpressionBody get(String name) { + return expressions.get(name); + } + + /** Returns a read-only map of the ranking constants in this indexed by name */ + public Map<String, RankExpressionBody> asMap() { + return Collections.unmodifiableMap(expressions); + } + + /** Initiate sending of these constants to some services over file distribution */ + public void sendTo(Collection<? extends AbstractService> services) { + expressions.values().forEach(constant -> constant.sendTo(services)); + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankExpressionBody.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankExpressionBody.java new file mode 100644 index 00000000000..8c6830de815 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankExpressionBody.java @@ -0,0 +1,10 @@ +package com.yahoo.searchdefinition; + +import java.nio.ByteBuffer; + +public class RankExpressionBody extends DistributableResource { + + public RankExpressionBody(String name, ByteBuffer body) { + super(name, body); + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankExpressionFile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankExpressionFile.java deleted file mode 100644 index 56385efeb0b..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankExpressionFile.java +++ /dev/null @@ -1,35 +0,0 @@ -package com.yahoo.searchdefinition; - -import com.yahoo.config.application.api.ApplicationPackage; -import com.yahoo.vespa.model.AbstractService; -import com.yahoo.vespa.model.utils.FileSender; - -import java.util.Collection; - -public class RankExpressionFile extends DistributableResource { - - public RankExpressionFile(String name, String path) { - super(name, path); - validate(); - } - - @Override - public void sendTo(Collection<? extends AbstractService> services) { - /* - * TODO This is a very dirty hack due to using both SEARCH_DEFINITIONS_DIR and SCHEMA_DIR - * and doing so inconsistently, combined with using both fields from application package on disk and in zookeeper. - * The mess is spread out nicely, but ZookeeperClient, and writeSearchDefinitions and ZkApplicationPackage and FilesApplicationPackage - * should be consolidated - */ - try { - setFileReference(FileSender.sendFileToServices(ApplicationPackage.SCHEMAS_DIR + "/" + getFileName(), services).value()); - } catch (IllegalArgumentException e1) { - try { - setFileReference(FileSender.sendFileToServices(ApplicationPackage.SEARCH_DEFINITIONS_DIR + "/" + getFileName(), services).value()); - } catch (IllegalArgumentException e2) { - throw new IllegalArgumentException("Failed to find expression file '" + getFileName() + "' in '" - + ApplicationPackage.SEARCH_DEFINITIONS_DIR + "' or '" + ApplicationPackage.SCHEMAS_DIR + "'.", e2); - } - } - } -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankExpressionFiles.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankExpressionFiles.java deleted file mode 100644 index 34ad912dd00..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankExpressionFiles.java +++ /dev/null @@ -1,47 +0,0 @@ -package com.yahoo.searchdefinition; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.vespa.model.AbstractService; - -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.logging.Level; - -public class RankExpressionFiles { - private final Map<String, RankExpressionFile> expressions = new HashMap<>(); - - //TODO Deploy logger should not be necessary, as redefinition is illegal, but legacy prevents enforcement starting now. - public void add(RankExpressionFile expression, DeployLogger deployLogger) { - expression.validate(); - String name = expression.getName(); - if (expressions.containsKey(name)) { - if ( expressions.get(name).getFileName().equals(expression.getFileName()) ) { - //TODO Throw instead, No later than Vespa 8 - deployLogger.logApplicationPackage(Level.WARNING, "Rank expression file '" + name + - "' defined twice with identical expression (illegal and will be enforced soon) '" + expression.getFileName() + "'."); - } else { - throw new IllegalArgumentException("Rank expression file '" + name + - "' defined twice (illegal but not enforced), but redefinition is not matching (illegal and enforced), " + - "previous = '" + expressions.get(name).getFileName() + "', new = '" + expression.getFileName() + "'."); - } - } - expressions.put(name, expression); - } - - /** Returns the ranking constant with the given name, or null if not present */ - public RankExpressionFile get(String name) { - return expressions.get(name); - } - - /** Returns a read-only map of the ranking constants in this indexed by name */ - public Map<String, RankExpressionFile> asMap() { - return Collections.unmodifiableMap(expressions); - } - - /** Initiate sending of these constants to some services over file distribution */ - public void sendTo(Collection<? extends AbstractService> services) { - expressions.values().forEach(constant -> constant.sendTo(services)); - } -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/Search.java b/config-model/src/main/java/com/yahoo/searchdefinition/Search.java index f11afef0eb2..9ce1b8bb330 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/Search.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/Search.java @@ -83,7 +83,7 @@ public class Search implements ImmutableSearch { private final Map<String, DocumentSummary> summaries = new LinkedHashMap<>(); /** External rank expression files of this */ - private final RankExpressionFiles rankExpressionFiles = new RankExpressionFiles(); + private final LargeRankExpressions largeRankExpressions = new LargeRankExpressions(); /** Ranking constants of this */ private final RankingConstants rankingConstants = new RankingConstants(); @@ -98,7 +98,7 @@ public class Search implements ImmutableSearch { private final DeployLogger deployLogger; private final ModelContext.Properties properties; - /** Testin only */ + /** Testing only */ public Search(String name) { this(name, null, new BaseDeployLogger(), new TestProperties()); } @@ -188,7 +188,7 @@ public class Search implements ImmutableSearch { } @Override - public RankExpressionFiles rankExpressionFiles() { return rankExpressionFiles; } + public LargeRankExpressions rankExpressionFiles() { return largeRankExpressions; } @Override public RankingConstants rankingConstants() { return rankingConstants; } @@ -198,7 +198,7 @@ public class Search implements ImmutableSearch { public void sendTo(Collection<? extends AbstractService> services) { rankingConstants.sendTo(services); - rankExpressionFiles.sendTo(services); + largeRankExpressions.sendTo(services); onnxModels.sendTo(services); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java index d414b9ed79f..7c533cce006 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java @@ -6,8 +6,7 @@ import com.yahoo.config.model.api.ModelContext; import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.OnnxModel; import com.yahoo.searchdefinition.OnnxModels; -import com.yahoo.searchdefinition.RankExpressionFile; -import com.yahoo.searchdefinition.RankExpressionFiles; +import com.yahoo.searchdefinition.LargeRankExpressions; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.RankingConstant; import com.yahoo.searchdefinition.RankingConstants; @@ -34,14 +33,14 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ private final Map<String, RawRankProfile> rankProfiles = new java.util.LinkedHashMap<>(); private final RankingConstants rankingConstants; - private final RankExpressionFiles rankExpressionFiles; + private final LargeRankExpressions largeRankExpressions; private final OnnxModels onnxModels; public static RankProfileList empty = new RankProfileList(); private RankProfileList() { rankingConstants = new RankingConstants(); - rankExpressionFiles = new RankExpressionFiles(); + largeRankExpressions = new LargeRankExpressions(); onnxModels = new OnnxModels(); } @@ -53,7 +52,7 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ */ public RankProfileList(Search search, RankingConstants rankingConstants, - RankExpressionFiles rankExpressionFiles, + LargeRankExpressions largeRankExpressions, AttributeFields attributeFields, RankProfileRegistry rankProfileRegistry, QueryProfileRegistry queryProfiles, @@ -61,7 +60,7 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ ModelContext.Properties deployProperties) { setName(search == null ? "default" : search.getName()); this.rankingConstants = rankingConstants; - this.rankExpressionFiles = rankExpressionFiles; + this.largeRankExpressions = largeRankExpressions; onnxModels = search == null ? new OnnxModels() : search.onnxModels(); // as ONNX models come from parsing rank expressions deriveRankProfiles(rankProfileRegistry, queryProfiles, importedModels, search, attributeFields, deployProperties); } @@ -74,7 +73,8 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ ModelContext.Properties deployProperties) { if (search != null) { // profiles belonging to a search have a default profile RawRankProfile defaultProfile = new RawRankProfile(rankProfileRegistry.get(search, "default"), - queryProfiles, importedModels, attributeFields, deployProperties); + largeRankExpressions, queryProfiles, importedModels, + attributeFields, deployProperties); rankProfiles.put(defaultProfile.getName(), defaultProfile); } @@ -84,7 +84,8 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ this.onnxModels.add(rank.onnxModels()); } - RawRankProfile rawRank = new RawRankProfile(rank, queryProfiles, importedModels, attributeFields, deployProperties); + RawRankProfile rawRank = new RawRankProfile(rank, largeRankExpressions, queryProfiles, importedModels, + attributeFields, deployProperties); rankProfiles.put(rawRank.getName(), rawRank); } } @@ -100,7 +101,7 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ public void sendTo(Collection<? extends AbstractService> services) { rankingConstants.sendTo(services); - rankExpressionFiles.sendTo(services); + largeRankExpressions.sendTo(services); onnxModels.sendTo(services); } @@ -115,7 +116,7 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ } public void getConfig(RankingExpressionsConfig.Builder builder) { - rankExpressionFiles.asMap().values().forEach((expr) -> builder.expression.add(new RankingExpressionsConfig.Expression.Builder().name(expr.getName()).fileref(expr.getFileReference()))); + largeRankExpressions.asMap().values().forEach((expr) -> builder.expression.add(new RankingExpressionsConfig.Expression.Builder().name(expr.getName()).fileref(expr.getFileReference()))); } public void getConfig(RankingConstantsConfig.Builder builder) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java index 6b589a22de5..97d695cead9 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java @@ -9,6 +9,8 @@ import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.OnnxModel; +import com.yahoo.searchdefinition.LargeRankExpressions; +import com.yahoo.searchdefinition.RankExpressionBody; import com.yahoo.searchdefinition.document.RankType; import com.yahoo.searchdefinition.RankProfile; import com.yahoo.searchdefinition.expressiontransforms.OnnxModelTransformer; @@ -20,6 +22,7 @@ import com.yahoo.searchlib.rankingexpression.rule.SerializationContext; import com.yahoo.tensor.TensorType; import com.yahoo.vespa.config.search.RankProfilesConfig; +import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.HashSet; @@ -27,6 +30,7 @@ import java.util.Iterator; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; +import java.util.ListIterator; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -55,19 +59,20 @@ public class RawRankProfile implements RankProfilesConfig.Producer { /** * Creates a raw rank profile from the given rank profile */ - public RawRankProfile(RankProfile rankProfile, QueryProfileRegistry queryProfiles, ImportedMlModels importedModels, + public RawRankProfile(RankProfile rankProfile, LargeRankExpressions largeExpressions, + QueryProfileRegistry queryProfiles, ImportedMlModels importedModels, AttributeFields attributeFields, ModelContext.Properties deployProperties) { this.name = rankProfile.getName(); compressedProperties = compress(new Deriver(rankProfile.compile(queryProfiles, importedModels), - attributeFields, deployProperties).derive()); + attributeFields, deployProperties).derive(largeExpressions)); } /** * Only for testing */ - public RawRankProfile(RankProfile rankProfile, QueryProfileRegistry queryProfiles, - ImportedMlModels importedModels, AttributeFields attributeFields) { - this(rankProfile, queryProfiles, importedModels, attributeFields, new TestProperties()); + public RawRankProfile(RankProfile rankProfile, QueryProfileRegistry queryProfiles, ImportedMlModels importedModels, + AttributeFields attributeFields) { + this(rankProfile, new LargeRankExpressions(), queryProfiles, importedModels, attributeFields, new TestProperties()); } private Compressor.Compression compress(List<Pair<String, String>> properties) { @@ -142,6 +147,9 @@ public class RawRankProfile implements RankProfilesConfig.Producer { private final int numSearchPartitions; private final double termwiseLimit; private final double rankScoreDropLimit; + private final int largeRankExpressionLimit; + private final boolean distributeLargeRankExpressions; + private final boolean useDistributedRankExpressions; /** * The rank type definitions used to derive settings for the native rank features @@ -150,6 +158,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { private final Map<String, String> attributeTypes; private final Map<String, String> queryFeatureTypes; private final Set<String> filterFields = new java.util.LinkedHashSet<>(); + private final String rankprofileName; private RankingExpression firstPhaseRanking; private RankingExpression secondPhaseRanking; @@ -159,6 +168,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { */ Deriver(RankProfile compiled, AttributeFields attributeFields, ModelContext.Properties deployProperties) { + rankprofileName = compiled.getName(); attributeTypes = compiled.getAttributeTypes(); queryFeatureTypes = compiled.getQueryFeatureTypes(); firstPhaseRanking = compiled.getFirstPhaseRanking(); @@ -174,6 +184,9 @@ public class RawRankProfile implements RankProfilesConfig.Producer { keepRankCount = compiled.getKeepRankCount(); rankScoreDropLimit = compiled.getRankScoreDropLimit(); ignoreDefaultRankFeatures = compiled.getIgnoreDefaultRankFeatures(); + largeRankExpressionLimit = deployProperties.featureFlags().largeRankExpressionLimit(); + distributeLargeRankExpressions = deployProperties.featureFlags().distributeExternalRankExpressions(); + useDistributedRankExpressions = deployProperties.featureFlags().useExternalRankExpressions(); rankProperties = new ArrayList<>(compiled.getRankProperties()); Map<String, RankProfile.RankingExpressionFunction> functions = compiled.getFunctions(); @@ -319,10 +332,10 @@ public class RawRankProfile implements RankProfilesConfig.Producer { } /** Derives the properties this produces */ - public List<Pair<String, String>> derive() { + public List<Pair<String, String>> derive(LargeRankExpressions largeRankExpressions) { List<Pair<String, String>> properties = new ArrayList<>(); for (RankProfile.RankProperty property : rankProperties) { - if (("rankingExpression(" + RankProfile.FIRST_PHASE + ").rankingScript").equals(property.getName())) { + if (RankingExpression.propertyName(RankProfile.FIRST_PHASE).equals(property.getName())) { // Could have been set by function expansion. Set expressions, then skip this property. try { firstPhaseRanking = new RankingExpression(property.getValue()); @@ -330,7 +343,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { throw new IllegalArgumentException("Could not parse first phase expression", e); } } - else if (("rankingExpression(" + RankProfile.SECOND_PHASE + ").rankingScript").equals(property.getName())) { + else if (RankingExpression.propertyName(RankProfile.SECOND_PHASE).equals(property.getName())) { try { secondPhaseRanking = new RankingExpression(property.getValue()); } catch (ParseException e) { @@ -419,7 +432,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { properties.add(new Pair<>("vespa.rank." + phase, expression.getRoot().toString())); } else { properties.add(new Pair<>("vespa.rank." + phase, "rankingExpression(" + name + ")")); - properties.add(new Pair<>("rankingExpression(" + name + ").rankingScript", expression.getRoot().toString())); + properties.add(new Pair<>(RankingExpression.propertyName(name), expression.getRoot().toString())); } return properties; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java b/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java index 8dea1b65079..ea0452a6c49 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java @@ -8,6 +8,7 @@ import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.vespa.defaults.Defaults; +import java.nio.ByteBuffer; import java.util.Collection; import java.util.LinkedHashMap; import java.util.LinkedHashSet; @@ -470,6 +471,10 @@ public abstract class AbstractService extends AbstractConfigProducer<AbstractCon return getRoot().getFileDistributor().sendUriToHost(uri, getHost().getHost()); } + public FileReference sendBlob(ByteBuffer blob) { + return getRoot().getFileDistributor().sendBlobToHost(blob, getHost().getHost()); + } + /** The service HTTP port for health status */ public int getHealthPort() { return -1;} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/ConfigSentinel.java b/config-model/src/main/java/com/yahoo/vespa/model/ConfigSentinel.java index d15db6b4a55..dd35787571e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/ConfigSentinel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/ConfigSentinel.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model; +import com.yahoo.config.model.api.ModelContext; import com.yahoo.cloud.config.SentinelConfig; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Zone; @@ -18,16 +19,20 @@ public class ConfigSentinel extends AbstractService implements SentinelConfig.Pr private final ApplicationId applicationId; private final Zone zone; + private final boolean requireConnectivityCheck; /** * Constructs a new ConfigSentinel for the given host. * * @param host Physical host on which to run. */ - public ConfigSentinel(Host host, ApplicationId applicationId, Zone zone) { + public ConfigSentinel(Host host, ApplicationId applicationId, Zone zone, + ModelContext.FeatureFlags featureFlags) + { super(host, "sentinel"); this.applicationId = applicationId; this.zone = zone; + this.requireConnectivityCheck = featureFlags.requireConnectivityCheck(); portsMeta.on(0).tag("rpc").tag("admin"); portsMeta.on(1).tag("telnet").tag("interactive").tag("http").tag("state"); setProp("clustertype", "hosts"); @@ -75,6 +80,19 @@ public class ConfigSentinel extends AbstractService implements SentinelConfig.Pr builder.service(getServiceConfig(s)); } } + builder.connectivity(getConnectivityConfig(requireConnectivityCheck)); + } + + private SentinelConfig.Connectivity.Builder getConnectivityConfig(boolean enable) { + var builder = new SentinelConfig.Connectivity.Builder(); + if (enable) { + builder.minOkPercent(50); + builder.maxBadCount(1); + } else { + builder.minOkPercent(0); + builder.maxBadCount(Integer.MAX_VALUE); + } + return builder; } private SentinelConfig.Application.Builder getApplicationConfig() { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java b/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java index 09bbd446803..53f42866d8d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java @@ -47,16 +47,15 @@ public class HostSystem extends AbstractConfigProducer<Host> { void checkName(String hostname) { // Give a warning if the host does not exist try { - @SuppressWarnings("unused") - Object ignore = java.net.InetAddress.getByName(hostname); + var inetAddr = java.net.InetAddress.getByName(hostname); + String canonical = inetAddr.getCanonicalHostName(); + if (! hostname.equals(canonical)) { + deployLogger.logApplicationPackage(Level.WARNING, "Host named '" + hostname + "' may not receive any config " + + "since it differs from its canonical hostname '" + canonical + "' (check DNS and /etc/hosts)."); + } } catch (UnknownHostException e) { deployLogger.logApplicationPackage(Level.WARNING, "Unable to lookup IP address of host: " + hostname); } - if (! hostname.contains(".")) { - deployLogger.logApplicationPackage(Level.WARNING, "Host named '" + hostname + "' may not receive any config " + - "since it is not a canonical hostname. " + - "Disregard this warning when testing in a Docker container."); - } } /** diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java index ab00e9d295f..d20247b79fc 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java @@ -33,7 +33,7 @@ import com.yahoo.container.QrConfig; import com.yahoo.path.Path; import com.yahoo.searchdefinition.OnnxModel; import com.yahoo.searchdefinition.OnnxModels; -import com.yahoo.searchdefinition.RankExpressionFiles; +import com.yahoo.searchdefinition.LargeRankExpressions; import com.yahoo.searchdefinition.RankProfile; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.RankingConstants; @@ -131,7 +131,7 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri private final RankingConstants rankingConstants = new RankingConstants(); /** External rank expression files of this */ - private final RankExpressionFiles rankExpressionFiles = new RankExpressionFiles(); + private final LargeRankExpressions largeRankExpressions = new LargeRankExpressions(); /** The validation overrides of this. This is never null. */ private final ValidationOverrides validationOverrides; @@ -187,7 +187,7 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri deployState.rankProfileRegistry(), deployState.getQueryProfiles()); rankProfileList = new RankProfileList(null, // null search -> global rankingConstants, - rankExpressionFiles, + largeRankExpressions, AttributeFields.empty, deployState.rankProfileRegistry(), deployState.getQueryProfiles().getRegistry(), @@ -266,7 +266,7 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri /** Returns the global ranking constants of this */ public RankingConstants rankingConstants() { return rankingConstants; } - public RankExpressionFiles rankExpressionFiles() { return rankExpressionFiles; } + public LargeRankExpressions rankExpressionFiles() { return largeRankExpressions; } /** Creates a mutable model with no services instantiated */ public static VespaModel createIncomplete(DeployState deployState) throws IOException, SAXException { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java index d6673cd49e9..b576d1cb5d2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java @@ -6,6 +6,7 @@ import com.google.inject.Inject; import com.yahoo.component.Version; import com.yahoo.component.provider.ComponentRegistry; import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.model.ConfigModelRegistry; import com.yahoo.config.model.MapConfigModelRegistry; import com.yahoo.config.model.NullConfigModelRegistry; @@ -23,6 +24,7 @@ import com.yahoo.config.provision.TransientException; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.config.VespaVersion; import com.yahoo.vespa.model.application.validation.Validation; +import com.yahoo.yolean.Exceptions; import org.xml.sax.SAXException; import java.io.IOException; @@ -31,6 +33,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.logging.Level; import java.util.logging.Logger; /** @@ -170,6 +173,13 @@ public class VespaModelFactory implements ModelFactory { private List<ConfigChangeAction> validateModel(VespaModel model, DeployState deployState, ValidationParameters validationParameters) { try { return Validation.validate(model, validationParameters, deployState); + } catch (ValidationOverrides.ValidationException e) { + if (deployState.isHosted() && zone.environment().isManuallyDeployed()) + deployState.getDeployLogger().logApplicationPackage(Level.WARNING, + "Auto-overriding validation which would be disallowed in production: " + + Exceptions.toMessageString(e)); + else + rethrowUnlessIgnoreErrors(e, validationParameters.ignoreValidationErrors()); } catch (IllegalArgumentException | TransientException e) { rethrowUnlessIgnoreErrors(e, validationParameters.ignoreValidationErrors()); } catch (Exception e) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java index e080ce43730..a2a6ada9093 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin; +import com.yahoo.config.model.api.ModelContext; import com.yahoo.cloud.config.SlobroksConfig; import com.yahoo.cloud.config.ZookeepersConfig; import com.yahoo.cloud.config.log.LogdConfig; @@ -242,7 +243,8 @@ public class Admin extends AbstractConfigProducer<Admin> implements Serializable } private void addCommonServices(HostResource host, DeployState deployState) { - addConfigSentinel(deployState.getDeployLogger(), host, deployState.getProperties().applicationId(), deployState.zone()); + addConfigSentinel(deployState.getDeployLogger(), host, deployState.getProperties().applicationId(), deployState.zone(), + deployState.featureFlags()); addLogd(deployState.getDeployLogger(), host); addConfigProxy(deployState.getDeployLogger(), host); addFileDistribution(host); @@ -262,8 +264,10 @@ public class Admin extends AbstractConfigProducer<Admin> implements Serializable } } - private void addConfigSentinel(DeployLogger deployLogger, HostResource host, ApplicationId applicationId, Zone zone) { - ConfigSentinel configSentinel = new ConfigSentinel(host.getHost(), applicationId, zone); + private void addConfigSentinel(DeployLogger deployLogger, HostResource host, + ApplicationId applicationId, Zone zone, ModelContext.FeatureFlags featureFlags) + { + ConfigSentinel configSentinel = new ConfigSentinel(host.getHost(), applicationId, zone, featureFlags); addAndInitializeService(deployLogger, host, configSentinel); host.getHost().setConfigSentinel(configSentinel); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java index 234892c5cc3..9dec27e17fe 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java @@ -45,16 +45,13 @@ public class MetricsProxyContainer extends Container implements final boolean isHostedVespa; private final Optional<ClusterMembership> clusterMembership; - private final ModelContext.FeatureFlags featureFlags; private final MetricsProxyContainerCluster cluster; private final String jvmGCOptions; - public MetricsProxyContainer(MetricsProxyContainerCluster cluster, HostResource host, int index, DeployState deployState) { super(cluster, host.getHostname(), index, deployState); this.isHostedVespa = deployState.isHosted(); this.clusterMembership = host.spec().membership(); - this.featureFlags = deployState.featureFlags(); this.cluster = cluster; this.jvmGCOptions = deployState.getProperties().jvmGCOptions(clusterMembership.map(membership -> membership.cluster().type())); setProp("clustertype", "admin"); @@ -157,7 +154,9 @@ public class MetricsProxyContainer extends Container implements cluster.getConfig(builder); if (clusterMembership.isPresent()) { - int maxHeapSize = featureFlags.metricsProxyMaxHeapSizeInMb(clusterMembership.get().cluster().type()); + int maxHeapSize = clusterMembership.get().cluster().type() == ClusterSpec.Type.admin + ? 128 + : 256; builder.jvm .gcopts(jvmGCOptions) .heapsize(maxHeapSize); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index 034bf772ffc..114a3e380ef 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -256,8 +256,11 @@ public class VespaMetricSet { metrics.add(new Metric("cluster-controller.node-event.count")); metrics.add(new Metric("cluster-controller.resource_usage.nodes_above_limit.last")); + metrics.add(new Metric("cluster-controller.resource_usage.nodes_above_limit.max")); metrics.add(new Metric("cluster-controller.resource_usage.max_memory_utilization.last")); + metrics.add(new Metric("cluster-controller.resource_usage.max_memory_utilization.max")); metrics.add(new Metric("cluster-controller.resource_usage.max_disk_utilization.last")); + metrics.add(new Metric("cluster-controller.resource_usage.max_disk_utilization.max")); metrics.add(new Metric("cluster-controller.resource_usage.disk_limit.last")); metrics.add(new Metric("cluster-controller.resource_usage.memory_limit.last")); @@ -762,6 +765,15 @@ public class VespaMetricSet { metrics.add(new Metric("vds.bouncer.clock_skew_aborts.count")); + metrics.add(new Metric("vds.mergethrottler.averagequeuewaitingtime.max")); + metrics.add(new Metric("vds.mergethrottler.averagequeuewaitingtime.sum")); + metrics.add(new Metric("vds.mergethrottler.averagequeuewaitingtime.count")); + metrics.add(new Metric("vds.mergethrottler.queuesize.max")); + metrics.add(new Metric("vds.mergethrottler.queuesize.sum")); + metrics.add(new Metric("vds.mergethrottler.queuesize.count")); + metrics.add(new Metric("vds.mergethrottler.bounced_due_to_back_pressure.rate")); + metrics.add(new Metric("vds.mergethrottler.locallyexecutedmerges.ok.rate")); + metrics.add(new Metric("vds.mergethrottler.mergechains.ok.rate")); return metrics; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java index d87c6596fa4..52dccbe96b5 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java @@ -8,7 +8,7 @@ import com.yahoo.log.InvalidLogFormatException; import com.yahoo.log.LogMessage; import com.yahoo.path.Path; import com.yahoo.searchdefinition.OnnxModel; -import com.yahoo.searchdefinition.RankExpressionFile; +import com.yahoo.searchdefinition.RankExpressionBody; import com.yahoo.vespa.config.search.core.RankingExpressionsConfig; import com.yahoo.vespa.defaults.Defaults; import com.yahoo.yolean.Exceptions; @@ -165,7 +165,7 @@ public class RankSetupValidator extends Validator { config.add(String.format("file[%d].path \"%s\"", config.size() / 2, modelPath)); } - for (RankExpressionFile expr : db.getDerivedConfiguration().getSearch().rankExpressionFiles().asMap().values()) { + for (RankExpressionBody expr : db.getDerivedConfiguration().getSearch().rankExpressionFiles().asMap().values()) { String modelPath = getFileRepositoryPath(expr.getFilePath(), expr.getFileReference()); config.add(String.format("file[%d].ref \"%s\"", config.size() / 2, expr.getFileReference())); config.add(String.format("file[%d].path \"%s\"", config.size() / 2, modelPath)); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java index 55443d4b260..84c7a48a998 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java @@ -20,10 +20,12 @@ import com.yahoo.vespa.model.application.validation.change.GlobalDocumentChangeV import com.yahoo.vespa.model.application.validation.change.IndexedSearchClusterChangeValidator; import com.yahoo.vespa.model.application.validation.change.IndexingModeChangeValidator; import com.yahoo.vespa.model.application.validation.change.NodeResourceChangeValidator; +import com.yahoo.vespa.model.application.validation.change.RedundancyIncreaseValidator; import com.yahoo.vespa.model.application.validation.change.ResourcesReductionValidator; import com.yahoo.vespa.model.application.validation.change.StartupCommandChangeValidator; import com.yahoo.vespa.model.application.validation.change.StreamingSearchClusterChangeValidator; import com.yahoo.vespa.model.application.validation.first.AccessControlOnFirstDeploymentValidator; +import com.yahoo.vespa.model.application.validation.first.RedundancyOnFirstDeploymentValidator; import java.time.Instant; import java.util.Arrays; @@ -53,6 +55,7 @@ public class Validation { * between the previous and current model * * @return a list of required changes needed to make this configuration live + * @throws ValidationOverrides.ValidationException if the change fails validation */ public static List<ConfigChangeAction> validate(VespaModel model, ValidationParameters validationParameters, DeployState deployState) { if (validationParameters.checkRouting()) { @@ -105,7 +108,8 @@ public class Validation { new ClusterSizeReductionValidator(), new ResourcesReductionValidator(), new ContainerRestartValidator(), - new NodeResourceChangeValidator() + new NodeResourceChangeValidator(), + new RedundancyIncreaseValidator() }; List<ConfigChangeAction> actions = Arrays.stream(validators) .flatMap(v -> v.validate(currentModel, nextModel, overrides, now).stream()) @@ -122,6 +126,7 @@ public class Validation { private static void validateFirstTimeDeployment(VespaModel model, DeployState deployState) { new AccessControlOnFirstDeploymentValidator().validate(model, deployState); + new RedundancyOnFirstDeploymentValidator().validate(model, deployState); } private static void deferConfigChangesForClustersToBeRestarted(List<ConfigChangeAction> actions, VespaModel model) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validator.java index f3bebbe7fb9..fee63828670 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validator.java @@ -15,7 +15,7 @@ public abstract class Validator { * Validates the input vespamodel * * @param model a VespaModel object - * @param deployState The {@link DeployState} built from building the model + * @param deployState the {@link DeployState} built from building the model */ public abstract void validate(VespaModel model, DeployState deployState); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ChangeValidator.java index b720cc13f42..4222d22563d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ChangeValidator.java @@ -25,6 +25,7 @@ public interface ChangeValidator { * @param now the instant to use as now * @return a list of actions specifying what needs to be done in order to activate the new model. * Return an empty list if nothing needs to be done + * @throws IllegalArgumentException if the change fails validation */ List<ConfigChangeAction> validate(VespaModel current, VespaModel next, ValidationOverrides overrides, Instant now); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/RedundancyIncreaseValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/RedundancyIncreaseValidator.java new file mode 100644 index 00000000000..dcf16222d35 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/RedundancyIncreaseValidator.java @@ -0,0 +1,45 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation.change; + +import com.yahoo.config.application.api.ValidationId; +import com.yahoo.config.application.api.ValidationOverrides; +import com.yahoo.config.model.api.ConfigChangeAction; +import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.content.cluster.ContentCluster; + +import java.time.Instant; +import java.util.List; + +/** + * Checks that redundancy is not increased (without a validation override), + * as that may easily cause the cluster to run out of reasources. + * + * @author bratseth + */ +public class RedundancyIncreaseValidator implements ChangeValidator { + + @Override + public List<ConfigChangeAction> validate(VespaModel current, VespaModel next, ValidationOverrides overrides, Instant now) { + for (ContentCluster currentCluster : current.getContentClusters().values()) { + ContentCluster nextCluster = next.getContentClusters().get(currentCluster.getSubId()); + if (nextCluster == null) continue; + if (redundancyOf(nextCluster) > redundancyOf(currentCluster)) { + overrides.invalid(ValidationId.redundancyIncrease, + "Increasing redundancy from " + redundancyOf(currentCluster) + " to " + + redundancyOf(nextCluster) + " in '" + currentCluster + ". " + + "This is a safe operation but verify that you have room for a " + + redundancyOf(nextCluster) + "/" + redundancyOf(currentCluster) + "x increase " + + "in content size", + now); + } + } + return List.of(); + } + + private int redundancyOf(ContentCluster cluster) { + return cluster.redundancy().finalRedundancy(); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/first/RedundancyOnFirstDeploymentValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/first/RedundancyOnFirstDeploymentValidator.java new file mode 100644 index 00000000000..e6117299269 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/first/RedundancyOnFirstDeploymentValidator.java @@ -0,0 +1,44 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation.first; + +import com.yahoo.config.application.api.ValidationId; +import com.yahoo.config.model.ConfigModelContext.ApplicationType; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.application.validation.Validator; +import com.yahoo.vespa.model.container.ApplicationContainerCluster; +import com.yahoo.vespa.model.container.Container; +import com.yahoo.vespa.model.container.ContainerCluster; +import com.yahoo.vespa.model.content.cluster.ContentCluster; + +import java.util.ArrayList; +import java.util.List; + +import static com.yahoo.collections.CollectionUtil.mkString; +import static com.yahoo.config.provision.InstanceName.defaultName; +import static com.yahoo.vespa.model.container.http.AccessControl.hasHandlerThatNeedsProtection; + +/** + * Validates that applications in prod zones do not have redundancy 1 (without a validation override). + * + * @author bratseth + */ +public class RedundancyOnFirstDeploymentValidator extends Validator { + + @Override + public void validate(VespaModel model, DeployState deployState) { + if ( ! deployState.isHosted()) return; + if ( ! deployState.zone().environment().isProduction()) return; + + for (ContentCluster cluster : model.getContentClusters().values()) { + if (cluster.redundancy().finalRedundancy() == 1 + && cluster.redundancy().totalNodes() > cluster.redundancy().groups()) + deployState.validationOverrides().invalid(ValidationId.redundancyOne, + cluster + " has redundancy 1, which will cause it to lose data " + + "if a node fails. This requires an override on first deployment " + + "in a production zone", + deployState.now()); + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterResourceLimits.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterResourceLimits.java index 70f2acd3c7b..638864d85bb 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterResourceLimits.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterResourceLimits.java @@ -37,13 +37,16 @@ public class ClusterResourceLimits { private final boolean enableFeedBlockInDistributor; private final boolean hostedVespa; + private final boolean throwIfSpecified; private final DeployLogger deployLogger; + private ResourceLimits.Builder ctrlBuilder = new ResourceLimits.Builder(); private ResourceLimits.Builder nodeBuilder = new ResourceLimits.Builder(); - public Builder(boolean enableFeedBlockInDistributor, boolean hostedVespa, DeployLogger deployLogger) { + public Builder(boolean enableFeedBlockInDistributor, boolean hostedVespa, boolean throwIfSpecified, DeployLogger deployLogger) { this.enableFeedBlockInDistributor = enableFeedBlockInDistributor; this.hostedVespa = hostedVespa; + this.throwIfSpecified = throwIfSpecified; this.deployLogger = deployLogger; } @@ -58,7 +61,7 @@ public class ClusterResourceLimits { private ResourceLimits.Builder createBuilder(ModelElement element) { return element == null ? new ResourceLimits.Builder() - : DomResourceLimitsBuilder.createBuilder(element, hostedVespa, deployLogger); + : DomResourceLimitsBuilder.createBuilder(element, hostedVespa, throwIfSpecified, deployLogger); } public void setClusterControllerBuilder(ResourceLimits.Builder builder) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java b/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java index ea52f9689ff..4a8002ba3dc 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java @@ -111,13 +111,15 @@ public class Content extends ConfigModel { return null; } - private static void checkThatExplicitIndexingChainInheritsCorrectly(ComponentRegistry<DocprocChain> allChains, ChainSpecification chainSpec) { + private static void checkThatExplicitIndexingChainInheritsCorrectly(ComponentRegistry<DocprocChain> allChains, + ChainSpecification chainSpec) { ChainSpecification.Inheritance inheritance = chainSpec.inheritance; for (ComponentSpecification componentSpec : inheritance.chainSpecifications) { ChainSpecification parentSpec = getChainSpec(allChains, componentSpec); if (containsIndexingChain(allChains, parentSpec)) return; } - throw new IllegalArgumentException("Docproc chain '" + chainSpec.componentId + "' does not inherit from 'indexing' chain."); + throw new IllegalArgumentException("Docproc chain '" + chainSpec.componentId + + "' must inherit from the 'indexing' chain"); } public static List<Content> getContent(ConfigModelRepo pc) { @@ -261,9 +263,17 @@ public class Content extends ConfigModel { if (cluster.hasExplicitIndexingChain()) { indexingChain = allChains.getComponent(cluster.getIndexingChainName()); if (indexingChain == null) { - throw new RuntimeException("Indexing cluster " + cluster.getClusterName() + " refers to docproc " + - "chain " + cluster.getIndexingChainName() + " for indexing, which does not exist."); - } else { + throw new IllegalArgumentException(cluster + " refers to docproc " + + "chain '" + cluster.getIndexingChainName() + + "' for indexing, but this chain does not exist"); + } + else if (indexingChain.getId().getName().equals("default")) { + throw new IllegalArgumentException(cluster + " specifies the chain " + + "'default' as indexing chain. As the 'default' chain is run by default, " + + "using it as the indexing chain will run it twice. " + + "Use a different name for the indexing chain."); + } + else { checkThatExplicitIndexingChainInheritsCorrectly(allChains, indexingChain.getChainSpecification()); } } else { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index 51949e78838..efb47e97ccb 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -316,12 +316,8 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> } public void handleRedundancy(Redundancy redundancy) { - if (hasIndexedCluster()) { - if (usesHierarchicDistribution()) { - indexedCluster.setMaxNodesDownPerFixedRow((redundancy.effectiveFinalRedundancy() / groupToSpecMap.size()) - 1); - } + if (hasIndexedCluster()) indexedCluster.setSearchableCopies(redundancy.readyCopies()); - } this.redundancy = redundancy; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/DispatchTuning.java b/config-model/src/main/java/com/yahoo/vespa/model/content/DispatchTuning.java index 3b694f8986c..786d032578f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/DispatchTuning.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/DispatchTuning.java @@ -15,7 +15,6 @@ public class DispatchTuning { private final Integer maxHitsPerPartition; private DispatchPolicy dispatchPolicy; - private final Double minGroupCoverage; private final Double minActiveDocsCoverage; public Double getTopkProbability() { @@ -27,7 +26,6 @@ public class DispatchTuning { private DispatchTuning(Builder builder) { maxHitsPerPartition = builder.maxHitsPerPartition; dispatchPolicy = builder.dispatchPolicy; - minGroupCoverage = builder.minGroupCoverage; minActiveDocsCoverage = builder.minActiveDocsCoverage; topkProbability = builder.topKProbability; } @@ -41,9 +39,6 @@ public class DispatchTuning { @SuppressWarnings("unused") public void setDispatchPolicy(DispatchPolicy dispatchPolicy) { this.dispatchPolicy = dispatchPolicy; } - /** Returns the percentage of nodes in a group which must be up for that group to receive queries */ - public Double getMinGroupCoverage() { return minGroupCoverage; } - /** Returns the percentage of documents which must be available in a group for that group to receive queries */ public Double getMinActiveDocsCoverage() { return minActiveDocsCoverage; } @@ -51,7 +46,6 @@ public class DispatchTuning { private Integer maxHitsPerPartition; private DispatchPolicy dispatchPolicy; - private Double minGroupCoverage; private Double minActiveDocsCoverage; private Double topKProbability; @@ -81,11 +75,6 @@ public class DispatchTuning { } } - public Builder setMinGroupCoverage(Double minGroupCoverage) { - this.minGroupCoverage = minGroupCoverage; - return this; - } - public Builder setMinActiveDocsCoverage(Double minCoverage) { this.minActiveDocsCoverage = minCoverage; return this; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index e0d311e6df6..c298b7f5f5a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -123,6 +123,7 @@ public class ContentCluster extends AbstractConfigProducer<AbstractConfigProduce boolean enableFeedBlockInDistributor = deployState.getProperties().featureFlags().enableFeedBlockInDistributor(); var resourceLimits = new ClusterResourceLimits.Builder(enableFeedBlockInDistributor, stateIsHosted(deployState), + deployState.featureFlags().throwIfResourceLimitsSpecified(), deployState.getDeployLogger()) .build(contentElement); c.clusterControllerConfig = new ClusterControllerConfig.Builder(getClusterId(contentElement), @@ -674,4 +675,9 @@ public class ContentCluster extends AbstractConfigProducer<AbstractConfigProduce // TODO } + @Override + public String toString() { + return "content cluster '" + clusterId + "'"; + } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomResourceLimitsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomResourceLimitsBuilder.java index 9f4852629d0..37adb73bc15 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomResourceLimitsBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomResourceLimitsBuilder.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.content.cluster; import com.yahoo.config.application.api.DeployLogger; @@ -14,18 +14,22 @@ import java.util.logging.Level; */ public class DomResourceLimitsBuilder { - public static ResourceLimits.Builder createBuilder(ModelElement contentXml, boolean hostedVespa, DeployLogger deployLogger) { + public static ResourceLimits.Builder createBuilder(ModelElement contentXml, + boolean hostedVespa, + boolean throwIfSpecified, + DeployLogger deployLogger) { ResourceLimits.Builder builder = new ResourceLimits.Builder(); ModelElement resourceLimits = contentXml.child("resource-limits"); if (resourceLimits == null) { return builder; } if (hostedVespa) { - deployLogger.logApplicationPackage(Level.WARNING, "Element " + resourceLimits + - " is not allowed, default limits will be used"); - // TODO: Throw exception when we are sure nobody is using this - //throw new IllegalArgumentException("Element " + element + " is not allowed to be set, default limits will be used"); - return builder; + String message = "Element '" + resourceLimits + "' is not allowed to be set"; + if (throwIfSpecified) + throw new IllegalArgumentException(message); + else + deployLogger.logApplicationPackage(Level.WARNING, message); } + if (resourceLimits.child("disk") != null) { builder.setDiskLimit(resourceLimits.childAsDouble("disk")); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomTuningDispatchBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomTuningDispatchBuilder.java index 64911acae1f..f429e40baa9 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomTuningDispatchBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomTuningDispatchBuilder.java @@ -25,12 +25,14 @@ public class DomTuningDispatchBuilder { builder.setMaxHitsPerPartition(dispatchElement.childAsInteger("max-hits-per-partition")); builder.setTopKProbability(dispatchElement.childAsDouble("top-k-probability")); builder.setDispatchPolicy(dispatchElement.childAsString("dispatch-policy")); - builder.setMinGroupCoverage(dispatchElement.childAsDouble("min-group-coverage")); builder.setMinActiveDocsCoverage(dispatchElement.childAsDouble("min-active-docs-coverage")); + if (dispatchElement.child("min-group-coverage") != null) + logger.logApplicationPackage(Level.WARNING, "Attribute 'min-group-coverage' is deprecated and ignored: " + + "Use min-active-docs-coverage instead."); if (dispatchElement.child("use-local-node") != null) logger.logApplicationPackage(Level.WARNING, "Attribute 'use-local-node' is deprecated and ignored: " + - "The local node will automatically be preferred when appropriate."); + "The local node will automatically be preferred when appropriate."); return builder.build(); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java index 40a634fbfe8..e89d45e8b83 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java @@ -6,6 +6,8 @@ import com.yahoo.vespa.config.content.core.StorServerConfig; import com.yahoo.vespa.model.content.cluster.ContentCluster; import com.yahoo.vespa.model.builder.xml.dom.ModelElement; +import java.util.Optional; + /** * Serves config for stor-server for storage clusters (clusters of storage nodes). */ @@ -14,7 +16,7 @@ public class StorServerProducer implements StorServerConfig.Producer { StorServerProducer build(ModelContext.Properties properties, ModelElement element) { ModelElement tuning = element.child("tuning"); - StorServerProducer producer = new StorServerProducer(ContentCluster.getClusterId(element)); + StorServerProducer producer = new StorServerProducer(ContentCluster.getClusterId(element), properties.featureFlags()); if (tuning == null) return producer; ModelElement merges = tuning.child("merges"); @@ -32,11 +34,15 @@ public class StorServerProducer implements StorServerConfig.Producer { private Integer bucketDBStripeBits; private StorServerProducer setMaxMergesPerNode(Integer value) { - maxMergesPerNode = value; + if (value != null) { + maxMergesPerNode = value; + } return this; } private StorServerProducer setMaxQueueSize(Integer value) { - queueSize = value; + if (value != null) { + queueSize = value; + } return this; } private StorServerProducer setBucketDBStripeBits(Integer value) { @@ -44,8 +50,10 @@ public class StorServerProducer implements StorServerConfig.Producer { return this; } - public StorServerProducer(String clusterName) { + StorServerProducer(String clusterName, ModelContext.FeatureFlags featureFlags) { this.clusterName = clusterName; + maxMergesPerNode = featureFlags.maxConcurrentMergesPerNode(); + queueSize = featureFlags.maxMergeQueueSize(); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java index 5bb57f4ff6c..d8da911e32f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java @@ -8,6 +8,7 @@ import com.yahoo.config.application.api.FileRegistry; import com.yahoo.vespa.model.ConfigProxy; import com.yahoo.vespa.model.Host; +import java.nio.ByteBuffer; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; @@ -58,6 +59,16 @@ public class FileDistributor { return addFileReference(fileRegistry.addUri(uri), host); } + /** + * Adds the given blob to the associated application packages' registry of file and marks the file + * for distribution to the given host. + * + * @return the reference to the file, created by the application package + */ + public FileReference sendBlobToHost(ByteBuffer blob, Host host) { + return addFileReference(fileRegistry.addBlob(blob), host); + } + private FileReference addFileReference(FileReference reference, Host host) { filesToHosts.computeIfAbsent(reference, k -> new HashSet<>()).add(host); return reference; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/DispatchGroup.java b/config-model/src/main/java/com/yahoo/vespa/model/search/DispatchGroup.java index 384f77737c1..3e70bda216b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/DispatchGroup.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/DispatchGroup.java @@ -54,19 +54,18 @@ public class DispatchGroup { public int getSearchableCopies() { return sc.getSearchableCopies(); } - public int getMaxNodesDownPerFixedRow() { - return sc.getMaxNodesDownPerFixedRow(); - } - static class Iterator implements java.util.Iterator<SearchInterface> { + private java.util.Iterator<Map<Integer, SearchInterface>> it1; private java.util.Iterator<SearchInterface> it2; + Iterator(Map<Integer, Map<Integer, SearchInterface> > s) { it1 = s.values().iterator(); if (it1.hasNext()) { it2 = it1.next().values().iterator(); } } + @Override public boolean hasNext() { if (it2 == null) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java index 99f1b3ad34e..c99549e82e9 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java @@ -46,7 +46,6 @@ public class IndexedSearchCluster extends SearchCluster private String routingSelector = null; private final List<DocumentDatabase> documentDbs = new LinkedList<>(); private final UnionConfiguration unionCfg; - private int maxNodesDownPerFixedRow = 0; private int searchableCopies = 1; @@ -261,13 +260,6 @@ public class IndexedSearchCluster extends SearchCluster return false; } - int getMaxNodesDownPerFixedRow() { - return maxNodesDownPerFixedRow; - } - - public void setMaxNodesDownPerFixedRow(int value) { - maxNodesDownPerFixedRow = value; - } public int getSearchableCopies() { return searchableCopies; } @@ -305,8 +297,6 @@ public class IndexedSearchCluster extends SearchCluster } if (tuning.dispatch.getMinActiveDocsCoverage() != null) builder.minActivedocsPercentage(tuning.dispatch.getMinActiveDocsCoverage()); - if (tuning.dispatch.getMinGroupCoverage() != null) - builder.minGroupCoverage(tuning.dispatch.getMinGroupCoverage()); if (tuning.dispatch.getDispatchPolicy() != null) { switch (tuning.dispatch.getDispatchPolicy()) { case ADAPTIVE: @@ -320,7 +310,6 @@ public class IndexedSearchCluster extends SearchCluster if (tuning.dispatch.getMaxHitsPerPartition() != null) builder.maxHitsPerNode(tuning.dispatch.getMaxHitsPerPartition()); - builder.maxNodesDownPerGroup(rootDispatch.getMaxNodesDownPerFixedRow()); builder.searchableCopies(rootDispatch.getSearchableCopies()); if (searchCoverage != null) { if (searchCoverage.getMinimum() != null) @@ -336,6 +325,11 @@ public class IndexedSearchCluster extends SearchCluster @Override public int getRowBits() { return 8; } + @Override + public String toString() { + return "Indexing cluster '" + getClusterName() + "'"; + } + /** * Class used to retrieve combined configuration from multiple document databases. * It is not a {@link com.yahoo.config.ConfigInstance.Producer} of those configs, diff --git a/config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java b/config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java index 5e7ac0cabec..52edec7114b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java @@ -12,6 +12,7 @@ import com.yahoo.vespa.config.ConfigPayloadBuilder; import com.yahoo.vespa.model.AbstractService; import java.io.Serializable; +import java.nio.ByteBuffer; import java.util.Collection; import java.util.HashMap; import java.util.Map; @@ -61,6 +62,20 @@ public class FileSender implements Serializable { return fileref; } + public static FileReference sendBlobToServices(ByteBuffer blob, Collection<? extends AbstractService> services) { + if (services.isEmpty()) { + throw new IllegalStateException("No service instances. Probably a standalone cluster setting up <nodes> " + + "using 'count' instead of <node> tags."); + } + + FileReference fileref = null; + for (AbstractService service : services) { + // The same reference will be returned from each call. + fileref = service.sendBlob(blob); + } + return fileref; + } + /** * Sends all user configured files for a producer to all given services. */ |