diff options
Diffstat (limited to 'config-model')
48 files changed, 657 insertions, 447 deletions
diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 75a1a167446..e6bf3a835c6 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -43,7 +43,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private String sequencerType = "LATENCY"; private String responseSequencerType = "ADAPTIVE"; private int responseNumThreads = 2; - private int maxPendingMoveOps = 10; private Optional<EndpointCertificateSecrets> endpointCertificateSecrets = Optional.empty(); private AthenzDomain athenzDomain; private ApplicationRoles applicationRoles; @@ -51,11 +50,9 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private boolean useAccessControlTlsHandshakeClientAuth; private boolean useAsyncMessageHandlingOnSchedule = false; private double feedConcurrency = 0.5; - private boolean useBucketExecutorForLidSpaceCompact; - private boolean useBucketExecutorForBucketMove; + private boolean useBucketExecutorForPruneRemoved; private boolean enableFeedBlockInDistributor = true; - private double maxDeadBytesRatio = 0.2; - private int clusterControllerMaxHeapSizeInMb = 256; + private int clusterControllerMaxHeapSizeInMb = 128; private int metricsProxyMaxHeapSizeInMb = 256; private int maxActivationInhibitedOutOfSyncGroups = 0; private List<TenantSecretStore> tenantSecretStores = Collections.emptyList(); @@ -85,7 +82,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public Optional<ApplicationRoles> applicationRoles() { return Optional.ofNullable(applicationRoles); } @Override public String responseSequencerType() { return responseSequencerType; } @Override public int defaultNumResponseThreads() { return responseNumThreads; } - @Override public int maxPendingMoveOps() { return maxPendingMoveOps; } @Override public boolean skipCommunicationManagerThread() { return false; } @Override public boolean skipMbusRequestThread() { return false; } @Override public boolean skipMbusReplyThread() { return false; } @@ -93,10 +89,8 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public boolean useAccessControlTlsHandshakeClientAuth() { return useAccessControlTlsHandshakeClientAuth; } @Override public boolean useAsyncMessageHandlingOnSchedule() { return useAsyncMessageHandlingOnSchedule; } @Override public double feedConcurrency() { return feedConcurrency; } - @Override public boolean useBucketExecutorForLidSpaceCompact() { return useBucketExecutorForLidSpaceCompact; } - @Override public boolean useBucketExecutorForBucketMove() { return useBucketExecutorForBucketMove; } + @Override public boolean useBucketExecutorForPruneRemoved() { return useBucketExecutorForPruneRemoved; } @Override public boolean enableFeedBlockInDistributor() { return enableFeedBlockInDistributor; } - @Override public double maxDeadBytesRatio() { return maxDeadBytesRatio; } @Override public int clusterControllerMaxHeapSizeInMb() { return clusterControllerMaxHeapSizeInMb; } @Override public int metricsProxyMaxHeapSizeInMb(ClusterSpec.Type type) { return metricsProxyMaxHeapSizeInMb; } @Override public int maxActivationInhibitedOutOfSyncGroups() { return maxActivationInhibitedOutOfSyncGroups; } @@ -131,10 +125,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea responseNumThreads = numThreads; return this; } - public TestProperties setMaxPendingMoveOps(int moveOps) { - maxPendingMoveOps = moveOps; - return this; - } + public TestProperties setDefaultTermwiseLimit(double limit) { defaultTermwiseLimit = limit; return this; @@ -200,13 +191,8 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } - public TestProperties useBucketExecutorForLidSpaceCompact(boolean enabled) { - useBucketExecutorForLidSpaceCompact = enabled; - return this; - } - - public TestProperties useBucketExecutorForBucketMove(boolean enabled) { - useBucketExecutorForBucketMove = enabled; + public TestProperties useBucketExecutorForPruneRemoved(boolean enabled) { + useBucketExecutorForPruneRemoved = enabled; return this; } @@ -215,11 +201,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } - public TestProperties maxDeadBytesRatio(double ratio) { - maxDeadBytesRatio = ratio; - return this; - } - public TestProperties clusterControllerMaxHeapSizeInMb(int heapSize) { clusterControllerMaxHeapSizeInMb = heapSize; return this; diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java index 188504edd18..e41e7309986 100644 --- a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java +++ b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java @@ -63,6 +63,8 @@ public class InMemoryProvisioner implements HostProvisioner { private final boolean useMaxResources; + private final boolean alwaysReturnOneNode; + private Provisioned provisioned = new Provisioned(); /** Creates this with a number of nodes with resources 1, 3, 9, 1 */ @@ -72,37 +74,39 @@ public class InMemoryProvisioner implements HostProvisioner { /** Creates this with a number of nodes with given resources */ public InMemoryProvisioner(int nodeCount, NodeResources resources, boolean sharedHosts) { - this(Map.of(resources, createHostInstances(nodeCount)), true, false, sharedHosts, 0); + this(Map.of(resources, createHostInstances(nodeCount)), true, false, false, sharedHosts, 0); } /** Creates this with a set of host names of the flavor 'default' */ public InMemoryProvisioner(boolean failOnOutOfCapacity, boolean sharedHosts, String... hosts) { - this(Map.of(defaultResources, toHostInstances(hosts)), failOnOutOfCapacity, false, sharedHosts, 0); + this(Map.of(defaultResources, toHostInstances(hosts)), failOnOutOfCapacity, false, false, sharedHosts, 0); } /** Creates this with a set of host names of the flavor 'default' */ public InMemoryProvisioner(boolean failOnOutOfCapacity, boolean sharedHosts, List<String> hosts) { - this(Map.of(defaultResources, toHostInstances(hosts.toArray(new String[0]))), failOnOutOfCapacity, false, sharedHosts, 0); + this(Map.of(defaultResources, toHostInstances(hosts.toArray(new String[0]))), failOnOutOfCapacity, false, false, sharedHosts, 0); } /** Creates this with a set of hosts of the flavor 'default' */ public InMemoryProvisioner(Hosts hosts, boolean failOnOutOfCapacity, boolean sharedHosts, String ... retiredHostNames) { - this(Map.of(defaultResources, hosts.asCollection()), failOnOutOfCapacity, false, sharedHosts, 0, retiredHostNames); + this(Map.of(defaultResources, hosts.asCollection()), failOnOutOfCapacity, false, false, sharedHosts, 0, retiredHostNames); } /** Creates this with a set of hosts of the flavor 'default' */ public InMemoryProvisioner(Hosts hosts, boolean failOnOutOfCapacity, boolean sharedHosts, int startIndexForClusters, String ... retiredHostNames) { - this(Map.of(defaultResources, hosts.asCollection()), failOnOutOfCapacity, false, sharedHosts, startIndexForClusters, retiredHostNames); + this(Map.of(defaultResources, hosts.asCollection()), failOnOutOfCapacity, false, false, sharedHosts, startIndexForClusters, retiredHostNames); } public InMemoryProvisioner(Map<NodeResources, Collection<Host>> hosts, boolean failOnOutOfCapacity, boolean useMaxResources, + boolean alwaysReturnOneNode, boolean sharedHosts, int startIndexForClusters, String ... retiredHostNames) { this.failOnOutOfCapacity = failOnOutOfCapacity; this.useMaxResources = useMaxResources; + this.alwaysReturnOneNode = alwaysReturnOneNode; for (Map.Entry<NodeResources, Collection<Host>> hostsWithResources : hosts.entrySet()) for (Host host : hostsWithResources.getValue()) freeNodes.put(hostsWithResources.getKey(), host); @@ -142,16 +146,20 @@ public class InMemoryProvisioner implements HostProvisioner { public List<HostSpec> prepare(ClusterSpec cluster, ClusterResources requested, boolean required, boolean canFail) { if (cluster.group().isPresent() && requested.groups() > 1) throw new IllegalArgumentException("Cannot both be specifying a group and ask for groups to be created"); - int capacity = failOnOutOfCapacity || required + + int nodes = failOnOutOfCapacity || required ? requested.nodes() : Math.min(requested.nodes(), freeNodes.get(defaultResources).size() + totalAllocatedTo(cluster)); - int groups = requested.groups() > capacity ? capacity : requested.groups(); + if (alwaysReturnOneNode) + nodes = 1; + + int groups = requested.groups() > nodes ? nodes : requested.groups(); List<HostSpec> allocation = new ArrayList<>(); if (groups == 1) { allocation.addAll(allocateHostGroup(cluster.with(Optional.of(ClusterSpec.Group.from(0))), requested.nodeResources(), - capacity, + nodes, startIndexForClusters, canFail)); } @@ -159,7 +167,7 @@ public class InMemoryProvisioner implements HostProvisioner { for (int i = 0; i < groups; i++) { allocation.addAll(allocateHostGroup(cluster.with(Optional.of(ClusterSpec.Group.from(i))), requested.nodeResources(), - capacity / groups, + nodes / groups, allocation.size(), canFail)); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/MapEvaluationTypeContext.java b/config-model/src/main/java/com/yahoo/searchdefinition/MapEvaluationTypeContext.java index e8ee5e9ed57..41cb40da4d6 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/MapEvaluationTypeContext.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/MapEvaluationTypeContext.java @@ -206,6 +206,14 @@ public class MapEvaluationTypeContext extends FunctionReferenceContext implement return featureTypes.get(reference); } + // the name of a constant feature? + if (reference.isIdentifier()) { + Reference asConst = FeatureNames.asConstantFeature(reference.name()); + if (featureTypes.containsKey(asConst)) { + return featureTypes.get(asConst); + } + } + // We do not know what this is - since we do not have complete knowledge about the match features // in Java we must assume this is a match feature and return the double type - which is the type of // all match features diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/OnnxModels.java b/config-model/src/main/java/com/yahoo/searchdefinition/OnnxModels.java index 1cc33664e8c..60733a4f5ba 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/OnnxModels.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/OnnxModels.java @@ -9,7 +9,7 @@ import java.util.HashMap; import java.util.Map; /** - * ONNX models tied to a search definition. + * ONNX models tied to a search definition or global. * * @author lesters */ @@ -23,6 +23,10 @@ public class OnnxModels { models.put(name, model); } + public void add(Map<String, OnnxModel> models) { + models.values().forEach(this::add); + } + public OnnxModel get(String name) { return models.get(name); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java index b460752d7bd..be246a143b2 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java @@ -161,7 +161,7 @@ public class RankProfile implements Cloneable { return search != null ? search.rankingConstants() : model.rankingConstants(); } - private Map<String, OnnxModel> onnxModels() { + public Map<String, OnnxModel> onnxModels() { return search != null ? search.onnxModels().asMap() : Collections.emptyMap(); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/Search.java b/config-model/src/main/java/com/yahoo/searchdefinition/Search.java index 64c5590b689..9b7434dccab 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/Search.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/Search.java @@ -61,6 +61,8 @@ public class Search implements ImmutableSearch { /** True if this doesn't define a search, just a document type */ private boolean documentsOnly = false; + private boolean rawAsBase64 = false; + /** The stemming setting of this search definition. Default is BEST. */ private Stemming stemming = Stemming.BEST; @@ -125,6 +127,16 @@ public class Search implements ImmutableSearch { } /** + * Returns true if 'raw' fields shall be presented as base64 in summary + * Note that tis is temporary and will disappear on Vespa 8 as it will become default, and only option. + * + * @return true if raw shall be encoded as base64 in summary + */ + public boolean isRawAsBase64() { return rawAsBase64; } + + public void enableRawAsBase64() { rawAsBase64 = true; } + + /** * Sets the stemming default of fields. Default is ALL * * @param stemming set default stemming for this searchdefinition diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java index 22a32c8fd65..5337d58fb82 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java @@ -57,8 +57,8 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ ModelContext.Properties deployProperties) { setName(search == null ? "default" : search.getName()); this.rankingConstants = rankingConstants; - deriveRankProfiles(rankProfileRegistry, queryProfiles, importedModels, search, attributeFields, deployProperties); this.onnxModels = search == null ? new OnnxModels() : search.onnxModels(); // as ONNX models come from parsing rank expressions + deriveRankProfiles(rankProfileRegistry, queryProfiles, importedModels, search, attributeFields, deployProperties); } private void deriveRankProfiles(RankProfileRegistry rankProfileRegistry, @@ -94,6 +94,10 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ rankingConstants.sendTo(services); } + public void sendOnnxModelsTo(Collection<? extends AbstractService> services) { + onnxModels.sendTo(services); + } + @Override public String getDerivedName() { return "rank-profiles"; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClass.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClass.java index b08e9948ecd..af2168545dc 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClass.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClass.java @@ -27,6 +27,7 @@ public class SummaryClass extends Derived { /** True if this summary class needs to access summary information on disk */ private boolean accessingDiskSummary = false; + private final boolean rawAsBase64; /** The summary fields of this indexed by name */ private Map<String,SummaryClassField> fields = new java.util.LinkedHashMap<>(); @@ -42,6 +43,7 @@ public class SummaryClass extends Derived { */ public SummaryClass(Search search, DocumentSummary summary, DeployLogger deployLogger) { this.deployLogger = deployLogger; + this.rawAsBase64 = search.isRawAsBase64(); deriveName(summary); deriveFields(search,summary); deriveImplicitFields(summary); @@ -74,12 +76,12 @@ public class SummaryClass extends Derived { private void addField(String name, DataType type, SummaryTransform transform) { if (fields.containsKey(name)) { SummaryClassField sf = fields.get(name); - if (!SummaryClassField.convertDataType(type, transform).equals(sf.getType())) { + if (!SummaryClassField.convertDataType(type, transform, rawAsBase64).equals(sf.getType())) { deployLogger.logApplicationPackage(Level.WARNING, "Conflicting definition of field " + name + ". " + "Declared as type " + sf.getType() + " and " + type); } } else { - fields.put(name, new SummaryClassField(name, type, transform)); + fields.put(name, new SummaryClassField(name, type, transform, rawAsBase64)); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClassField.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClassField.java index 4375b446e98..3c29971a74c 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClassField.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClassField.java @@ -5,7 +5,19 @@ import com.yahoo.document.CollectionDataType; import com.yahoo.document.DataType; import com.yahoo.document.MapDataType; import com.yahoo.document.ReferenceDataType; -import com.yahoo.document.datatypes.*; +import com.yahoo.document.datatypes.BoolFieldValue; +import com.yahoo.document.datatypes.ByteFieldValue; +import com.yahoo.document.datatypes.DoubleFieldValue; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.Float16FieldValue; +import com.yahoo.document.datatypes.FloatFieldValue; +import com.yahoo.document.datatypes.IntegerFieldValue; +import com.yahoo.document.datatypes.LongFieldValue; +import com.yahoo.document.datatypes.PredicateFieldValue; +import com.yahoo.document.datatypes.Raw; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.document.datatypes.Struct; +import com.yahoo.document.datatypes.TensorFieldValue; import com.yahoo.vespa.documentmodel.SummaryTransform; /** @@ -32,6 +44,7 @@ public class SummaryClassField { DOUBLE("double"), STRING("string"), DATA("data"), + RAW("raw"), LONGSTRING("longstring"), LONGDATA("longdata"), XMLSTRING("xmlstring"), @@ -39,7 +52,7 @@ public class SummaryClassField { JSONSTRING("jsonstring"), TENSOR("tensor"); - private String name; + private final String name; Type(String name) { this.name = name; @@ -55,9 +68,9 @@ public class SummaryClassField { } } - public SummaryClassField(String name, DataType type, SummaryTransform transform) { + public SummaryClassField(String name, DataType type, SummaryTransform transform, boolean rawAsBase64) { this.name = name; - this.type = convertDataType(type, transform); + this.type = convertDataType(type, transform, rawAsBase64); } public String getName() { return name; } @@ -65,7 +78,7 @@ public class SummaryClassField { public Type getType() { return type; } /** Converts to the right summary field type from a field datatype and a transform*/ - public static Type convertDataType(DataType fieldType, SummaryTransform transform) { + public static Type convertDataType(DataType fieldType, SummaryTransform transform, boolean rawAsBase64) { FieldValue fval = fieldType.createFieldValue(); if (fval instanceof StringFieldValue) { if (transform != null && transform.equals(SummaryTransform.RANKFEATURES)) { @@ -90,7 +103,7 @@ public class SummaryClassField { } else if (fval instanceof ByteFieldValue) { return Type.BYTE; } else if (fval instanceof Raw) { - return Type.DATA; + return rawAsBase64 ? Type.RAW : Type.DATA; } else if (fval instanceof Struct) { return Type.JSONSTRING; } else if (fval instanceof PredicateFieldValue) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ConstantTensorTransformer.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ConstantTensorTransformer.java index 6991e2b978b..31025b0511d 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ConstantTensorTransformer.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ConstantTensorTransformer.java @@ -49,11 +49,16 @@ public class ConstantTensorTransformer extends ExpressionTransformer<RankProfile } private ExpressionNode transformConstantReference(ReferenceNode node, RankProfileTransformContext context) { + String constantName = node.getName(); Reference constantReference = node.reference(); - if ( ! FeatureNames.isConstantFeature(constantReference) && constantReference.isIdentifier()) - constantReference = FeatureNames.asConstantFeature(node.getName()); - - Value value = context.constants().get(node.getName()); + if (FeatureNames.isConstantFeature(constantReference)) { + constantName = constantReference.simpleArgument().orElse(null); + } else if (constantReference.isIdentifier()) { + constantReference = FeatureNames.asConstantFeature(constantName); + } else { + return node; + } + Value value = context.constants().get(constantName); if (value == null || value.type().rank() == 0) return node; TensorValue tensorValue = (TensorValue)value; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ExactMatch.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ExactMatch.java index d5b22988e36..d03dfba2863 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ExactMatch.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ExactMatch.java @@ -62,9 +62,9 @@ public class ExactMatch extends Processor { && ! field.getMatching().getExactMatchTerminator().equals("")) { exactTerminator = field.getMatching().getExactMatchTerminator(); } else { - warn(search, field, - "With 'exact' matching, an exact-terminator is needed " + - "(using '" + exactTerminator +"' as terminator)"); + info(search, field, + "With 'exact' matching, an exact-terminator is needed," + + " using default value '" + exactTerminator +"' as terminator"); } field.addQueryCommand("exact " + exactTerminator); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java b/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java index 5aeedb7ceb0..ef041d06978 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.model; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.api.HostInfo; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeResources; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java index 0b0f466342b..e080ce43730 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java @@ -9,6 +9,7 @@ import com.yahoo.config.model.ConfigModelContext.ApplicationType; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.model.AbstractService; import com.yahoo.vespa.model.ConfigProxy; @@ -59,11 +60,13 @@ public class Admin extends AbstractConfigProducer<Admin> implements Serializable private Logserver logserver; private LogForwarder.Config logForwarderConfig = null; + private boolean logForwarderIncludeAdmin = false; private ApplicationType applicationType = ApplicationType.DEFAULT; - public void setLogForwarderConfig(LogForwarder.Config cfg) { + public void setLogForwarderConfig(LogForwarder.Config cfg, boolean includeAdmin) { this.logForwarderConfig = cfg; + this.logForwarderIncludeAdmin = includeAdmin; } /** @@ -216,7 +219,8 @@ public class Admin extends AbstractConfigProducer<Admin> implements Serializable if (slobroks.isEmpty()) // TODO: Move to caller slobroks.addAll(createDefaultSlobrokSetup(deployState.getDeployLogger())); - addMetricsProxyCluster(hosts, deployState); + if (! deployState.isHosted() || ! deployState.getProperties().applicationId().instance().isTester()) + addMetricsProxyCluster(hosts, deployState); for (HostResource host : hosts) { if (!host.getHost().runsConfigServer()) { @@ -243,7 +247,18 @@ public class Admin extends AbstractConfigProducer<Admin> implements Serializable addConfigProxy(deployState.getDeployLogger(), host); addFileDistribution(host); if (logForwarderConfig != null) { - addLogForwarder(deployState.getDeployLogger(), host); + boolean actuallyAdd = true; + var membership = host.spec().membership(); + if (membership.isPresent()) { + var clustertype = membership.get().cluster().type(); + // XXX should skip only if this.isHostedVespa is true? + if (clustertype == ClusterSpec.Type.admin) { + actuallyAdd = logForwarderIncludeAdmin; + } + } + if (actuallyAdd) { + addLogForwarder(deployState.getDeployLogger(), host); + } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java index 89993780869..dbff597b251 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java @@ -1,7 +1,6 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin.clustercontroller; -import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.api.Reindexing; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; @@ -19,14 +18,12 @@ import java.util.Optional; */ public class ClusterControllerContainerCluster extends ContainerCluster<ClusterControllerContainer> { - private final ModelContext.FeatureFlags featureFlags; private final ReindexingContext reindexingContext; public ClusterControllerContainerCluster( AbstractConfigProducer<?> parent, String subId, String name, DeployState deployState) { super(parent, subId, name, deployState, false); addDefaultHandlersWithVip(); - this.featureFlags = deployState.featureFlags(); this.reindexingContext = createReindexingContext(deployState); setJvmGCOptions(deployState.getProperties().jvmGCOptions(Optional.of(ClusterSpec.Type.admin))); } @@ -40,8 +37,7 @@ public class ClusterControllerContainerCluster extends ContainerCluster<ClusterC public void getConfig(QrStartConfig.Builder builder) { super.getConfig(builder); - builder.jvm - .heapsize(featureFlags.clusterControllerMaxHeapSizeInMb()); + builder.jvm.heapsize(128); } public ReindexingContext reindexingContext() { return reindexingContext; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index 7929dc1e93f..034bf772ffc 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -183,6 +183,7 @@ public class VespaMetricSet { metrics.add(new Metric("athenz-tenant-cert.expiry.seconds.last")); metrics.add(new Metric("jdisc.http.request.prematurely_closed.rate")); + addMetric(metrics, "jdisc.http.request.requests_per_connection", List.of("sum", "count", "min", "max", "average")); metrics.add(new Metric("http.status.1xx.rate")); metrics.add(new Metric("http.status.2xx.rate")); @@ -239,6 +240,14 @@ public class VespaMetricSet { metrics.add(new Metric("cluster-controller.stopping.count.last")); metrics.add(new Metric("cluster-controller.up.count.last")); metrics.add(new Metric("cluster-controller.cluster-state-change.count")); + metrics.add(new Metric("cluster-controller.busy-tick-time-ms.last")); + metrics.add(new Metric("cluster-controller.busy-tick-time-ms.max")); + metrics.add(new Metric("cluster-controller.busy-tick-time-ms.sum")); + metrics.add(new Metric("cluster-controller.busy-tick-time-ms.count")); + metrics.add(new Metric("cluster-controller.idle-tick-time-ms.last")); + metrics.add(new Metric("cluster-controller.idle-tick-time-ms.max")); + metrics.add(new Metric("cluster-controller.idle-tick-time-ms.sum")); + metrics.add(new Metric("cluster-controller.idle-tick-time-ms.count")); metrics.add(new Metric("cluster-controller.is-master.last")); metrics.add(new Metric("cluster-controller.remote-task-queue.size.last")); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java index d2b465e9d02..d22affaf5a3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java @@ -3,15 +3,17 @@ package com.yahoo.vespa.model.application.validation; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.HostSpec; +import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.SystemName; import com.yahoo.vespa.model.VespaModel; +import org.jetbrains.annotations.NotNull; import java.math.BigDecimal; import java.util.Locale; import java.util.Map; -import java.util.Objects; +import java.util.Set; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -23,6 +25,7 @@ import java.util.stream.Collectors; public class QuotaValidator extends Validator { private static final Logger log = Logger.getLogger(QuotaValidator.class.getName()); + private static final Capacity zeroCapacity = Capacity.from(new ClusterResources(0, 0, NodeResources.zero())); @Override public void validate(VespaModel model, DeployState deployState) { @@ -32,18 +35,35 @@ public class QuotaValidator extends Validator { } private void validateBudget(BigDecimal budget, VespaModel model, SystemName systemName) { - var spend = model.allocatedHosts().getHosts().stream() + + var maxSpend = model.allClusters().stream() + .filter(id -> !adminClusterIds(model).contains(id)) + .map(id -> model.provisioned().all().getOrDefault(id, zeroCapacity)) + .mapToDouble(c -> c.maxResources().cost()) + .sum(); + + var actualSpend = model.allocatedHosts().getHosts().stream() .filter(hostSpec -> hostSpec.membership().get().cluster().type() != ClusterSpec.Type.admin) .mapToDouble(hostSpec -> hostSpec.advertisedResources().cost()) .sum(); - if (Math.abs(spend) < 0.01) { + if (Math.abs(actualSpend) < 0.01) { log.warning("Deploying application " + model.applicationPackage().getApplicationId() + " with zero budget use. This is suspicious, but not blocked"); return; } - throwIfBudgetNegative(spend, budget, systemName); - throwIfBudgetExceeded(spend, budget, systemName); + throwIfBudgetNegative(actualSpend, budget, systemName); + throwIfBudgetExceeded(actualSpend, budget, systemName); + throwIfBudgetExceeded(maxSpend, budget, systemName); + } + + @NotNull + private Set<ClusterSpec.Id> adminClusterIds(VespaModel model) { + return model.allocatedHosts().getHosts().stream() + .map(hostSpec -> hostSpec.membership().orElseThrow().cluster()) + .filter(cluster -> cluster.type() == ClusterSpec.Type.admin) + .map(ClusterSpec::id) + .collect(Collectors.toUnmodifiableSet()); } /** Check that all clusters in the application do not exceed the quota max cluster size. */ diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaReindexAction.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaReindexAction.java index 8b4060e7d19..c2d23844687 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaReindexAction.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaReindexAction.java @@ -46,7 +46,7 @@ public class VespaReindexAction extends VespaConfigChangeAction implements Confi return new VespaReindexAction(clusterId(), validationId, newMessage, newServices, documentType); } - @Override public Optional<ValidationId> validationId() { return Optional.of(validationId); } + @Override public Optional<ValidationId> validationId() { return Optional.ofNullable(validationId); } @Override public String getDocumentType() { return documentType; } @Override public boolean ignoreForInternalRedeploy() { return false; } @Override public String toString() { return super.toString() + ", documentType='" + documentType + "'"; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidator.java index 91e370211f1..20872bcf326 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidator.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.model.application.validation.change.search; import com.yahoo.config.application.api.ValidationId; -import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.document.ImmutableSDField; @@ -13,8 +12,8 @@ import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; import com.yahoo.vespa.model.application.validation.change.VespaConfigChangeAction; import com.yahoo.vespa.model.application.validation.change.VespaReindexAction; -import java.time.Instant; import java.util.ArrayList; +import java.util.LinkedHashSet; import java.util.List; import java.util.Optional; @@ -37,12 +36,18 @@ public class IndexingScriptChangeValidator { public List<VespaConfigChangeAction> validate() { List<VespaConfigChangeAction> result = new ArrayList<>(); - for (ImmutableSDField nextField : nextSearch.allConcreteFields()) { + for (ImmutableSDField nextField : new LinkedHashSet<>(nextSearch.allConcreteFields())) { String fieldName = nextField.getName(); ImmutableSDField currentField = currentSearch.getConcreteField(fieldName); if (currentField != null) { validateScripts(currentField, nextField).ifPresent(r -> result.add(r)); } + else if (nextField.isExtraField()) { + result.add(VespaReindexAction.of(id, + null, + "Non-document field '" + nextField.getName() + + "' added; this may be populated by reindexing")); + } } return result; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java index c5edfb9bbf7..963d2dde7fc 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java @@ -114,14 +114,14 @@ public abstract class DomAdminBuilderBase extends VespaDomBuilder.DomConfigProdu void addLogForwarders(ModelElement logForwardingElement, Admin admin) { if (logForwardingElement == null) return; - + boolean alsoForAdminCluster = logForwardingElement.booleanAttribute("include-admin"); for (ModelElement e : logForwardingElement.children("splunk")) { LogForwarder.Config cfg = LogForwarder.cfg() .withSplunkHome(e.stringAttribute("splunk-home")) .withDeploymentServer(e.stringAttribute("deployment-server")) .withClientName(e.stringAttribute("client-name")) .withPhoneHomeInterval(e.integerAttribute("phone-home-interval")); - admin.setLogForwarderConfig(cfg); + admin.setLogForwarderConfig(cfg, alsoForAdminCluster); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomFederationSearcherBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomFederationSearcherBuilder.java index 3a4e8a70613..36bf792ee82 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomFederationSearcherBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomFederationSearcherBuilder.java @@ -43,7 +43,6 @@ public class DomFederationSearcherBuilder extends VespaDomBuilder.DomConfigProdu return XML.getChild(searcherSpec, "source-set") != null; } - private List<FederationSearcherModel.TargetSpec> readSources(Element searcherSpec) { List<FederationSearcherModel.TargetSpec> sources = new ArrayList<>(); for (Element source : XML.getChildren(searcherSpec, "source")) { @@ -76,14 +75,14 @@ public class DomFederationSearcherBuilder extends VespaDomBuilder.DomConfigProdu } @Override - protected FederationSearcher doBuild(DeployState deployState, AbstractConfigProducer ancestor, Element searcherElement) { + protected FederationSearcher doBuild(DeployState deployState, AbstractConfigProducer<?> ancestor, Element searcherElement) { FederationSearcherModel model = new FederationSearcherModelBuilder(searcherElement).build(); Optional<Component> targetSelector = buildTargetSelector(deployState, ancestor, searcherElement, model.getComponentId()); return new FederationSearcher(model, targetSelector); } - private Optional<Component> buildTargetSelector(DeployState deployState, AbstractConfigProducer ancestor, Element searcherElement, ComponentId namespace) { + private Optional<Component> buildTargetSelector(DeployState deployState, AbstractConfigProducer<?> ancestor, Element searcherElement, ComponentId namespace) { Element targetSelectorElement = XML.getChild(searcherElement, "target-selector"); if (targetSelectorElement == null) return Optional.empty(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/SearchChainsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/SearchChainsBuilder.java index 0106123666d..9fb19efbf75 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/SearchChainsBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/SearchChainsBuilder.java @@ -16,9 +16,10 @@ import java.util.List; import java.util.Map; /** + * Creates top level search chains(searchchain, provider) from xml. + * * @author Tony Vaagenes * @author gjoranv - * Creates top level search chains(searchchain, provider) from xml. */ public class SearchChainsBuilder extends ChainsBuilder<Searcher<?>, SearchChain> { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index f0c62664988..4e78f44d0fe 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -23,6 +23,7 @@ import com.yahoo.jdisc.http.ServletPathsConfig; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.search.config.QrStartConfig; import com.yahoo.vespa.config.search.RankProfilesConfig; +import com.yahoo.vespa.config.search.core.OnnxModelsConfig; import com.yahoo.vespa.config.search.core.RankingConstantsConfig; import com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainer; import com.yahoo.vespa.model.container.component.BindingPattern; @@ -56,6 +57,7 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat QrStartConfig.Producer, RankProfilesConfig.Producer, RankingConstantsConfig.Producer, + OnnxModelsConfig.Producer, ServletPathsConfig.Producer, ContainerMbusConfig.Producer, MetricsProxyApiConfig.Producer, @@ -227,6 +229,11 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat } @Override + public void getConfig(OnnxModelsConfig.Builder builder) { + if (modelEvaluation != null) modelEvaluation.getConfig(builder); + } + + @Override public void getConfig(ContainerMbusConfig.Builder builder) { if (mbusParams != null) { if (mbusParams.maxConcurrentFactor != null) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index 8868c55becb..d130b08d083 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -188,6 +188,7 @@ public abstract class ContainerCluster<CONTAINER extends Container> addSimpleComponent("com.yahoo.container.handler.VipStatus"); addSimpleComponent(com.yahoo.container.handler.ClustersStatus.class.getName()); addSimpleComponent("com.yahoo.container.jdisc.DisabledConnectionLogProvider"); + addSimpleComponent(com.yahoo.jdisc.http.server.jetty.Janitor.class); addJaxProviders(); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java index 72f1921e6a2..510d2fe3d99 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java @@ -5,6 +5,7 @@ import ai.vespa.models.evaluation.ModelsEvaluator; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.searchdefinition.derived.RankProfileList; import com.yahoo.vespa.config.search.RankProfilesConfig; +import com.yahoo.vespa.config.search.core.OnnxModelsConfig; import com.yahoo.vespa.config.search.core.RankingConstantsConfig; import com.yahoo.vespa.model.container.component.Handler; import com.yahoo.vespa.model.container.component.SystemBindingPattern; @@ -17,7 +18,10 @@ import java.util.Objects; * * @author bratseth */ -public class ContainerModelEvaluation implements RankProfilesConfig.Producer, RankingConstantsConfig.Producer { +public class ContainerModelEvaluation implements RankProfilesConfig.Producer, + RankingConstantsConfig.Producer, + OnnxModelsConfig.Producer +{ private final static String BUNDLE_NAME = "model-evaluation"; private final static String EVALUATOR_NAME = ModelsEvaluator.class.getName(); @@ -35,6 +39,7 @@ public class ContainerModelEvaluation implements RankProfilesConfig.Producer, Ra public void prepare(List<ApplicationContainer> containers) { rankProfileList.sendConstantsTo(containers); + rankProfileList.sendOnnxModelsTo(containers); } @Override @@ -47,6 +52,11 @@ public class ContainerModelEvaluation implements RankProfilesConfig.Producer, Ra rankProfileList.getConfig(builder); } + @Override + public void getConfig(OnnxModelsConfig.Builder builder) { + rankProfileList.getConfig(builder); + } + public static Handler<?> getHandler() { Handler<?> handler = new Handler<>(new ComponentModel(REST_HANDLER_NAME, null, BUNDLE_NAME)); handler.addServerBindings( diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/DocprocChain.java b/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/DocprocChain.java index ee246b5e485..2b2b17c76c3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/DocprocChain.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/DocprocChain.java @@ -23,8 +23,9 @@ public class DocprocChain extends Chain<DocumentProcessor> { } /** - * The field name schema map that applies to this whole chain - * @return doctype,from → to + * The field name schema map that applies to this whole chain. + * + * @return doctype, from → to */ public Map<Pair<String,String>,String> fieldNameSchemaMap() { return fieldNameSchemaMap; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java index ce79a124e81..d13709114bf 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java @@ -14,8 +14,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; -import static com.yahoo.component.ComponentSpecification.fromString; - /** * @author Einar M R Rosenvinge * @author bjorncs diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/FederationSearcher.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/FederationSearcher.java index ceb48732116..6b4cb003cda 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/FederationSearcher.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/FederationSearcher.java @@ -11,7 +11,12 @@ import com.yahoo.search.federation.FederationConfig; import com.yahoo.search.searchchain.model.federation.FederationSearcherModel.TargetSpec; import com.yahoo.vespa.model.container.component.Component; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; /** * Config producer for the FederationSearcher. @@ -26,8 +31,8 @@ public class FederationSearcher extends Searcher<FederationSearcherModel> implem * Generates config for a single search chain contained in a target. */ private static final class SearchChainConfig { + private final SearchChain searchChain; - //Zero if not applicable final ComponentId providerId; final FederationOptions targetOptions; final List<String> documentTypes; @@ -61,6 +66,7 @@ public class FederationSearcher extends Searcher<FederationSearcherModel> implem * which can be federated to as a single entity. */ private static abstract class Target { + final ComponentId id; final FederationOptions targetOptions; @@ -79,41 +85,36 @@ public class FederationSearcher extends Searcher<FederationSearcherModel> implem } protected abstract void getSearchChainsConfig(FederationConfig.Target.Builder tb); + } private static class SearchChainTarget extends Target { + private final SearchChainConfig searchChainConfig; - public SearchChainTarget(SearchChain searchChain, - FederationOptions targetOptions) { + public SearchChainTarget(SearchChain searchChain, FederationOptions targetOptions) { super(searchChain.getComponentId(), targetOptions); - searchChainConfig = new SearchChainConfig( - searchChain, - null, - targetOptions, - searchChain.getDocumentTypes()); + searchChainConfig = new SearchChainConfig(searchChain, null, targetOptions, searchChain.getDocumentTypes()); } @Override protected void getSearchChainsConfig(FederationConfig.Target.Builder tB) { tB.searchChain(searchChainConfig.getSearchChainConfig()); } + } private static class SourceGroupTarget extends Target { + private final SearchChainConfig leaderConfig; - private final List<SearchChainConfig> participantsConfig = - new ArrayList<>(); + private final List<SearchChainConfig> participantsConfig = new ArrayList<>(); - public SourceGroupTarget(SourceGroup group, - FederationOptions targetOptions) { + public SourceGroupTarget(SourceGroup group, FederationOptions targetOptions) { super(group.getComponentId(), applyDefaultSourceGroupOptions(targetOptions)); leaderConfig = createConfig(group.leader(), targetOptions); - for (Source participant : group.participants()) { - participantsConfig.add( - createConfig(participant, targetOptions)); - } + for (Source participant : group.participants()) + participantsConfig.add(createConfig(participant, targetOptions)); } private static FederationOptions applyDefaultSourceGroupOptions(FederationOptions targetOptions) { @@ -121,64 +122,49 @@ public class FederationSearcher extends Searcher<FederationSearcherModel> implem return targetOptions.inherit(defaultSourceGroupOption); } - private SearchChainConfig createConfig(Source source, - FederationOptions targetOptions) { - return new SearchChainConfig( - source, - source.getParentProvider().getComponentId(), - targetOptions, - source.getDocumentTypes()); + private SearchChainConfig createConfig(Source source, FederationOptions targetOptions) { + return new SearchChainConfig(source, + source.getParentProvider().getComponentId(), + targetOptions, + source.getDocumentTypes()); } @Override protected void getSearchChainsConfig(FederationConfig.Target.Builder tB) { tB.searchChain(leaderConfig.getSearchChainConfig()); - for (SearchChainConfig participant : participantsConfig) { + for (SearchChainConfig participant : participantsConfig) tB.searchChain(participant.getSearchChainConfig()); - } } } private static class TargetResolver { + final ComponentRegistry<SearchChain> searchChainRegistry; final SourceGroupRegistry sourceGroupRegistry; - /** - * @return true if searchChain.id newer than sourceGroup.id - */ - private boolean newerVersion(SearchChain searchChain, - SourceGroup sourceGroup) { - if (searchChain == null || sourceGroup == null) { - return false; - } else { - return newerVersion(searchChain.getComponentId(), sourceGroup.getComponentId()); - } + /** Returns true if searchChain.id newer than sourceGroup.id */ + private boolean newerVersion(SearchChain searchChain, SourceGroup sourceGroup) { + if (searchChain == null || sourceGroup == null) return false; + return newerVersion(searchChain.getComponentId(), sourceGroup.getComponentId()); } - /** - * @return true if a newer than b - */ + /** Returns true if a newer than b */ private boolean newerVersion(ComponentId a, ComponentId b) { return a.compareTo(b) > 0; } - - TargetResolver(ComponentRegistry<SearchChain> searchChainRegistry, - SourceGroupRegistry sourceGroupRegistry) { + TargetResolver(ComponentRegistry<SearchChain> searchChainRegistry, SourceGroupRegistry sourceGroupRegistry) { this.searchChainRegistry = searchChainRegistry; this.sourceGroupRegistry = sourceGroupRegistry; } Target resolve(FederationSearcherModel.TargetSpec specification) { - SearchChain searchChain = searchChainRegistry.getComponent( - specification.sourceSpec); - SourceGroup sourceGroup = sourceGroupRegistry.getComponent( - specification.sourceSpec); + SearchChain searchChain = searchChainRegistry.getComponent(specification.sourceSpec); + SourceGroup sourceGroup = sourceGroupRegistry.getComponent(specification.sourceSpec); if (searchChain == null && sourceGroup == null) { return null; - } else if (sourceGroup == null || - newerVersion(searchChain, sourceGroup)) { + } else if (sourceGroup == null || newerVersion(searchChain, sourceGroup)) { return new SearchChainTarget(searchChain, specification.federationOptions); } else { return new SourceGroupTarget(sourceGroup, specification.federationOptions); @@ -186,26 +172,21 @@ public class FederationSearcher extends Searcher<FederationSearcherModel> implem } } - private final Map<ComponentId, Target> resolvedTargets = - new LinkedHashMap<>(); + private final Map<ComponentId, Target> resolvedTargets = new LinkedHashMap<>(); public FederationSearcher(FederationSearcherModel searcherModel, Optional<Component> targetSelector) { super(searcherModel); this.targetSelector = targetSelector; - if (targetSelector.isPresent()) - addChild(targetSelector.get()); + targetSelector.ifPresent(selector -> addChild(selector)); } @Override public void getConfig(FederationConfig.Builder builder) { - for (Target target : resolvedTargets.values()) { + for (Target target : resolvedTargets.values()) builder.target(target.getTargetConfig()); - } - if (targetSelector.isPresent()) { - builder.targetSelector(targetSelector.get().getGlobalComponentId().stringValue()); - } + targetSelector.ifPresent(selector -> builder.targetSelector(selector.getGlobalComponentId().stringValue())); } @Override @@ -213,10 +194,8 @@ public class FederationSearcher extends Searcher<FederationSearcherModel> implem initialize(getSearchChains().allChains(), getSearchChains().allSourceGroups()); } - void initialize(ComponentRegistry<SearchChain> searchChainRegistry, - SourceGroupRegistry sourceGroupRegistry) { - TargetResolver targetResolver = new TargetResolver( - searchChainRegistry, sourceGroupRegistry); + void initialize(ComponentRegistry<SearchChain> searchChainRegistry, SourceGroupRegistry sourceGroupRegistry) { + TargetResolver targetResolver = new TargetResolver(searchChainRegistry, sourceGroupRegistry); addSourceTargets(targetResolver, model.targets); @@ -229,16 +208,14 @@ public class FederationSearcher extends Searcher<FederationSearcherModel> implem Target target = targetResolver.resolve(targetSpec); if (target == null) { - throw new RuntimeException("Can't find source " + - targetSpec.sourceSpec + - " used as a source for federation '" + - getComponentId() + "'"); + throw new RuntimeException("Can't find source " + targetSpec.sourceSpec + + " used as a source for federation '" + getComponentId() + "'"); } Target duplicate = resolvedTargets.put(target.id, target); if (duplicate != null && !duplicate.targetOptions.equals(target.targetOptions)) { - throw new RuntimeException("Search chain " + target.id + " added twice with different federation options" - + " to the federation searcher " + getComponentId()); + throw new RuntimeException("Search chain " + target.id + " added twice with different federation options" + + " to the federation searcher " + getComponentId()); } } } @@ -248,23 +225,21 @@ public class FederationSearcher extends Searcher<FederationSearcherModel> implem for (GenericTarget genericTarget : defaultTargets(searchChainRegistry.allComponents())) { ComponentSpecification specification = genericTarget.getComponentId().toSpecification(); - //Can't use genericTarget directly, as it might be part of a source group. + // Can't use genericTarget directly, as it might be part of a source group. Target federationTarget = targetResolver.resolve(new TargetSpec(specification, new FederationOptions())); - //Do not replace manually added sources, as they might have manually configured federation options + // Do not replace manually added sources, as they might have manually configured federation options if (!resolvedTargets.containsKey(federationTarget.id)) resolvedTargets.put(federationTarget.id, federationTarget); } } - private static List<GenericTarget> defaultTargets(Collection<SearchChain> allSearchChains) { - Collection<Provider> providers = - CollectionUtil.filter(allSearchChains, Provider.class); + Collection<Provider> providers = CollectionUtil.filter(allSearchChains, Provider.class); List<GenericTarget> defaultTargets = new ArrayList<>(); - for (Provider provider : providers) { + for (Provider provider : providers) defaultTargets.addAll(provider.defaultFederationTargets()); - } return defaultTargets; } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/Provider.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/Provider.java index ee4edf3fd8c..10e0f3e55da 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/Provider.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/Provider.java @@ -7,6 +7,7 @@ import com.yahoo.vespa.model.container.component.ConfigProducerGroup; import java.util.Arrays; import java.util.Collection; +import java.util.List; /** * Base config producer for search chains that communicate with backends. @@ -15,7 +16,7 @@ import java.util.Collection; */ public class Provider extends GenericTarget { - private ConfigProducerGroup<Source> sources; + private final ConfigProducerGroup<Source> sources; public Provider(ChainSpecification specWithoutInnerSearchers, FederationOptions federationOptions) { super(specWithoutInnerSearchers, federationOptions); @@ -37,9 +38,10 @@ public class Provider extends GenericTarget { public Collection<? extends GenericTarget> defaultFederationTargets() { if (sources.getComponents().isEmpty()) { - return Arrays.asList(this); + return List.of(this); } else { return sources.getComponents(); } } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index 1dd5074aedb..18580249ddc 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -64,11 +64,8 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> private final Map<StorageGroup, NodeSpec> groupToSpecMap = new LinkedHashMap<>(); private Optional<ResourceLimits> resourceLimits = Optional.empty(); private final ProtonConfig.Indexing.Optimize.Enum feedSequencerType; - private final int maxPendingMoveOps; private final double defaultFeedConcurrency; - private final boolean useBucketExecutorForLidSpaceCompact; - private final boolean useBucketExecutorForBucketMove; - private final double defaultMaxDeadBytesRatio; + private final boolean useBucketExecutorForPruneRemoved; /** Whether the nodes of this cluster also hosts a container cluster in a hosted system */ private final boolean combined; @@ -211,12 +208,9 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> this.syncTransactionLog = syncTransactionLog; this.combined = combined; - maxPendingMoveOps = featureFlags.maxPendingMoveOps(); feedSequencerType = convertFeedSequencerType(featureFlags.feedSequencerType()); defaultFeedConcurrency = featureFlags.feedConcurrency(); - useBucketExecutorForLidSpaceCompact = featureFlags.useBucketExecutorForLidSpaceCompact(); - useBucketExecutorForBucketMove = featureFlags.useBucketExecutorForBucketMove(); - defaultMaxDeadBytesRatio = featureFlags.maxDeadBytesRatio(); + useBucketExecutorForPruneRemoved = featureFlags.useBucketExecutorForPruneRemoved(); } public void setVisibilityDelay(double delay) { @@ -383,7 +377,6 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> .configid(getConfigId()) .visibilitydelay(visibilityDelay) .global(globalDocType); - ddbB.allocation.max_dead_bytes_ratio(defaultMaxDeadBytesRatio); if (hasIndexingModeStreaming(type)) { hasAnyNonIndexedCluster = true; @@ -434,9 +427,7 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> } else { builder.indexing.optimize(feedSequencerType); } - builder.maintenancejobs.maxoutstandingmoveops(maxPendingMoveOps); - builder.lidspacecompaction.usebucketexecutor(useBucketExecutorForLidSpaceCompact); - builder.bucketmove.usebucketexecutor(useBucketExecutorForBucketMove); + builder.pruneremoveddocuments.usebucketexecutor(useBucketExecutorForPruneRemoved); } private boolean isGloballyDistributed(NewDocumentType docType) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java index f106b1f7bd5..c764b5ab449 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java @@ -54,7 +54,7 @@ public class StorageGroup { * * @param isHosted true if this is in a hosted setup * @param name the name of this group - * @param index the distribution-key index og this group + * @param index the distribution-key index of this group * @param partitions the distribution strategy to use to distribute content to subgroups or empty * (meaning that the "*" distribution will be used) only if this is a leaf group * (having nodes, not subgroups as children). @@ -162,10 +162,10 @@ public class StorageGroup { } /** Returns the total number of nodes below this group */ - public int countNodes() { - int nodeCount = nodes.size(); + public int countNodes(boolean includeRetired) { + int nodeCount = (int)nodes.stream().filter(node -> includeRetired || ! node.isRetired()).count(); for (StorageGroup group : subgroups) - nodeCount += group.countNodes(); + nodeCount += group.countNodes(includeRetired); return nodeCount; } @@ -220,7 +220,7 @@ public class StorageGroup { ? groupBuilder.buildHosted(deployState, owner, Optional.empty()) : groupBuilder.buildNonHosted(deployState, owner, Optional.empty()); Redundancy redundancy = redundancyBuilder.build(owner.getName(), owner.isHosted(), storageGroup.subgroups.size(), - storageGroup.getNumberOfLeafGroups(), storageGroup.countNodes(), + storageGroup.getNumberOfLeafGroups(), storageGroup.countNodes(false), maxRedundancy); owner.setRedundancy(redundancy); if (storageGroup.partitions.isEmpty() && (redundancy.groups() > 1)) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexingDocprocChain.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexingDocprocChain.java index 2c1d979e2c4..8fe6b51f2b4 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexingDocprocChain.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexingDocprocChain.java @@ -2,14 +2,12 @@ package com.yahoo.vespa.model.search; import com.yahoo.component.ComponentId; -import com.yahoo.component.ComponentSpecification; import com.yahoo.component.chain.Phase; import com.yahoo.component.chain.model.ChainSpecification; import com.yahoo.vespa.configdefinition.SpecialtokensConfig; import com.yahoo.vespa.model.container.docproc.DocprocChain; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Set; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/TransactionLogServer.java b/config-model/src/main/java/com/yahoo/vespa/model/search/TransactionLogServer.java index ed12a161805..82477b811d5 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/TransactionLogServer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/TransactionLogServer.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.search; -import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.searchlib.TranslogserverConfig; import com.yahoo.config.model.producer.AbstractConfigProducer; @@ -15,18 +14,9 @@ import org.w3c.dom.Element; */ public class TransactionLogServer extends AbstractService { - private static final long serialVersionUID = 1L; - - private static TranslogserverConfig.Compression.Type.Enum convertCompressionType(String type) { - try { - return TranslogserverConfig.Compression.Type.Enum.valueOf(type); - } catch (Throwable t) { - return TranslogserverConfig.Compression.Type.NONE; - } - } - private final Boolean useFsync; - public TransactionLogServer(AbstractConfigProducer searchNode, String clusterName, Boolean useFsync) { + + public TransactionLogServer(AbstractConfigProducer<?> searchNode, String clusterName, Boolean useFsync) { super(searchNode, "transactionlogserver"); portsMeta.on(0).tag("tls"); this.useFsync = useFsync; @@ -44,7 +34,7 @@ public class TransactionLogServer extends AbstractService { } @Override - protected TransactionLogServer doBuild(DeployState deployState, AbstractConfigProducer ancestor, Element producerSpec) { + protected TransactionLogServer doBuild(DeployState deployState, AbstractConfigProducer<?> ancestor, Element producerSpec) { return new TransactionLogServer(ancestor, clusterName, useFsync); } diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj index 88c45a18e4e..8f99c388fb1 100644 --- a/config-model/src/main/javacc/SDParser.jj +++ b/config-model/src/main/javacc/SDParser.jj @@ -237,6 +237,7 @@ TOKEN : | < RANKPROFILE: "rank-profile" > | < RANKDEGRADATIONFREQ: "rank-degradation-frequency" > | < RANKDEGRADATION: "rank-degradation" > +| < RAW_AS_BASE64_IN_SUMMARY: "raw-as-base64-in-summary" > | < RPBINSIZE: "doc-frequency" > | < RPBINLOW: "min-fullrank-docs"> | < RPPOSBINSIZE: "occurrences-per-doc" > @@ -451,6 +452,7 @@ Search rootSchema(String dir) : Object rootSchemaItem(Search search) : { } { ( document(search) + | rawAsBase64(search) | documentSummary(search) | field(null, search) | index(search, null) @@ -561,6 +563,12 @@ Object documentBody(SDDocumentType document, Search search) : { return null; } } +void rawAsBase64(Search search) : +{} +{ + <RAW_AS_BASE64_IN_SUMMARY> { search.enableRawAsBase64(); } +} + /** * Consumes a document head block. * diff --git a/config-model/src/main/resources/schema/admin.rnc b/config-model/src/main/resources/schema/admin.rnc index 784fb82d319..a75b51a567a 100644 --- a/config-model/src/main/resources/schema/admin.rnc +++ b/config-model/src/main/resources/schema/admin.rnc @@ -105,6 +105,7 @@ ClusterControllers = element cluster-controllers { } LogForwarding = element logforwarding { + attribute include-admin { xsd:boolean }? & element splunk { attribute splunk-home { xsd:string }? & attribute deployment-server { xsd:string } & diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java index 96e228ca1f7..79e3e869b52 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java @@ -35,7 +35,6 @@ import com.yahoo.vespa.model.test.VespaModelTester; import com.yahoo.vespa.model.test.utils.ApplicationPackageUtils; import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; import com.yahoo.yolean.Exceptions; -import org.junit.Ignore; import org.junit.Test; import java.io.StringReader; @@ -896,6 +895,89 @@ public class ModelProvisioningTest { } @Test + public void testLogForwarderNotInAdminCluster() { + String services = + "<?xml version='1.0' encoding='utf-8' ?>\n" + + "<services>" + + " <admin version='4.0'>" + + " <logservers>" + + " <nodes count='1' dedicated='true'/>" + + " </logservers>" + + " <logforwarding>" + + " <splunk deployment-server='bardeplserv:123' client-name='barclinam' phone-home-interval='987' />" + + " </logforwarding>" + + " </admin>" + + " <container version='1.0' id='foo'>" + + " <nodes count='1'/>" + + " </container>" + + "</services>"; + + int numberOfHosts = 2; + VespaModelTester tester = new VespaModelTester(); + tester.addHosts(numberOfHosts+1); + + VespaModel model = tester.createModel(Zone.defaultZone(), services, true); + assertThat(model.getRoot().hostSystem().getHosts().size(), is(numberOfHosts)); + + Admin admin = model.getAdmin(); + Logserver logserver = admin.getLogserver(); + HostResource hostResource = logserver.getHostResource(); + + assertNotNull(hostResource.getService("logserver")); + assertNull(hostResource.getService("container")); + assertNull(hostResource.getService("logforwarder")); + + var clist = model.getContainerClusters().get("foo").getContainers(); + assertThat(clist.size(), is(1)); + hostResource = clist.get(0).getHostResource(); + assertNull(hostResource.getService("logserver")); + assertNotNull(hostResource.getService("container")); + assertNotNull(hostResource.getService("logforwarder")); + } + + + @Test + public void testLogForwarderInAdminCluster() { + String services = + "<?xml version='1.0' encoding='utf-8' ?>\n" + + "<services>" + + " <admin version='4.0'>" + + " <logservers>" + + " <nodes count='1' dedicated='true'/>" + + " </logservers>" + + " <logforwarding include-admin='true'>" + + " <splunk deployment-server='bardeplserv:123' client-name='barclinam' phone-home-interval='987' />" + + " </logforwarding>" + + " </admin>" + + " <container version='1.0' id='foo'>" + + " <nodes count='1'/>" + + " </container>" + + "</services>"; + + int numberOfHosts = 2; + VespaModelTester tester = new VespaModelTester(); + tester.addHosts(numberOfHosts+1); + + VespaModel model = tester.createModel(Zone.defaultZone(), services, true); + assertThat(model.getRoot().hostSystem().getHosts().size(), is(numberOfHosts)); + + Admin admin = model.getAdmin(); + Logserver logserver = admin.getLogserver(); + HostResource hostResource = logserver.getHostResource(); + + assertNotNull(hostResource.getService("logserver")); + assertNull(hostResource.getService("container")); + assertNotNull(hostResource.getService("logforwarder")); + + var clist = model.getContainerClusters().get("foo").getContainers(); + assertThat(clist.size(), is(1)); + hostResource = clist.get(0).getHostResource(); + assertNull(hostResource.getService("logserver")); + assertNotNull(hostResource.getService("container")); + assertNotNull(hostResource.getService("logforwarder")); + } + + @Test public void testImplicitLogserverContainer() { String services = "<?xml version='1.0' encoding='utf-8' ?>\n" + @@ -1018,6 +1100,35 @@ public class ModelProvisioningTest { } @Test + public void testRedundancy2DownscaledToOneNodeButOneRetired() { + String services = + "<?xml version='1.0' encoding='utf-8' ?>" + + "<services>" + + " <content version='1.0' id='bar'>" + + " <redundancy>2</redundancy>" + + " <documents>" + + " <document type='type1' mode='index'/>" + + " </documents>" + + " <nodes count='2'/>" + + " </content>" + + "</services>"; + + int numberOfHosts = 3; + VespaModelTester tester = new VespaModelTester(); + tester.addHosts(numberOfHosts); + VespaModel model = tester.createModel(services, false, false, true, "node-1-3-10-03"); + assertEquals(numberOfHosts, model.getRoot().hostSystem().getHosts().size()); + + ContentCluster cluster = model.getContentClusters().get("bar"); + assertEquals(2, cluster.getStorageNodes().getChildren().size()); + assertEquals(1, cluster.redundancy().effectiveInitialRedundancy()); + assertEquals(1, cluster.redundancy().effectiveFinalRedundancy()); + assertEquals(1, cluster.redundancy().effectiveReadyCopies()); + assertEquals(2, cluster.getRootGroup().getNodes().size()); + assertEquals(0, cluster.getRootGroup().getSubgroups().size()); + } + + @Test public void testUsingNodesCountAttributesAndGettingTooFewNodes() { String services = "<?xml version='1.0' encoding='utf-8' ?>" + @@ -1483,7 +1594,7 @@ public class ModelProvisioningTest { assertEquals("We get 1 node per cluster and no admin node apart from the dedicated cluster controller", 3, model.getHosts().size()); assertEquals(1, model.getContainerClusters().size()); assertEquals(1, model.getContainerClusters().get("foo").getContainers().size()); - assertEquals(1, model.getContentClusters().get("bar").getRootGroup().countNodes()); + assertEquals(1, model.getContentClusters().get("bar").getRootGroup().countNodes(true)); assertEquals(1, model.getAdmin().getClusterControllers().getContainers().size()); } @@ -1536,7 +1647,7 @@ public class ModelProvisioningTest { assertEquals(6, model.getRoot().hostSystem().getHosts().size()); assertEquals(3, model.getAdmin().getSlobroks().size()); assertEquals(2, model.getContainerClusters().get("foo").getContainers().size()); - assertEquals(1, model.getContentClusters().get("bar").getRootGroup().countNodes()); + assertEquals(1, model.getContentClusters().get("bar").getRootGroup().countNodes(true)); } @Test @@ -1606,7 +1717,7 @@ public class ModelProvisioningTest { assertEquals(1, model.getRoot().hostSystem().getHosts().size()); assertEquals(1, model.getAdmin().getSlobroks().size()); assertEquals(1, model.getContainerClusters().get("foo").getContainers().size()); - assertEquals(1, model.getContentClusters().get("bar").getRootGroup().countNodes()); + assertEquals(1, model.getContentClusters().get("bar").getRootGroup().countNodes(true)); } /** Recreate the combination used in some factory tests */ @@ -1889,7 +2000,7 @@ public class ModelProvisioningTest { assertTrue("Initial servers are not joining", config.build().server().stream().noneMatch(ZookeeperServerConfig.Server::joining)); } { - VespaModel nextModel = tester.createModel(Zone.defaultZone(), servicesXml.apply(5), true, false, 0, Optional.of(model), new DeployState.Builder()); + VespaModel nextModel = tester.createModel(Zone.defaultZone(), servicesXml.apply(5), true, false, false, 0, Optional.of(model), new DeployState.Builder()); ApplicationContainerCluster cluster = nextModel.getContainerClusters().get("zk"); ZookeeperServerConfig.Builder config = new ZookeeperServerConfig.Builder(); cluster.getContainers().forEach(c -> c.getConfig(config)); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/SummaryTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/SummaryTestCase.java index 91599e6f607..f8d03d3574b 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/SummaryTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/SummaryTestCase.java @@ -6,6 +6,7 @@ import com.yahoo.vespa.documentmodel.DocumentSummary; import com.yahoo.vespa.model.test.utils.DeployLoggerStub; import com.yahoo.vespa.objects.FieldBase; import org.junit.Test; +import static com.yahoo.config.model.test.TestUtil.joinLines; import java.util.Collection; import java.util.List; @@ -25,21 +26,17 @@ public class SummaryTestCase { @Test public void testMemorySummary() throws ParseException { - String sd = - "search memorysummary {\n" + - "\n" + - " document memorysummary {\n" + - "\n" + - " field inmemory type string {\n" + - " indexing: attribute | summary\n" + - " }\n" + - " field ondisk type string {\n" + - " indexing: index # no summary, so ignored\n" + - " }\n" + - "\n" + - " }\n" + - "\n" + - "}"; + String sd = joinLines( + "search memorysummary {", + " document memorysummary {", + " field inmemory type string {", + " indexing: attribute | summary", + " }", + " field ondisk type string {", + " indexing: index # no summary, so ignored", + " }", + " }", + "}"); DeployLoggerStub logger = new DeployLoggerStub(); SearchBuilder.createFromString(sd, logger); assertTrue(logger.entries.isEmpty()); @@ -47,25 +44,21 @@ public class SummaryTestCase { @Test public void testDiskSummary() throws ParseException { - String sd = - "search disksummary {\n" + - "\n" + - " document-summary foobar {\n" + - " summary foo1 type string { source: inmemory }\n" + - " summary foo2 type string { source: ondisk }\n" + - " }\n" + - " document disksummary {\n" + - "\n" + - " field inmemory type string {\n" + - " indexing: attribute | summary\n" + - " }\n" + - " field ondisk type string {\n" + - " indexing: index | summary\n" + - " }\n" + - "\n" + - " }\n" + - "\n" + - "}"; + String sd = joinLines( + "search disksummary {", + " document-summary foobar {", + " summary foo1 type string { source: inmemory }", + " summary foo2 type string { source: ondisk }", + " }", + " document disksummary {", + " field inmemory type string {", + " indexing: attribute | summary", + " }", + " field ondisk type string {", + " indexing: index | summary", + " }", + " }", + "}"); DeployLoggerStub logger = new DeployLoggerStub(); SearchBuilder.createFromString(sd, logger); assertEquals(1, logger.entries.size()); @@ -78,27 +71,22 @@ public class SummaryTestCase { @Test public void testDiskSummaryExplicit() throws ParseException { - String sd = - "search disksummary {\n" + - "\n" + - " document disksummary {\n" + - "\n" + - " field inmemory type string {\n" + - " indexing: attribute | summary\n" + - " }\n" + - " field ondisk type string {\n" + - " indexing: index | summary\n" + - " }\n" + - "\n" + - " }\n" + - "\n" + - " document-summary foobar {\n" + - " summary foo1 type string { source: inmemory }\n" + - " summary foo2 type string { source: ondisk }\n" + - " from-disk\n" + - " }\n" + - "\n" + - "}"; + String sd = joinLines( + "search disksummary {", + " document disksummary {", + " field inmemory type string {", + " indexing: attribute | summary", + " }", + " field ondisk type string {", + " indexing: index | summary", + " }", + " }", + " document-summary foobar {", + " summary foo1 type string { source: inmemory }", + " summary foo2 type string { source: ondisk }", + " from-disk", + " }", + "}"); DeployLoggerStub logger = new DeployLoggerStub(); SearchBuilder.createFromString(sd, logger); assertTrue(logger.entries.isEmpty()); @@ -106,31 +94,30 @@ public class SummaryTestCase { @Test public void testStructMemorySummary() throws ParseException { - String sd = - "search structmemorysummary {\n" + - " document structmemorysummary {\n" + - " struct elem {\n" + - " field name type string {}\n" + - " field weight type int {}\n" + - " }\n" + - " field elem_array type array<elem> {\n" + - " indexing: summary\n" + - " struct-field name {\n" + - " indexing: attribute\n" + - " }\n" + - " struct-field weight {\n" + - " indexing: attribute\n" + - " }\n" + - " }\n" + - " }\n" + - " document-summary filtered {\n" + - " summary elem_array_filtered type array<elem> {\n" + - " source: elem_array\n" + - " matched-elements-only\n" + - " }\n" + - " }\n" + - "\n" + - "}"; + String sd = joinLines( + "search structmemorysummary {", + " document structmemorysummary {", + " struct elem {", + " field name type string {}", + " field weight type int {}", + " }", + " field elem_array type array<elem> {", + " indexing: summary", + " struct-field name {", + " indexing: attribute", + " }", + " struct-field weight {", + " indexing: attribute", + " }", + " }", + " }", + " document-summary filtered {", + " summary elem_array_filtered type array<elem> {", + " source: elem_array", + " matched-elements-only", + " }", + " }", + "}"); DeployLoggerStub logger = new DeployLoggerStub(); SearchBuilder.createFromString(sd, logger); assertTrue(logger.entries.isEmpty()); @@ -138,40 +125,35 @@ public class SummaryTestCase { @Test public void testInheritance() throws Exception { - String sd = - "search music {\n" + - "\n" + - " document music {\n" + - " field title type string {\n" + - " indexing: summary | attribute | index\n" + - " }\n" + - " \n" + - " field artist type string {\n" + - " indexing: summary | attribute | index\n" + - " }\n" + - " \n" + - " field album type string {\n" + - " indexing: summary | attribute | index\n" + - " }\n" + - " }\n" + - " \n" + - " document-summary title {\n" + - " summary title type string {\n" + - " source: title\n" + - " }\n" + - " }\n" + - " document-summary title_artist inherits title {\n" + - " summary artist type string {\n" + - " source: artist\n" + - " }\n" + - " }\n" + - " document-summary everything inherits title_artist {\n" + - " summary album type string {\n" + - " source: album\n" + - " }\n" + - " }\n" + - "\n" + - "}"; + String sd = joinLines( + "search music {", + " document music {", + " field title type string {", + " indexing: summary | attribute | index", + " }", + " field artist type string {", + " indexing: summary | attribute | index", + " }", + " field album type string {", + " indexing: summary | attribute | index", + " }", + " }", + " document-summary title {", + " summary title type string {", + " source: title", + " }", + " }", + " document-summary title_artist inherits title {", + " summary artist type string {", + " source: artist", + " }", + " }", + " document-summary everything inherits title_artist {", + " summary album type string {", + " source: album", + " }", + " }", + "}"); var logger = new DeployLoggerStub(); var search = SearchBuilder.createFromString(sd, logger).getSearch(); assertEquals(List.of(), logger.entries); @@ -202,30 +184,27 @@ public class SummaryTestCase { @Test public void testRedeclaringInheritedFieldFails() throws Exception { - String sd = - "search music {\n" + - "\n" + - " document music {\n" + - " field title type string {\n" + - " indexing: summary | attribute | index\n" + - " }\n" + - " field title_short type string {\n" + - " indexing: summary | attribute | index\n" + - " }\n" + - " }\n" + - " \n" + - " document-summary title {\n" + - " summary title type string {\n" + - " source: title\n" + - " }\n" + - " }\n" + - " document-summary title2 inherits title {\n" + - " summary title type string {\n" + - " source: title_short\n" + - " }\n" + - " }\n" + - " \n" + - "}"; + String sd = joinLines( + "search music {", + " document music {", + " field title type string {", + " indexing: summary | attribute | index", + " }", + " field title_short type string {", + " indexing: summary | attribute | index", + " }", + " }", + " document-summary title {", + " summary title type string {", + " source: title", + " }", + " }", + " document-summary title2 inherits title {", + " summary title type string {", + " source: title_short", + " }", + " }", + "}"); var logger = new DeployLoggerStub(); try { SearchBuilder.createFromString(sd, logger); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryTestCase.java index afbc9f52f6b..bfc738a4f87 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryTestCase.java @@ -23,6 +23,37 @@ import static org.junit.Assert.assertNull; public class SummaryTestCase extends SchemaTestCase { @Test + public void deriveRawAsBase64() throws ParseException { + String sd = joinLines( + "schema s {", + " raw-as-base64-in-summary", + " document s {", + " field raw_field type raw {", + " indexing: summary", + " }", + " }", + "}"); + Search search = SearchBuilder.createFromString(sd).getSearch(); + SummaryClass summary = new SummaryClass(search, search.getSummary("default"), new BaseDeployLogger()); + assertEquals(SummaryClassField.Type.RAW, summary.getField("raw_field").getType()); + } + + @Test + public void deriveRawAsLegacy() throws ParseException { + String sd = joinLines( + "schema s {", + " document s {", + " field raw_field type raw {", + " indexing: summary", + " }", + " }", + "}"); + Search search = SearchBuilder.createFromString(sd).getSearch(); + SummaryClass summary = new SummaryClass(search, search.getSummary("default"), new BaseDeployLogger()); + assertEquals(SummaryClassField.Type.DATA, summary.getField("raw_field").getType()); + } + + @Test public void testDeriving() throws IOException, ParseException { Search search = SearchBuilder.buildFromFile("src/test/examples/simple.sd"); SummaryClass summary = new SummaryClass(search, search.getSummary("default"), new BaseDeployLogger()); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolverTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolverTestCase.java index b149dafab95..12fe7e151c0 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolverTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolverTestCase.java @@ -57,6 +57,56 @@ public class RankingExpressionTypeResolverTestCase { } } + + @Test + public void tensorFirstPhaseFromConstantMustProduceDouble() throws Exception { + try { + SearchBuilder builder = new SearchBuilder(); + builder.importString(joinLines( + "search test {", + " document test { ", + " field a type tensor(d0[3]) {", + " indexing: attribute", + " }", + " }", + " rank-profile my_rank_profile {", + " function my_func() {", + " expression: x_tensor*2.0", + " }", + " function inline other_func() {", + " expression: z_tensor+3.0", + " }", + " first-phase {", + " expression: reduce(attribute(a),sum,d0)+y_tensor+my_func+other_func", + " }", + " constants {", + " x_tensor {", + " type: tensor(x{})", + " value: { {x:bar}:17 }", + " }", + " y_tensor {", + " type: tensor(y{})", + " value: { {y:foo}:42 }", + " }", + " z_tensor {", + " type: tensor(z{})", + " value: { {z:qux}:666 }", + " }", + " }", + " }", + "}" + )); + builder.build(); + fail("Expected exception"); + } + catch (IllegalArgumentException expected) { + assertEquals("In search definition 'test', rank profile 'my_rank_profile': The first-phase expression must produce a double (a tensor with no dimensions), but produces tensor(x{},y{},z{})", + Exceptions.toMessageString(expected)); + } + } + + + @Test public void tensorSecondPhaseMustProduceDouble() throws Exception { try { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorTestCase.java index a1231a1418b..10ba6eff169 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorTestCase.java @@ -33,6 +33,26 @@ public class RankingExpressionWithTensorTestCase { } @Test + public void requireConstantTensorCanBeReferredViaConstantFeature() throws ParseException { + RankProfileSearchFixture f = new RankProfileSearchFixture( + " rank-profile my_profile {\n" + + " first-phase {\n" + + " expression: sum(constant(my_tensor))\n" + + " }\n" + + " constants {\n" + + " my_tensor {\n" + + " value: { {x:1,y:2}:1, {x:2,y:1}:2 }\n" + + " type: tensor(x{},y{})\n" + + " }\n" + + " }\n" + + " }"); + f.compileRankProfile("my_profile"); + f.assertFirstPhaseExpression("reduce(constant(my_tensor), sum)", "my_profile"); + f.assertRankProperty("tensor(x{},y{}):{{x:1,y:2}:1.0,{x:2,y:1}:2.0}", "constant(my_tensor).value", "my_profile"); + f.assertRankProperty("tensor(x{},y{})", "constant(my_tensor).type", "my_profile"); + } + + @Test public void requireThatMultiLineConstantTensorAndTypeCanBeParsed() throws ParseException { RankProfileSearchFixture f = new RankProfileSearchFixture( " rank-profile my_profile {\n" + diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/ClusterControllerTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/ClusterControllerTestCase.java index 989ae87913d..cf142dae2c7 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/ClusterControllerTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/ClusterControllerTestCase.java @@ -13,7 +13,6 @@ import com.yahoo.config.model.api.Reindexing; import com.yahoo.config.model.application.provider.SimpleApplicationValidator; import com.yahoo.config.model.builder.xml.test.DomBuilderTest; import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.config.model.test.TestDriver; import com.yahoo.config.model.test.TestRoot; @@ -393,7 +392,7 @@ public class ClusterControllerTestCase extends DomBuilderTest { model.getConfig(qrBuilder, "admin/cluster-controllers/0/components/clustercontroller-bar-configurer"); QrStartConfig qrStartConfig = new QrStartConfig(qrBuilder); assertEquals(32, qrStartConfig.jvm().minHeapsize()); - assertEquals(256, qrStartConfig.jvm().heapsize()); + assertEquals(128, qrStartConfig.jvm().heapsize()); assertEquals(0, qrStartConfig.jvm().heapSizeAsPercentageOfPhysicalMemory()); assertEquals(2, qrStartConfig.jvm().availableProcessors()); assertFalse(qrStartConfig.jvm().verbosegc()); @@ -407,44 +406,6 @@ public class ClusterControllerTestCase extends DomBuilderTest { } @Test - public void testQrStartConfigWithFeatureFlagForMaxHeap() throws Exception { - String xml = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n" + - "<services>\n" + - "\n" + - " <admin version=\"2.0\">\n" + - " <adminserver hostalias=\"configserver\" />\n" + - " <logserver hostalias=\"logserver\" />\n" + - " <slobroks>\n" + - " <slobrok hostalias=\"configserver\" />\n" + - " <slobrok hostalias=\"logserver\" />\n" + - " </slobroks>\n" + - " </admin>\n" + - " <content version='1.0' id='bar'>" + - " <redundancy>1</redundancy>\n" + - " <documents>" + - " <document type=\"type1\" mode=\"store-only\"/>\n" + - " </documents>\n" + - " <group>" + - " <node hostalias='node0' distribution-key='0' />" + - " </group>" + - " </content>" + - "\n" + - "</services>"; - - VespaModel model = createVespaModel(xml, new DeployState.Builder().properties(new TestProperties().clusterControllerMaxHeapSizeInMb(256))); - assertTrue(model.getService("admin/cluster-controllers/0").isPresent()); - - QrStartConfig.Builder qrBuilder = new QrStartConfig.Builder(); - model.getConfig(qrBuilder, "admin/cluster-controllers/0/components/clustercontroller-bar-configurer"); - QrStartConfig qrStartConfig = new QrStartConfig(qrBuilder); - // Taken from ContainerCluster - assertEquals(32, qrStartConfig.jvm().minHeapsize()); - // Overridden values from ClusterControllerContainerCluster - assertEquals(256, qrStartConfig.jvm().heapsize()); - assertFalse(qrStartConfig.jvm().verbosegc()); - } - - @Test public void testUnconfiguredNoContent() throws Exception { String xml = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n" + "<services>\n" + diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/DedicatedAdminV4Test.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/DedicatedAdminV4Test.java index f1f794c5057..60672c7df07 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/DedicatedAdminV4Test.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/DedicatedAdminV4Test.java @@ -147,7 +147,7 @@ public class DedicatedAdminV4Test { " <admin version='4.0'>" + " <slobroks><nodes count='2' dedicated='true'/></slobroks>" + " <logservers><nodes count='1' dedicated='true'/></logservers>" + - " <logforwarding>" + + " <logforwarding include-admin='true'>" + " <splunk deployment-server='foo:123' client-name='foocli' phone-home-interval='900'/>" + " </logforwarding>" + " </admin>" + diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java index d92ace2939a..e99a92b530a 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java @@ -26,7 +26,7 @@ public class QuotaValidatorTest { @Test public void test_deploy_under_quota() { var tester = new ValidationTester(8, false, new TestProperties().setHostedVespa(true).setQuota(quota).setZone(publicZone)); - tester.deploy(null, getServices("testCluster", 5), Environment.prod, null); + tester.deploy(null, getServices("testCluster", 4), Environment.prod, null); } @Test @@ -54,7 +54,7 @@ public class QuotaValidatorTest { @Test public void test_deploy_above_quota_budget_in_publiccd() { - var tester = new ValidationTester(13, false, new TestProperties().setHostedVespa(true).setQuota(quota).setZone(publicCdZone)); + var tester = new ValidationTester(13, false, new TestProperties().setHostedVespa(true).setQuota(quota.withBudget(BigDecimal.ONE)).setZone(publicCdZone)); try { tester.deploy(null, getServices("testCluster", 10), Environment.prod, null); fail(); @@ -65,6 +65,19 @@ public class QuotaValidatorTest { } @Test + public void test_deploy_max_resources_above_quota() { + var tester = new ValidationTester(13, false, new TestProperties().setHostedVespa(true).setQuota(quota).setZone(publicCdZone)); + try { + tester.deploy(null, getServices("testCluster", 10), Environment.prod, null); + fail(); + } catch (RuntimeException e) { + assertEquals("publiccd: Please free up some capacity! This deployment's quota use ($-.--) exceeds reserved quota ($-.--)!", + ValidationTester.censorNumbers(e.getMessage())); + + } + } + + @Test public void test_deploy_with_negative_budget() { var quota = Quota.unlimited().withBudget(BigDecimal.valueOf(-1)); var tester = new ValidationTester(13, false, new TestProperties().setHostedVespa(true).setQuota(quota).setZone(publicZone)); @@ -88,7 +101,7 @@ public class QuotaValidatorTest { " <document type='music' mode='index'/>" + " </documents>" + " <nodes count='" + nodeCount + "'>" + - " <resources vcpu=\"[0.5, 1]\" memory=\"[1Gb, 3Gb]\" disk=\"[1Gb, 9Gb]\"/>\n" + + " <resources vcpu=\"[0.5, 2]\" memory=\"[1Gb, 6Gb]\" disk=\"[1Gb, 18Gb]\"/>\n" + " </nodes>" + " </content>" + "</services>"; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidatorTest.java index 20f5a9c841c..b1fda081b64 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidatorTest.java @@ -2,14 +2,12 @@ package com.yahoo.vespa.model.application.validation.change.search; import com.yahoo.config.application.api.ValidationId; -import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; import com.yahoo.vespa.model.application.validation.change.VespaConfigChangeAction; import com.yahoo.vespa.model.application.validation.change.VespaReindexAction; import org.junit.Test; -import java.time.Instant; import java.util.Arrays; import java.util.List; @@ -130,7 +128,7 @@ public class IndexingScriptChangeValidatorTest { } @Test - public void requireThatAddingIndexFieldIsOk() throws Exception { + public void requireThatAddingDocumentIndexFieldIsOk() throws Exception { new Fixture("", "field f1 type string { indexing: index | summary }"). assertValidation(); } @@ -142,12 +140,22 @@ public class IndexingScriptChangeValidatorTest { } @Test - public void requireThatAddingFieldIsOk() throws Exception { + public void requireThatAddingDocumentFieldIsOk() throws Exception { new Fixture("", FIELD + " { indexing: attribute | summary }"). assertValidation(); } @Test + public void requireThatAddingExtraFieldRequiresReindexing() throws Exception { + new Fixture(" field f1 type string { indexing: index }", + " field f1 type string { indexing: index } } " + + " field f2 type string { indexing: input f1 | summary ") + .assertValidation(VespaReindexAction.of(ClusterSpec.Id.from("test"), + null, + "Non-document field 'f2' added; this may be populated by reindexing")); + } + + @Test public void requireThatAddingSummaryAspectIsOk() throws Exception { new Fixture(FIELD + " { indexing: attribute }", FIELD + " { indexing: attribute | summary }"). diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java index 3a3dde0cf87..ad4603e5c6b 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java @@ -824,55 +824,22 @@ public class ContentBuilderTest extends DomBuilderTest { verifyThatFeatureFlagControlsVisibilityDelayDefault(0.6, 0.6); } - private void verifyThatFeatureFlagControlsUseBucketExecutorForLidSpaceCompact(boolean flag) { - DeployState.Builder deployStateBuilder = new DeployState.Builder().properties(new TestProperties().useBucketExecutorForLidSpaceCompact(flag)); + private void verifyThatFeatureFlagControlsUseBucketExecutorForPruneRemoved(boolean flag) { + DeployState.Builder deployStateBuilder = new DeployState.Builder().properties(new TestProperties().useBucketExecutorForPruneRemoved(flag)); VespaModel model = new VespaModelCreatorWithMockPkg(new MockApplicationPackage.Builder() .withServices(singleNodeContentXml()) .withSearchDefinition(MockApplicationPackage.MUSIC_SEARCHDEFINITION) .build()) .create(deployStateBuilder); ProtonConfig config = getProtonConfig(model.getContentClusters().values().iterator().next()); - assertEquals(flag, config.lidspacecompaction().usebucketexecutor()); + assertEquals(flag, config.pruneremoveddocuments().usebucketexecutor()); } - private void verifyThatFeatureFlagControlsUseBucketExecutorForBucketMove(boolean flag) { - DeployState.Builder deployStateBuilder = new DeployState.Builder().properties(new TestProperties().useBucketExecutorForBucketMove(flag)); - VespaModel model = new VespaModelCreatorWithMockPkg(new MockApplicationPackage.Builder() - .withServices(singleNodeContentXml()) - .withSearchDefinition(MockApplicationPackage.MUSIC_SEARCHDEFINITION) - .build()) - .create(deployStateBuilder); - ProtonConfig config = getProtonConfig(model.getContentClusters().values().iterator().next()); - assertEquals(flag, config.bucketmove().usebucketexecutor()); - } - - private void verifyThatFeatureFlagControlsMaxpendingMoveOps(int moveOps) { - DeployState.Builder deployStateBuilder = new DeployState.Builder().properties(new TestProperties().setMaxPendingMoveOps(moveOps)); - VespaModel model = new VespaModelCreatorWithMockPkg(new MockApplicationPackage.Builder() - .withServices(singleNodeContentXml()) - .withSearchDefinition(MockApplicationPackage.MUSIC_SEARCHDEFINITION) - .build()) - .create(deployStateBuilder); - ProtonConfig config = getProtonConfig(model.getContentClusters().values().iterator().next()); - assertEquals(moveOps, config.maintenancejobs().maxoutstandingmoveops()); - } - - @Test - public void verifyMaxPendingMoveOps() { - verifyThatFeatureFlagControlsMaxpendingMoveOps(13); - verifyThatFeatureFlagControlsMaxpendingMoveOps(107); - } - - @Test - public void verifyUseBucketExecutorForLidSpaceCompact() { - verifyThatFeatureFlagControlsUseBucketExecutorForLidSpaceCompact(true); - verifyThatFeatureFlagControlsUseBucketExecutorForLidSpaceCompact(false); - } @Test - public void verifyUseBucketExecutorForBucketMove() { - verifyThatFeatureFlagControlsUseBucketExecutorForBucketMove(true); - verifyThatFeatureFlagControlsUseBucketExecutorForBucketMove(false); + public void verifyUseBucketExecutorForPruneRemoved() { + verifyThatFeatureFlagControlsUseBucketExecutorForPruneRemoved(true); + verifyThatFeatureFlagControlsUseBucketExecutorForPruneRemoved(false); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java index 46026a5528c..ee9c9ccb681 100755 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java @@ -175,7 +175,7 @@ public class ContainerClusterTest { cluster.getConfig(qrBuilder); QrStartConfig qrStartConfig = new QrStartConfig(qrBuilder); assertEquals(32, qrStartConfig.jvm().minHeapsize()); - assertEquals(256, qrStartConfig.jvm().heapsize()); + assertEquals(128, qrStartConfig.jvm().heapsize()); assertEquals(32, qrStartConfig.jvm().compressedClassSpaceSize()); assertEquals(0, qrStartConfig.jvm().heapSizeAsPercentageOfPhysicalMemory()); root.freezeModelTopology(); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java index 4aadc0e3f05..953c42243a6 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java @@ -1005,22 +1005,6 @@ public class ContentClusterTest extends ContentBaseTest { assertTrue(resolveThreePhaseUpdateConfigWithFeatureFlag(true)); } - private double resolveMaxDeadBytesRatio(double maxDeadBytesRatio) { - VespaModel model = createEnd2EndOneNode(new TestProperties().maxDeadBytesRatio(maxDeadBytesRatio)); - ContentCluster cc = model.getContentClusters().get("storage"); - ProtonConfig.Builder protonBuilder = new ProtonConfig.Builder(); - cc.getSearch().getConfig(protonBuilder); - ProtonConfig protonConfig = new ProtonConfig(protonBuilder); - assertEquals(1, protonConfig.documentdb().size()); - return protonConfig.documentdb(0).allocation().max_dead_bytes_ratio(); - } - - @Test - public void default_max_dead_bytes_ratio_config_controlled_by_properties() { - assertEquals(0.2, resolveMaxDeadBytesRatio(0.2), 1e-5); - assertEquals(0.1, resolveMaxDeadBytesRatio(0.1), 1e-5); - } - void assertZookeeperServerImplementation(String expectedClassName, ClusterControllerContainerCluster clusterControllerCluster) { for (ClusterControllerContainer c : clusterControllerCluster.getContainers()) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java b/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java index fc6a4ee2783..d0196ace766 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java @@ -15,6 +15,7 @@ import com.yahoo.path.Path; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; import com.yahoo.vespa.config.search.RankProfilesConfig; +import com.yahoo.vespa.config.search.core.OnnxModelsConfig; import com.yahoo.vespa.config.search.core.RankingConstantsConfig; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.container.ApplicationContainerCluster; @@ -95,6 +96,10 @@ public class ModelEvaluationTest { cluster.getConfig(cb); RankingConstantsConfig constantsConfig = new RankingConstantsConfig(cb); + OnnxModelsConfig.Builder ob = new OnnxModelsConfig.Builder(); + cluster.getConfig(ob); + OnnxModelsConfig onnxModelsConfig = new OnnxModelsConfig(ob); + assertEquals(4, config.rankprofile().size()); Set<String> modelNames = config.rankprofile().stream().map(v -> v.name()).collect(Collectors.toSet()); assertTrue(modelNames.contains("xgboost_2_2")); @@ -109,7 +114,7 @@ public class ModelEvaluationTest { assertEquals(profile, sb.toString()); ModelsEvaluator evaluator = new ModelsEvaluator(new ToleratingMissingConstantFilesRankProfilesConfigImporter(MockFileAcquirer.returnFile(null)) - .importFrom(config, constantsConfig)); + .importFrom(config, constantsConfig, onnxModelsConfig)); assertEquals(4, evaluator.models().size()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java index b72ae088484..ba975e52d1a 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java @@ -52,6 +52,7 @@ public class VespaModelTester { private final Map<NodeResources, Collection<Host>> hostsByResources = new HashMap<>(); private ApplicationId applicationId = ApplicationId.defaultId(); private boolean useDedicatedNodeForLogserver = false; + private HostProvisioner provisioner; public VespaModelTester() { this(new NullConfigModelRegistry()); @@ -61,6 +62,12 @@ public class VespaModelTester { this.configModelRegistry = configModelRegistry; } + public HostProvisioner provisioner() { + if (provisioner instanceof ProvisionerAdapter) + return ((ProvisionerAdapter)provisioner).provisioner(); + return provisioner; + } + /** Adds some nodes with resources 1, 3, 10 */ public Hosts addHosts(int count) { return addHosts(InMemoryProvisioner.defaultResources, count); } @@ -108,37 +115,43 @@ public class VespaModelTester { /** Creates a model which uses 0 as start index */ public VespaModel createModel(String services, boolean failOnOutOfCapacity, String ... retiredHostNames) { - return createModel(Zone.defaultZone(), services, failOnOutOfCapacity, false, 0, + return createModel(Zone.defaultZone(), services, failOnOutOfCapacity, false, false, 0, Optional.empty(), new DeployState.Builder(), retiredHostNames); } /** Creates a model which uses 0 as start index */ public VespaModel createModel(String services, boolean failOnOutOfCapacity, DeployState.Builder builder) { - return createModel(Zone.defaultZone(), services, failOnOutOfCapacity, false, 0, Optional.empty(), builder); + return createModel(Zone.defaultZone(), services, failOnOutOfCapacity, false, false, 0, Optional.empty(), builder); } /** Creates a model which uses 0 as start index */ public VespaModel createModel(String services, boolean failOnOutOfCapacity, boolean useMaxResources, String ... retiredHostNames) { - return createModel(Zone.defaultZone(), services, failOnOutOfCapacity, useMaxResources, 0, + return createModel(Zone.defaultZone(), services, failOnOutOfCapacity, useMaxResources, false, 0, + Optional.empty(), new DeployState.Builder(), retiredHostNames); + } + + /** Creates a model which uses 0 as start index */ + public VespaModel createModel(String services, boolean failOnOutOfCapacity, boolean useMaxResources, boolean alwaysReturnOneNode, String ... retiredHostNames) { + return createModel(Zone.defaultZone(), services, failOnOutOfCapacity, useMaxResources, alwaysReturnOneNode, 0, Optional.empty(), new DeployState.Builder(), retiredHostNames); } /** Creates a model which uses 0 as start index */ public VespaModel createModel(String services, boolean failOnOutOfCapacity, int startIndexForClusters, String ... retiredHostNames) { - return createModel(Zone.defaultZone(), services, failOnOutOfCapacity, false, startIndexForClusters, + return createModel(Zone.defaultZone(), services, failOnOutOfCapacity, false, false, startIndexForClusters, Optional.empty(), new DeployState.Builder(), retiredHostNames); } /** Creates a model which uses 0 as start index */ public VespaModel createModel(Zone zone, String services, boolean failOnOutOfCapacity, String ... retiredHostNames) { - return createModel(zone, services, failOnOutOfCapacity, false, 0, + return createModel(zone, services, failOnOutOfCapacity, false, false, 0, Optional.empty(), new DeployState.Builder(), retiredHostNames); } /** Creates a model which uses 0 as start index */ public VespaModel createModel(Zone zone, String services, boolean failOnOutOfCapacity, DeployState.Builder deployStateBuilder, String ... retiredHostNames) { - return createModel(zone, services, failOnOutOfCapacity, false, 0, + return createModel(zone, services, failOnOutOfCapacity, false, false, 0, Optional.empty(), deployStateBuilder, retiredHostNames); } @@ -152,15 +165,16 @@ public class VespaModelTester { * @return the resulting model */ public VespaModel createModel(Zone zone, String services, boolean failOnOutOfCapacity, boolean useMaxResources, + boolean alwaysReturnOneNode, int startIndexForClusters, Optional<VespaModel> previousModel, DeployState.Builder deployStatebuilder, String ... retiredHostNames) { VespaModelCreatorWithMockPkg modelCreatorWithMockPkg = new VespaModelCreatorWithMockPkg(null, services, ApplicationPackageUtils.generateSearchDefinition("type1")); ApplicationPackage appPkg = modelCreatorWithMockPkg.appPkg; - HostProvisioner provisioner = hosted ? - new ProvisionerAdapter(new InMemoryProvisioner(hostsByResources, + provisioner = hosted ? new ProvisionerAdapter(new InMemoryProvisioner(hostsByResources, failOnOutOfCapacity, useMaxResources, + alwaysReturnOneNode, false, startIndexForClusters, retiredHostNames)) : @@ -184,12 +198,14 @@ public class VespaModelTester { /** To verify that we don't call allocateHost(alias) in hosted environments */ private static class ProvisionerAdapter implements HostProvisioner { - private final HostProvisioner provisioner; + private final InMemoryProvisioner provisioner; - public ProvisionerAdapter(HostProvisioner provisioner) { + public ProvisionerAdapter(InMemoryProvisioner provisioner) { this.provisioner = provisioner; } + public InMemoryProvisioner provisioner() { return provisioner; } + @Override public HostSpec allocateHost(String alias) { throw new UnsupportedOperationException("Allocating hosts using <node> tags is not supported in hosted environments, " + |