diff options
Diffstat (limited to 'config-model/src/main/java/com/yahoo')
35 files changed, 456 insertions, 243 deletions
diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 686cb7ce7c6..397b11661e5 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -1,4 +1,4 @@ -// Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.model.deploy; import com.google.common.collect.ImmutableList; @@ -47,12 +47,13 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private Quota quota = Quota.unlimited(); private boolean useAccessControlTlsHandshakeClientAuth; private boolean useAsyncMessageHandlingOnSchedule = false; - private int mergeChunkSize = 0x400000 - 0x1000; // 4M -4k private double feedConcurrency = 0.5; private boolean enableAutomaticReindexing = false; private boolean reconfigurableZookeeperServer = false; private boolean useBucketExecutorForLidSpaceCompact; private boolean enableFeedBlockInDistributor = false; + private double maxDeadBytesRatio = 0.2; + private int clusterControllerMaxHeapSizeInMb = 512; @Override public ModelContext.FeatureFlags featureFlags() { return this; } @Override public boolean multitenant() { return multitenant; } @@ -82,23 +83,19 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public Quota quota() { return quota; } @Override public boolean useAccessControlTlsHandshakeClientAuth() { return useAccessControlTlsHandshakeClientAuth; } @Override public boolean useAsyncMessageHandlingOnSchedule() { return useAsyncMessageHandlingOnSchedule; } - @Override public int mergeChunkSize() { return mergeChunkSize; } @Override public double feedConcurrency() { return feedConcurrency; } @Override public boolean enableAutomaticReindexing() { return enableAutomaticReindexing; } @Override public boolean reconfigurableZookeeperServer() { return reconfigurableZookeeperServer; } @Override public boolean useBucketExecutorForLidSpaceCompact() { return useBucketExecutorForLidSpaceCompact; } @Override public boolean enableFeedBlockInDistributor() { return enableFeedBlockInDistributor; } + @Override public double maxDeadBytesRatio() { return maxDeadBytesRatio; } + @Override public int clusterControllerMaxHeapSizeInMb() { return clusterControllerMaxHeapSizeInMb; } public TestProperties setFeedConcurrency(double feedConcurrency) { this.feedConcurrency = feedConcurrency; return this; } - public TestProperties setMergeChunkSize(int size) { - mergeChunkSize = size; - return this; - } - public TestProperties setAsyncMessageHandlingOnSchedule(boolean value) { useAsyncMessageHandlingOnSchedule = value; return this; @@ -202,6 +199,16 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } + public TestProperties maxDeadBytesRatio(double ratio) { + maxDeadBytesRatio = ratio; + return this; + } + + public TestProperties clusterControllerMaxHeapSizeInMb(int heapSize) { + clusterControllerMaxHeapSizeInMb = heapSize; + return this; + } + public static class Spec implements ConfigServerSpec { private final String hostName; diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java index 4302a4dcd48..8fc36b06544 100644 --- a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java +++ b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java @@ -80,6 +80,11 @@ public class InMemoryProvisioner implements HostProvisioner { this(Map.of(defaultResources, toHostInstances(hosts)), failOnOutOfCapacity, false, sharedHosts, 0); } + /** Creates this with a set of host names of the flavor 'default' */ + public InMemoryProvisioner(boolean failOnOutOfCapacity, boolean sharedHosts, List<String> hosts) { + this(Map.of(defaultResources, toHostInstances(hosts.toArray(new String[0]))), failOnOutOfCapacity, false, sharedHosts, 0); + } + /** Creates this with a set of hosts of the flavor 'default' */ public InMemoryProvisioner(Hosts hosts, boolean failOnOutOfCapacity, boolean sharedHosts, String ... retiredHostNames) { this(Map.of(defaultResources, hosts.asCollection()), failOnOutOfCapacity, false, sharedHosts, 0, retiredHostNames); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TokenTransformer.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TokenTransformer.java index 192fb9baa9a..032341297bf 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TokenTransformer.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TokenTransformer.java @@ -114,14 +114,20 @@ public class TokenTransformer extends ExpressionTransformer<RankProfileTransform /** * Transforms a feature of the form * - * tokenTypeIds(128, a, ...) + * tokenTypeIds(128, a, b, ...) * * to an expression that generates a tensor that has values 0 for "a" * (including CLS and SEP tokens) and 1 for the rest of the sequence. * * Concretely, transforms to a tensor generation expression: * - * tensor(d0[1],d1[128])(if(d1 < length_a + 2, 0, 1)) + * tensor(d0[1],d1[128])( + * if (d1 < 1 + length_a + 1, + * 0, + * if (d1 < 1 + length_a + 1 + length_b + 1 + ..., + * 1, + * 0 + * ))) */ private ExpressionNode transformTokenTypeIds(ReferenceNode feature, RankProfileTransformContext context) { checkArguments(feature); @@ -131,11 +137,18 @@ public class TokenTransformer extends ExpressionTransformer<RankProfileTransform // we need to add functions calculating the token lengths of the arguments createTokenLengthFunctions(feature, context); - ReferenceNode arg = (ReferenceNode) feature.getArguments().expressions().get(1); - ExpressionNode argLength = new ReferenceNode(lengthFunctionName(arg)); - ExpressionNode lengthExpr = new ArithmeticNode(argLength, ArithmeticOperator.PLUS, TWO); - ComparisonNode comparison = new ComparisonNode(new ReferenceNode("d1"), TruthOperator.SMALLER, lengthExpr); - ExpressionNode expr = new IfNode(comparison, ZERO, ONE); + List<ExpressionNode> tokenSequence = createTokenSequence(feature); + ExpressionNode queryLengthExpr = createLengthExpr(2, tokenSequence); + ExpressionNode restLengthExpr = createLengthExpr(tokenSequence.size() - 1, tokenSequence); + ExpressionNode expr = new IfNode( + new ComparisonNode(new ReferenceNode("d1"), TruthOperator.SMALLER, queryLengthExpr), + ZERO, + new IfNode( + new ComparisonNode(new ReferenceNode("d1"), TruthOperator.SMALLER, restLengthExpr), + ONE, + ZERO + ) + ); return new TensorFunctionNode(Generate.bound(type, wrapScalar(expr))); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java index 778610bd63b..56b74134ebe 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java @@ -221,7 +221,7 @@ public class Admin extends AbstractConfigProducer<Admin> implements Serializable for (var host : hosts) { // Send hostname to be used in configId (instead of index), as the sorting of hosts seems to be unstable // between config changes, even when the set of hosts is unchanged. - var container = new MetricsProxyContainer(metricsProxyCluster, deployState.featureFlags(), host.getHostname(), index, deployState.isHosted()); + var container = new MetricsProxyContainer(metricsProxyCluster, host.getHostname(), index, deployState.isHosted()); addAndInitializeService(deployState.getDeployLogger(), host, container); metricsProxyCluster.addContainer(container); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java index 880e0e8c574..356453dcf5b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java @@ -1,7 +1,6 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin; -import com.yahoo.config.model.api.ModelContext.FeatureFlags; import com.yahoo.config.model.api.container.ContainerServiceType; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.vespa.model.container.Container; @@ -15,9 +14,11 @@ import com.yahoo.vespa.model.container.component.AccessLogComponent.CompressionT */ public class LogserverContainer extends Container { - public LogserverContainer(AbstractConfigProducer parent, FeatureFlags featureFlags, boolean isHostedVespa) { - super(parent, featureFlags, "" + 0, 0, isHostedVespa); - addComponent(new AccessLogComponent(AccessLogType.jsonAccessLog, CompressionType.GZIP, ((LogserverContainerCluster) parent).getName(), true)); + public LogserverContainer(AbstractConfigProducer<?> parent, boolean isHostedVespa) { + super(parent, "" + 0, 0, isHostedVespa); + LogserverContainerCluster cluster = (LogserverContainerCluster) parent; + addComponent(new AccessLogComponent( + cluster, AccessLogType.jsonAccessLog, CompressionType.GZIP, cluster.getName(), true)); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/ZooKeepersConfigProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/ZooKeepersConfigProvider.java index 25dbd62e647..9d8a9707d12 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/ZooKeepersConfigProvider.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/ZooKeepersConfigProvider.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin; import com.yahoo.collections.CollectionUtil; @@ -12,7 +12,7 @@ import java.util.List; */ public class ZooKeepersConfigProvider implements ZookeepersConfig.Producer { - public static final int zkPort = Integer.getInteger("zk_port", 2181); + public static final int zooKeeperClientPort = 2181; private final List<Configserver> configServers; @@ -27,7 +27,7 @@ public class ZooKeepersConfigProvider implements ZookeepersConfig.Producer { public List<String> getZooKeepers() { List<String> servers = new ArrayList<>(); for (Configserver server : configServers) { - servers.add(server.getHostName() + ":" + zkPort); + servers.add(server.getHostName() + ":" + zooKeeperClientPort); } return servers; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java index 2ad3bc8f84a..4b39c67cda4 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.model.admin.clustercontroller; import com.yahoo.cloud.config.ZookeeperServerConfig; import com.yahoo.component.ComponentSpecification; +import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.api.container.ContainerServiceType; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; @@ -40,13 +41,15 @@ public class ClusterControllerContainer extends Container implements private static final ComponentSpecification REINDEXING_CONTROLLER_BUNDLE = new ComponentSpecification("clustercontroller-reindexer"); private final Set<String> bundles = new TreeSet<>(); + private final ModelContext.FeatureFlags featureFlags; public ClusterControllerContainer( AbstractConfigProducer<?> parent, int index, boolean runStandaloneZooKeeper, DeployState deployState) { - super(parent, deployState.featureFlags(), "" + index, index, deployState.isHosted()); + super(parent, "" + index, index, deployState.isHosted()); + this.featureFlags = deployState.featureFlags(); addHandler("clustercontroller-status", "com.yahoo.vespa.clustercontroller.apps.clustercontroller.StatusHandler", "/clustercontroller-status/*", @@ -55,7 +58,7 @@ public class ClusterControllerContainer extends Container implements "com.yahoo.vespa.clustercontroller.apps.clustercontroller.StateRestApiV2Handler", "/cluster/v2/*", CLUSTERCONTROLLER_BUNDLE); - addComponent(new AccessLogComponent(AccessLogComponent.AccessLogType.jsonAccessLog, + addComponent(new AccessLogComponent(containerCluster().orElse(null), AccessLogComponent.AccessLogType.jsonAccessLog, AccessLogComponent.CompressionType.GZIP, "controller", deployState.isHosted())); @@ -144,6 +147,7 @@ public class ClusterControllerContainer extends Container implements @Override public void getConfig(ZookeeperServerConfig.Builder builder) { builder.myid(index()); + builder.dynamicReconfiguration(featureFlags.reconfigurableZookeeperServer()); } @Override @@ -155,7 +159,6 @@ public class ClusterControllerContainer extends Container implements } builder.enabled(ctx.reindexing().enabled()); - builder.windowSizeIncrement(ctx.windowSizeIncrement()); for (String clusterId : ctx.clusterIds()) { ReindexingConfig.Clusters.Builder clusterBuilder = new ReindexingConfig.Clusters.Builder(); for (NewDocumentType type : ctx.documentTypesForCluster(clusterId)) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java index 3fe6ce3ff27..4fc73de3b48 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java @@ -1,9 +1,11 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin.clustercontroller; +import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.api.Reindexing; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.search.config.QrStartConfig; import com.yahoo.vespa.model.container.ContainerCluster; /** @@ -12,15 +14,16 @@ import com.yahoo.vespa.model.container.ContainerCluster; * @author gjoranv * @author bjorncs */ -public class ClusterControllerContainerCluster extends ContainerCluster<ClusterControllerContainer> -{ +public class ClusterControllerContainerCluster extends ContainerCluster<ClusterControllerContainer> { + private final ModelContext.FeatureFlags featureFlags; private final ReindexingContext reindexingContext; public ClusterControllerContainerCluster( AbstractConfigProducer<?> parent, String subId, String name, DeployState deployState) { super(parent, subId, name, deployState, false); addDefaultHandlersWithVip(); + this.featureFlags = deployState.featureFlags(); this.reindexingContext = createReindexingContext(deployState); } @@ -29,13 +32,20 @@ public class ClusterControllerContainerCluster extends ContainerCluster<ClusterC @Override protected boolean messageBusEnabled() { return false; } + @Override + public void getConfig(QrStartConfig.Builder builder) { + super.getConfig(builder); + int maxHeapSize = featureFlags.clusterControllerMaxHeapSizeInMb(); + boolean verboseGc = (maxHeapSize < 512); + builder.jvm + .verbosegc(verboseGc) + .heapsize(maxHeapSize); + } + public ReindexingContext reindexingContext() { return reindexingContext; } private static ReindexingContext createReindexingContext(DeployState deployState) { - Reindexing reindexing = deployState.featureFlags().enableAutomaticReindexing() - ? deployState.reindexing().orElse(Reindexing.DISABLED_INSTANCE) - : Reindexing.DISABLED_INSTANCE; - return new ReindexingContext(reindexing, deployState.featureFlags().reindexerWindowSizeIncrement()); + return new ReindexingContext(deployState.reindexing().orElse(Reindexing.DISABLED_INSTANCE)); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ReindexingContext.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ReindexingContext.java index 7380b950fb2..026c0b93712 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ReindexingContext.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ReindexingContext.java @@ -21,11 +21,9 @@ public class ReindexingContext { private final Object monitor = new Object(); private final Map<String, Set<NewDocumentType>> documentTypesPerCluster = new HashMap<>(); private final Reindexing reindexing; - private final double windowSizeIncrement; - public ReindexingContext(Reindexing reindexing, double windowSizeIncrement) { + public ReindexingContext(Reindexing reindexing) { this.reindexing = Objects.requireNonNull(reindexing); - this.windowSizeIncrement = windowSizeIncrement; } public void addDocumentType(String clusterId, NewDocumentType type) { @@ -49,6 +47,4 @@ public class ReindexingContext { public Reindexing reindexing() { return reindexing; } - public double windowSizeIncrement() { return windowSizeIncrement; } - } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java index f7a4f2e52a2..aef7c846a19 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java @@ -11,7 +11,6 @@ import ai.vespa.metricsproxy.rpc.RpcConnector; import ai.vespa.metricsproxy.rpc.RpcConnectorConfig; import ai.vespa.metricsproxy.service.VespaServices; import ai.vespa.metricsproxy.service.VespaServicesConfig; -import com.yahoo.config.model.api.ModelContext.FeatureFlags; import com.yahoo.config.model.api.container.ContainerServiceType; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.config.provision.ClusterMembership; @@ -40,8 +39,8 @@ public class MetricsProxyContainer extends Container implements final boolean isHostedVespa; - public MetricsProxyContainer(AbstractConfigProducer parent, FeatureFlags featureFlags, String hostname, int index, boolean isHostedVespa) { - super(parent, featureFlags, hostname, index, isHostedVespa); + public MetricsProxyContainer(AbstractConfigProducer<?> parent, String hostname, int index, boolean isHostedVespa) { + super(parent, hostname, index, isHostedVespa); this.isHostedVespa = isHostedVespa; setProp("clustertype", "admin"); setProp("index", String.valueOf(index)); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index 5dffe3d0a28..219095ae41f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -241,6 +241,10 @@ public class VespaMetricSet { // DO NOT RELY ON THIS METRIC YET. metrics.add(new Metric("cluster-controller.node-event.count")); + metrics.add(new Metric("cluster-controller.resource_usage.nodes_above_limit.last")); + metrics.add(new Metric("cluster-controller.resource_usage.max_memory_utilization.last")); + metrics.add(new Metric("cluster-controller.resource_usage.max_disk_utilization.last")); + metrics.add(new Metric("reindexing.progress.last")); return metrics; @@ -290,6 +294,8 @@ public class VespaMetricSet { metrics.add(new Metric("hits_per_query.sum")); metrics.add(new Metric("hits_per_query.count")); metrics.add(new Metric("hits_per_query.average")); // TODO: Remove in Vespa 8 + metrics.add(new Metric("hits_per_query.95percentile")); + metrics.add(new Metric("hits_per_query.99percentile")); metrics.add(new Metric("query_hit_offset.max")); metrics.add(new Metric("query_hit_offset.sum")); metrics.add(new Metric("query_hit_offset.count")); @@ -302,6 +308,8 @@ public class VespaMetricSet { metrics.add(new Metric("totalhits_per_query.sum")); metrics.add(new Metric("totalhits_per_query.count")); metrics.add(new Metric("totalhits_per_query.average")); // TODO: Remove in Vespa 8 + metrics.add(new Metric("totalhits_per_query.95percentile")); + metrics.add(new Metric("totalhits_per_query.99percentile")); metrics.add(new Metric("empty_results.rate")); metrics.add(new Metric("requestsOverQuota.rate")); metrics.add(new Metric("requestsOverQuota.count")); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java index 2bb12654fc4..11039528fc7 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java @@ -11,6 +11,7 @@ import java.math.BigDecimal; import java.util.Locale; import java.util.Map; import java.util.Objects; +import java.util.logging.Logger; import java.util.stream.Collectors; /** @@ -19,6 +20,8 @@ import java.util.stream.Collectors; * @author ogronnesby */ public class QuotaValidator extends Validator { + private static final Logger log = Logger.getLogger(QuotaValidator.class.getName()); + @Override public void validate(VespaModel model, DeployState deployState) { var quota = deployState.getProperties().quota(); @@ -33,9 +36,13 @@ public class QuotaValidator extends Validator { .mapToDouble(clusterCapacity -> clusterCapacity.nodeResources().cost() * clusterCapacity.nodes()) .sum(); - if (budget.doubleValue() < spend) { - throwBudgetExceeded(spend, budget, systemName); + if (Math.abs(spend) < 0.01) { + log.warning("Deploying application " + model.applicationPackage().getApplicationId() + " with zero budget use. This is suspicious, but not blocked"); + return; } + + throwIfBudgetNegative(spend, budget, systemName); + throwIfBudgetExceeded(spend, budget, systemName); } /** Check that all clusters in the application do not exceed the quota max cluster size. */ @@ -57,8 +64,20 @@ public class QuotaValidator extends Validator { } } - private void throwBudgetExceeded(double spend, BigDecimal budget, SystemName systemName) { - var message = String.format(Locale.US, "Hourly spend for maximum specified resources ($%.2f) exceeds budget from quota ($%.2f)!", spend, budget); + private void throwIfBudgetNegative(double spend, BigDecimal budget, SystemName systemName) { + if (budget.doubleValue() < 0) { + throwBudgetException("Please free up some capacity! This deployment's quota use is ($%.2f) and reserved quota is below zero! ($%.2f)", spend, budget, systemName); + } + } + + private void throwIfBudgetExceeded(double spend, BigDecimal budget, SystemName systemName) { + if (budget.doubleValue() < spend) { + throwBudgetException("Please free up some capacity! This deployment's quota use ($%.2f) exceeds reserved quota ($%.2f)!", spend, budget, systemName); + } + } + + private void throwBudgetException(String formatMessage, double spend, BigDecimal budget, SystemName systemName) { + var message = String.format(Locale.US, formatMessage, spend, budget); var messageWithSystem = (systemName.equals(SystemName.Public) ? "" : systemName.value() + ": ") + message; throw new IllegalArgumentException(messageWithSystem); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java index d7681f578ff..a28475c94f3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java @@ -105,7 +105,7 @@ public class DomAdminV4Builder extends DomAdminBuilderBase { ContainerModel logserverClusterModel = new ContainerModel(context.withParent(admin).withId(logServerCluster.getSubId())); logserverClusterModel.setCluster(logServerCluster); - LogserverContainer container = new LogserverContainer(logServerCluster, deployState.featureFlags(), deployState.isHosted()); + LogserverContainer container = new LogserverContainer(logServerCluster, deployState.isHosted()); container.setHostResource(hostResource); container.initService(deployState.getDeployLogger()); logServerCluster.addContainer(container); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java index eccfc11952f..fddfcabeb0e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.model.container; import com.yahoo.cloud.config.ZookeeperServerConfig; -import com.yahoo.config.model.api.ModelContext.FeatureFlags; import com.yahoo.config.model.api.container.ContainerServiceType; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.config.provision.NodeResources; @@ -25,12 +24,12 @@ public final class ApplicationContainer extends Container implements private final boolean isHostedVespa; - public ApplicationContainer(AbstractConfigProducer<?> parent, FeatureFlags featureFlags, String name, int index, boolean isHostedVespa) { - this(parent, featureFlags, name, false, index, isHostedVespa); + public ApplicationContainer(AbstractConfigProducer<?> parent, String name, int index, boolean isHostedVespa) { + this(parent, name, false, index, isHostedVespa); } - public ApplicationContainer(AbstractConfigProducer<?> parent, FeatureFlags featureFlags, String name, boolean retired, int index, boolean isHostedVespa) { - super(parent, featureFlags, name, retired, index, isHostedVespa); + public ApplicationContainer(AbstractConfigProducer<?> parent, String name, boolean retired, int index, boolean isHostedVespa) { + super(parent, name, retired, index, isHostedVespa); this.isHostedVespa = isHostedVespa; addComponent(getFS4ResourcePool()); // TODO Remove when FS4 based search protocol is gone diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index f8ff7bcdc18..dafce877ddd 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -275,8 +275,8 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat .id(container.index()) .joining(!previousHosts.isEmpty() && !previousHosts.contains(container.getHostName())); - builder.server(serverBuilder) - .dynamicReconfiguration(true); + builder.server(serverBuilder); + builder.dynamicReconfiguration(true); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java index 0c7a38ec4dd..5e95403313c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.model.container; import com.yahoo.component.ComponentId; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.config.model.api.ModelContext.FeatureFlags; import com.yahoo.config.model.api.container.ContainerServiceType; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.container.ComponentsConfig; @@ -79,17 +78,17 @@ public abstract class Container extends AbstractService implements private final JettyHttpServer defaultHttpServer; - protected Container(AbstractConfigProducer<?> parent, FeatureFlags featureFlags, String name, int index, boolean isHostedVespa) { - this(parent, featureFlags, name, false, index, isHostedVespa); + protected Container(AbstractConfigProducer<?> parent, String name, int index, boolean isHostedVespa) { + this(parent, name, false, index, isHostedVespa); } - protected Container(AbstractConfigProducer<?> parent, FeatureFlags featureFlags, String name, boolean retired, int index, boolean isHostedVespa) { + protected Container(AbstractConfigProducer<?> parent, String name, boolean retired, int index, boolean isHostedVespa) { super(parent, name); this.name = name; this.parent = parent; this.retired = retired; this.index = index; - this.defaultHttpServer = new JettyHttpServer(new ComponentId("DefaultHttpServer"), containerClusterOrNull(parent), featureFlags, isHostedVespa); + this.defaultHttpServer = new JettyHttpServer(new ComponentId("DefaultHttpServer"), containerClusterOrNull(parent), isHostedVespa); if (getHttp() == null) { addChild(defaultHttpServer); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index 8b85c0a46ae..5e9125f560d 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -530,11 +530,9 @@ public abstract class ContainerCluster<CONTAINER extends Container> if (containerSearch != null) containerSearch.connectSearchClusters(clusterMap); } - public void addDefaultSearchAccessLog(DeployState deployState) { - var compressionType = isHostedVespa && deployState.featureFlags().enableZstdCompressionAccessLog() - ? AccessLogComponent.CompressionType.ZSTD - : AccessLogComponent.CompressionType.GZIP; - addComponent(new AccessLogComponent(AccessLogComponent.AccessLogType.jsonAccessLog, compressionType, getName(), isHostedVespa)); + public void addDefaultSearchAccessLog() { + var compressionType = isHostedVespa ? AccessLogComponent.CompressionType.ZSTD : AccessLogComponent.CompressionType.GZIP; + addComponent(new AccessLogComponent(this, AccessLogComponent.AccessLogType.jsonAccessLog, compressionType, getName(), isHostedVespa)); } @Override @@ -622,6 +620,8 @@ public abstract class ContainerCluster<CONTAINER extends Container> public void setEnvironmentVars(String environmentVars) { this.environmentVars = environmentVars; } + public String getEnvironmentVars() { return environmentVars; } + public Optional<String> getJvmGCOptions() { return Optional.ofNullable(jvmGCOptions); } public final void setRpcServerEnabled(boolean rpcServerEnabled) { this.rpcServerEnabled = rpcServerEnabled; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerThreadpool.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerThreadpool.java index 5074f1ecbbe..911e5e2e9bd 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerThreadpool.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerThreadpool.java @@ -22,8 +22,6 @@ public class ContainerThreadpool extends SimpleComponent implements ContainerThr private final String name; private final UserOptions userOptions; - public ContainerThreadpool(String name) { this(name, null); } - public ContainerThreadpool(String name, UserOptions userOptions) { super(new ComponentModel( BundleInstantiationSpecification.getFromStrings( diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/DefaultThreadpoolProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/container/DefaultThreadpoolProvider.java index 2ca97b297b0..3dae3160c51 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/DefaultThreadpoolProvider.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/DefaultThreadpoolProvider.java @@ -5,6 +5,7 @@ import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.container.handler.ThreadPoolProvider; import com.yahoo.container.handler.ThreadpoolConfig; import com.yahoo.osgi.provider.model.ComponentModel; +import com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster; import com.yahoo.vespa.model.container.component.SimpleComponent; /** @@ -25,11 +26,18 @@ class DefaultThreadpoolProvider extends SimpleComponent implements ThreadpoolCon this.cluster = cluster; } + private int defaultThreadsByClusterType() { + if (cluster instanceof MetricsProxyContainerCluster) { + return 4; + } + return 10; + } + @Override public void getConfig(ThreadpoolConfig.Builder builder) { if (!(cluster instanceof ApplicationContainerCluster)) { // Container clusters such as logserver, metricsproxy and clustercontroller - int defaultWorkerThreads = 10; + int defaultWorkerThreads = defaultThreadsByClusterType(); builder.maxthreads(defaultWorkerThreads); builder.corePoolSize(defaultWorkerThreads); builder.queueSize(50); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java index ffb7b876fa2..2905471b02e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java @@ -3,9 +3,12 @@ package com.yahoo.vespa.model.container.component; import com.yahoo.container.core.AccessLogConfig; import com.yahoo.container.core.AccessLogConfig.FileHandler.CompressionFormat; -import com.yahoo.container.logging.VespaAccessLog; import com.yahoo.container.logging.JSONAccessLog; +import com.yahoo.container.logging.VespaAccessLog; import com.yahoo.osgi.provider.model.ComponentModel; +import com.yahoo.vespa.model.container.ContainerCluster; + +import java.util.OptionalInt; /** * @author Tony Vaagenes @@ -13,6 +16,7 @@ import com.yahoo.osgi.provider.model.ComponentModel; */ public final class AccessLogComponent extends SimpleComponent implements AccessLogConfig.Producer { + public enum AccessLogType { queryAccessLog, yApacheAccessLog, jsonAccessLog } public enum CompressionType { GZIP, ZSTD } @@ -22,10 +26,11 @@ public final class AccessLogComponent extends SimpleComponent implements AccessL private final boolean isHostedVespa; private final String symlinkName; private final CompressionType compressionType; + private final int queueSize; - public AccessLogComponent(AccessLogType logType, CompressionType compressionType, String clusterName, boolean isHostedVespa) + public AccessLogComponent(ContainerCluster<?> cluster, AccessLogType logType, CompressionType compressionType, String clusterName, boolean isHostedVespa) { - this(logType, compressionType, + this(cluster, logType, compressionType, String.format("logs/vespa/qrs/%s.%s.%s", capitalize(logType.name()), clusterName, "%Y%m%d%H%M%S"), null, null, isHostedVespa, capitalize(logType.name()) + "." + clusterName); @@ -35,7 +40,8 @@ public final class AccessLogComponent extends SimpleComponent implements AccessL return name.substring(0, 1).toUpperCase() + name.substring(1); } - public AccessLogComponent(AccessLogType logType, + public AccessLogComponent(ContainerCluster<?> cluster, + AccessLogType logType, CompressionType compressionType, String fileNamePattern, String rotationInterval, @@ -50,11 +56,19 @@ public final class AccessLogComponent extends SimpleComponent implements AccessL this.isHostedVespa = isHostedVespa; this.symlinkName = symlinkName; this.compressionType = compressionType; + this.queueSize = queueSize(cluster).orElse(-1); if (fileNamePattern == null) throw new RuntimeException("File name pattern required when configuring access log."); } + private static OptionalInt queueSize(ContainerCluster<?> cluster) { + if (cluster == null) return OptionalInt.empty(); + double vcpu = cluster.vcpu().orElse(0); + if (vcpu <= 0) return OptionalInt.empty(); + return OptionalInt.of((int) Math.max(4096, Math.ceil(vcpu * 256.0))); + } + private static String accessLogClass(AccessLogType logType) { switch (logType) { case queryAccessLog: @@ -84,6 +98,9 @@ public final class AccessLogComponent extends SimpleComponent implements AccessL } else if (isHostedVespa) { builder.compressOnRotation(true); } + if (queueSize >= 0) { + builder.queueSize(queueSize); + } switch (compressionType) { case GZIP: builder.compressionFormat(CompressionFormat.GZIP); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/ConnectionLogComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/ConnectionLogComponent.java index bf279c5c364..8eff7bf7201 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/ConnectionLogComponent.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/ConnectionLogComponent.java @@ -5,18 +5,40 @@ package com.yahoo.vespa.model.container.component; import com.yahoo.container.logging.ConnectionLog; import com.yahoo.container.logging.ConnectionLogConfig; import com.yahoo.osgi.provider.model.ComponentModel; +import com.yahoo.vespa.model.container.ContainerCluster; + +import java.util.OptionalInt; public class ConnectionLogComponent extends SimpleComponent implements ConnectionLogConfig.Producer { + private final String logDirectoryName; private final String clusterName; + private final int queueSize; + + public ConnectionLogComponent(ContainerCluster<?> cluster, Class<? extends ConnectionLog> cls, String logDirectoryName) { + this(cluster, cls, logDirectoryName, cluster.getName()); + } - public ConnectionLogComponent(Class<? extends ConnectionLog> cls, String clusterName) { + public ConnectionLogComponent(ContainerCluster<?> cluster, Class<? extends ConnectionLog> cls, String logDirectoryName, String clusterName) { super(new ComponentModel(cls.getName(), null, "jdisc_http_service", null)); + this.logDirectoryName = logDirectoryName; this.clusterName = clusterName; + this.queueSize = queueSize(cluster).orElse(-1); + } + + private static OptionalInt queueSize(ContainerCluster<?> cluster) { + if (cluster == null) return OptionalInt.empty(); + double vcpu = cluster.vcpu().orElse(0); + if (vcpu <= 0) return OptionalInt.empty(); + return OptionalInt.of((int) Math.max(4096, Math.ceil(vcpu * 512.0))); } @Override public void getConfig(ConnectionLogConfig.Builder builder) { builder.cluster(clusterName); + builder.logDirectoryName(logDirectoryName); + if (queueSize >= 0) { + builder.queueSize(queueSize); + } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java index 69c1f9ca10e..b28a2edff0e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.model.container.http; import com.yahoo.component.ComponentId; import com.yahoo.component.ComponentSpecification; -import com.yahoo.config.model.api.ModelContext; import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.jdisc.http.ServerConfig; import com.yahoo.osgi.provider.model.ComponentModel; @@ -24,11 +23,10 @@ import static com.yahoo.component.ComponentSpecification.fromString; public class JettyHttpServer extends SimpleComponent implements ServerConfig.Producer { private final ContainerCluster<?> cluster; - private final boolean isHostedVespa; + private volatile boolean isHostedVespa; private final List<ConnectorFactory> connectorFactories = new ArrayList<>(); - private final boolean enableJdiscConnectionLog; - public JettyHttpServer(ComponentId id, ContainerCluster<?> cluster, ModelContext.FeatureFlags featureFlags, boolean isHostedVespa) { + public JettyHttpServer(ComponentId id, ContainerCluster<?> cluster, boolean isHostedVespa) { super(new ComponentModel( new BundleInstantiationSpecification(id, fromString("com.yahoo.jdisc.http.server.jetty.JettyHttpServer"), @@ -39,9 +37,10 @@ public class JettyHttpServer extends SimpleComponent implements ServerConfig.Pro final FilterBindingsProviderComponent filterBindingsProviderComponent = new FilterBindingsProviderComponent(id); addChild(filterBindingsProviderComponent); inject(filterBindingsProviderComponent); - this.enableJdiscConnectionLog = featureFlags.enableJdiscConnectionLog(); } + public void setHostedVespa(boolean isHostedVespa) { this.isHostedVespa = isHostedVespa; } + public void addConnector(ConnectorFactory connectorFactory) { connectorFactories.add(connectorFactory); addChild(connectorFactory); @@ -62,6 +61,9 @@ public class JettyHttpServer extends SimpleComponent implements ServerConfig.Pro builder.accessLog(new ServerConfig.AccessLog.Builder() .remoteAddressHeaders(List.of()) .remotePortHeaders(List.of())); + + // Enable connection log hosted Vespa + builder.connectionLog(new ServerConfig.ConnectionLog.Builder().enabled(true)); } else { // TODO Vespa 8: Remove legacy Yahoo headers builder.accessLog(new ServerConfig.AccessLog.Builder() @@ -69,8 +71,6 @@ public class JettyHttpServer extends SimpleComponent implements ServerConfig.Pro .remotePortHeaders(List.of("X-Forwarded-Port", "y-rp"))); } configureJettyThreadpool(builder); - builder.connectionLog(new ServerConfig.ConnectionLog.Builder() - .enabled(enableJdiscConnectionLog)); } private void configureJettyThreadpool(ServerConfig.Builder builder) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyHttpServerBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyHttpServerBuilder.java index 301bcc07aa8..f3e82d515df 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyHttpServerBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyHttpServerBuilder.java @@ -23,9 +23,9 @@ public class JettyHttpServerBuilder extends VespaDomBuilder.DomConfigProducerBui } @Override - protected JettyHttpServer doBuild(DeployState deployState, AbstractConfigProducer ancestor, Element http) { + protected JettyHttpServer doBuild(DeployState deployState, AbstractConfigProducer<?> ancestor, Element http) { JettyHttpServer jettyHttpServer = new JettyHttpServer( - new ComponentId("jdisc-jetty"), cluster, deployState.featureFlags(), deployState.isHosted()); + new ComponentId("jdisc-jetty"), cluster, deployState.isHosted()); for (Element serverSpec: XML.getChildren(http, "server")) { ConnectorFactory connectorFactory = new JettyConnectorBuilder().build(deployState, ancestor, serverSpec); jettyHttpServer.addConnector(connectorFactory); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java index 6a858bd2e02..f84d6f0724b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java @@ -55,8 +55,9 @@ public class AccessLogBuilder { @Override protected AccessLogComponent doBuild(DeployState deployState, AbstractConfigProducer<?> ancestor, Element spec) { return new AccessLogComponent( + (ContainerCluster<?>) ancestor, accessLogType, - compressionType(spec, deployState, isHostedVespa), + compressionType(spec, isHostedVespa), fileNamePattern(spec), rotationInterval(spec), compressOnRotation(spec), @@ -81,7 +82,8 @@ public class AccessLogBuilder { return nullIfEmpty(spec.getAttribute("fileNamePattern")); } - private static CompressionType compressionType(Element spec, DeployState deployState, boolean isHostedVespa) { + private static CompressionType compressionType(Element spec, boolean isHostedVespa) { + CompressionType fallback = isHostedVespa ? CompressionType.ZSTD : CompressionType.GZIP; return Optional.ofNullable(spec.getAttribute("compressionType")) .filter(value -> !value.isBlank()) .map(value -> { @@ -94,7 +96,7 @@ public class AccessLogBuilder { throw new IllegalArgumentException("Unknown compression type: " + value); } }) - .orElse(isHostedVespa && deployState.featureFlags().enableZstdCompressionAccessLog() ? CompressionType.ZSTD : CompressionType.GZIP); + .orElse(fallback); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java index ab734c506c1..7e0be6e448b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java @@ -2,8 +2,12 @@ package com.yahoo.vespa.model.container.xml; import com.yahoo.config.model.ConfigModelContext; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.container.logging.FileConnectionLog; import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.ContainerModel; +import com.yahoo.vespa.model.container.component.AccessLogComponent; +import com.yahoo.vespa.model.container.component.ConnectionLogComponent; import com.yahoo.vespa.model.container.configserver.ConfigserverCluster; import com.yahoo.vespa.model.container.configserver.option.CloudConfigOptions; import org.w3c.dom.Element; @@ -34,7 +38,29 @@ public class ConfigServerContainerModelBuilder extends ContainerModelBuilder { // in ConfigModelContext.DeployState.properties are not set) @Override protected void addStatusHandlers(ApplicationContainerCluster cluster, boolean isHostedVespa) { - super.addStatusHandlers(cluster, options.hostedVespa().orElse(Boolean.FALSE)); + super.addStatusHandlers(cluster, isHosted()); } + // Override access log configuration for hosted configserver/controller + @Override + protected void addAccessLogs(DeployState deployState, ApplicationContainerCluster cluster, Element spec) { + if (isHosted()){ + cluster.addComponent( + new AccessLogComponent( + cluster, AccessLogComponent.AccessLogType.jsonAccessLog, AccessLogComponent.CompressionType.ZSTD, + "logs/vespa/configserver/access-json.log.%Y%m%d%H%M%S", null, true, true, "access-json.log")); + cluster.addComponent(new ConnectionLogComponent(cluster, FileConnectionLog.class, "configserver")); + } else { + super.addAccessLogs(deployState, cluster, spec); + } + } + + @Override + protected void addHttp(DeployState deployState, Element spec, ApplicationContainerCluster cluster, ConfigModelContext context) { + super.addHttp(deployState, spec, cluster, context); + cluster.getHttp().getHttpServer().get().setHostedVespa(isHosted()); + } + + /** Note: using {@link CloudConfigOptions} as {@link DeployState#isHosted()} returns <em>false</em> for hosted configserver/controller */ + private boolean isHosted() { return options.hostedVespa().orElse(Boolean.FALSE); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index d650b10a910..d9543c2e917 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -14,7 +14,6 @@ import com.yahoo.config.model.ConfigModelContext.ApplicationType; import com.yahoo.config.model.api.ConfigServerSpec; import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.EndpointCertificateSecrets; -import com.yahoo.config.model.api.ModelContext.FeatureFlags; import com.yahoo.config.model.application.provider.IncludeDirs; import com.yahoo.config.model.builder.xml.ConfigModelBuilder; import com.yahoo.config.model.builder.xml.ConfigModelId; @@ -338,7 +337,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { addConfiguredComponents(deployState, cluster, spec, "server"); } - private void addAccessLogs(DeployState deployState, ApplicationContainerCluster cluster, Element spec) { + protected void addAccessLogs(DeployState deployState, ApplicationContainerCluster cluster, Element spec) { List<Element> accessLogElements = getAccessLogElements(spec); for (Element accessLog : accessLogElements) { @@ -346,13 +345,13 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { } if (accessLogElements.isEmpty() && deployState.getAccessLoggingEnabledByDefault()) - cluster.addDefaultSearchAccessLog(deployState); + cluster.addDefaultSearchAccessLog(); // Add connection log if access log is configured if (cluster.getAllComponents().stream().anyMatch(component -> component instanceof AccessLogComponent)) { - cluster.addComponent(new ConnectionLogComponent(FileConnectionLog.class, cluster.getName())); + cluster.addComponent(new ConnectionLogComponent(cluster, FileConnectionLog.class, "qrs")); } else { - cluster.addComponent(new ConnectionLogComponent(VoidConnectionLog.class, cluster.getName())); + cluster.addComponent(new ConnectionLogComponent(cluster, VoidConnectionLog.class, "qrs")); } } @@ -361,13 +360,13 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { } - private void addHttp(DeployState deployState, Element spec, ApplicationContainerCluster cluster, ConfigModelContext context) { + protected void addHttp(DeployState deployState, Element spec, ApplicationContainerCluster cluster, ConfigModelContext context) { Element httpElement = XML.getChild(spec, "http"); if (httpElement != null) { cluster.setHttp(buildHttp(deployState, cluster, httpElement)); } if (isHostedTenantApplication(context)) { - addHostedImplicitHttpIfNotPresent(cluster, deployState); + addHostedImplicitHttpIfNotPresent(cluster); addHostedImplicitAccessControlIfNotPresent(deployState, cluster); addDefaultConnectorHostedFilterBinding(cluster); addAdditionalHostedConnector(deployState, cluster, context); @@ -414,13 +413,13 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { return deployState.isHosted() && context.getApplicationType() == ApplicationType.DEFAULT && !isTesterApplication; } - private static void addHostedImplicitHttpIfNotPresent(ApplicationContainerCluster cluster, DeployState deployState) { + private static void addHostedImplicitHttpIfNotPresent(ApplicationContainerCluster cluster) { if(cluster.getHttp() == null) { cluster.setHttp(new Http(new FilterChains(cluster))); } JettyHttpServer httpServer = cluster.getHttp().getHttpServer().orElse(null); if (httpServer == null) { - httpServer = new JettyHttpServer(new ComponentId("DefaultHttpServer"), cluster, deployState.featureFlags(), cluster.isHostedVespa()); + httpServer = new JettyHttpServer(new ComponentId("DefaultHttpServer"), cluster, cluster.isHostedVespa()); cluster.getHttp().setHttpServer(httpServer); } int defaultPort = Defaults.getDefaults().vespaWebServicePort(); @@ -489,7 +488,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { addIncludes(searchElement); cluster.setSearch(buildSearch(deployState, cluster, searchElement)); - addSearchHandler(cluster, searchElement, deployState); + addSearchHandler(cluster, searchElement); addGUIHandler(cluster); validateAndAddConfiguredComponents(deployState, cluster, searchElement, "renderer", ContainerModelBuilder::validateRendererElement); } @@ -554,13 +553,13 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { private void addNodes(ApplicationContainerCluster cluster, Element spec, ConfigModelContext context) { if (standaloneBuilder) - addStandaloneNode(cluster, context.featureFlags()); + addStandaloneNode(cluster); else addNodesFromXml(cluster, spec, context); } - private void addStandaloneNode(ApplicationContainerCluster cluster, FeatureFlags featureFlags) { - ApplicationContainer container = new ApplicationContainer(cluster, featureFlags, "standalone", cluster.getContainers().size(), cluster.isHostedVespa()); + private void addStandaloneNode(ApplicationContainerCluster cluster) { + ApplicationContainer container = new ApplicationContainer(cluster, "standalone", cluster.getContainers().size(), cluster.isHostedVespa()); cluster.addContainers(Collections.singleton(container)); } @@ -724,13 +723,13 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { false, !deployState.getProperties().isBootstrap()); var hosts = hostSystem.allocateHosts(clusterSpec, capacity, log); - return createNodesFromHosts(log, context.featureFlags(), hosts, cluster); + return createNodesFromHosts(log, hosts, cluster); } return singleHostContainerCluster(cluster, hostSystem.getHost(Container.SINGLENODE_CONTAINER_SERVICESPEC), context); } private List<ApplicationContainer> singleHostContainerCluster(ApplicationContainerCluster cluster, HostResource host, ConfigModelContext context) { - ApplicationContainer node = new ApplicationContainer(cluster, context.featureFlags(), "container.0", 0, cluster.isHostedVespa()); + ApplicationContainer node = new ApplicationContainer(cluster, "container.0", 0, cluster.isHostedVespa()); node.setHostResource(host); node.initService(context.getDeployLogger()); return List.of(node); @@ -743,7 +742,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { ClusterSpec.Id.from(cluster.getName()), log, hasZooKeeper(containerElement)); - return createNodesFromHosts(context.getDeployLogger(), context.featureFlags(), hosts, cluster); + return createNodesFromHosts(context.getDeployLogger(), hosts, cluster); } private List<ApplicationContainer> createNodesFromNodeType(ApplicationContainerCluster cluster, Element nodesElement, ConfigModelContext context) { @@ -755,7 +754,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { Map<HostResource, ClusterMembership> hosts = cluster.getRoot().hostSystem().allocateHosts(clusterSpec, Capacity.fromRequiredNodeType(type), log); - return createNodesFromHosts(context.getDeployLogger(), context.featureFlags(), hosts, cluster); + return createNodesFromHosts(context.getDeployLogger(), hosts, cluster); } private List<ApplicationContainer> createNodesFromContentServiceReference(ApplicationContainerCluster cluster, Element nodesElement, ConfigModelContext context) { @@ -773,14 +772,14 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { referenceId, cluster.getRoot().hostSystem(), context.getDeployLogger()); - return createNodesFromHosts(context.getDeployLogger(), context.featureFlags(), hosts, cluster); + return createNodesFromHosts(context.getDeployLogger(), hosts, cluster); } - private List<ApplicationContainer> createNodesFromHosts(DeployLogger deployLogger, FeatureFlags featureFlags, Map<HostResource, ClusterMembership> hosts, ApplicationContainerCluster cluster) { + private List<ApplicationContainer> createNodesFromHosts(DeployLogger deployLogger, Map<HostResource, ClusterMembership> hosts, ApplicationContainerCluster cluster) { List<ApplicationContainer> nodes = new ArrayList<>(); for (Map.Entry<HostResource, ClusterMembership> entry : hosts.entrySet()) { String id = "container." + entry.getValue().index(); - ApplicationContainer container = new ApplicationContainer(cluster, featureFlags, id, entry.getValue().retired(), entry.getValue().index(), cluster.isHostedVespa()); + ApplicationContainer container = new ApplicationContainer(cluster, id, entry.getValue().retired(), entry.getValue().index(), cluster.isHostedVespa()); container.setHostResource(entry.getKey()); container.initService(deployLogger); nodes.add(container); @@ -818,7 +817,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { container.setPreLoad(nodesElement.getAttribute(VespaDomBuilder.PRELOAD_ATTRIB_NAME)); } - private void addSearchHandler(ApplicationContainerCluster cluster, Element searchElement, DeployState deployState) { + private void addSearchHandler(ApplicationContainerCluster cluster, Element searchElement) { // Magic spell is needed to receive the chains config :-| cluster.addComponent(new ProcessingHandler<>(cluster.getSearch().getChains(), "com.yahoo.search.searchchain.ExecutionFactory")); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerServiceBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerServiceBuilder.java index ea4924d9aec..3fae9d52929 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerServiceBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerServiceBuilder.java @@ -21,8 +21,8 @@ public class ContainerServiceBuilder extends VespaDomBuilder.DomConfigProducerBu } @Override - protected ApplicationContainer doBuild(DeployState deployState, AbstractConfigProducer parent, Element nodeElem) { - return new ApplicationContainer(parent, deployState.featureFlags(), id, index, deployState.isHosted()); + protected ApplicationContainer doBuild(DeployState deployState, AbstractConfigProducer<?> parent, Element nodeElem) { + return new ApplicationContainer(parent, id, index, deployState.isHosted()); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterControllerConfig.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterControllerConfig.java index 8d6127970c8..66ec0d81947 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterControllerConfig.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterControllerConfig.java @@ -19,12 +19,14 @@ import org.w3c.dom.Element; public class ClusterControllerConfig extends AbstractConfigProducer<ClusterControllerConfig> implements FleetcontrollerConfig.Producer { public static class Builder extends VespaDomBuilder.DomConfigProducerBuilder<ClusterControllerConfig> { - String clusterName; - ModelElement clusterElement; + private final String clusterName; + private final ModelElement clusterElement; + private final ResourceLimits resourceLimits; - public Builder(String clusterName, ModelElement clusterElement) { + public Builder(String clusterName, ModelElement clusterElement, ResourceLimits resourceLimits) { this.clusterName = clusterName; this.clusterElement = clusterElement; + this.resourceLimits = resourceLimits; } @Override @@ -51,27 +53,29 @@ public class ClusterControllerConfig extends AbstractConfigProducer<ClusterContr tuning.childAsDouble("min-storage-up-ratio"), bucketSplittingMinimumBits, minNodeRatioPerGroup, - enableClusterFeedBlock); + enableClusterFeedBlock, + resourceLimits); } else { return new ClusterControllerConfig(ancestor, clusterName, null, null, null, null, null, null, bucketSplittingMinimumBits, minNodeRatioPerGroup, - enableClusterFeedBlock); + enableClusterFeedBlock, resourceLimits); } } } - String clusterName; - Duration initProgressTime; - Duration transitionTime; - Long maxPrematureCrashes; - Duration stableStateTimePeriod; - Double minDistributorUpRatio; - Double minStorageUpRatio; - Integer minSplitBits; - private Double minNodeRatioPerGroup; - private boolean enableClusterFeedBlock = false; + private final String clusterName; + private final Duration initProgressTime; + private final Duration transitionTime; + private final Long maxPrematureCrashes; + private final Duration stableStateTimePeriod; + private final Double minDistributorUpRatio; + private final Double minStorageUpRatio; + private final Integer minSplitBits; + private final Double minNodeRatioPerGroup; + private final boolean enableClusterFeedBlock; + private final ResourceLimits resourceLimits; // TODO refactor; too many args private ClusterControllerConfig(AbstractConfigProducer parent, @@ -84,7 +88,8 @@ public class ClusterControllerConfig extends AbstractConfigProducer<ClusterContr Double minStorageUpRatio, Integer minSplitBits, Double minNodeRatioPerGroup, - boolean enableClusterFeedBlock) { + boolean enableClusterFeedBlock, + ResourceLimits resourceLimits) { super(parent, "fleetcontroller"); this.clusterName = clusterName; @@ -97,6 +102,7 @@ public class ClusterControllerConfig extends AbstractConfigProducer<ClusterContr this.minSplitBits = minSplitBits; this.minNodeRatioPerGroup = minNodeRatioPerGroup; this.enableClusterFeedBlock = enableClusterFeedBlock; + this.resourceLimits = resourceLimits; } @Override @@ -139,18 +145,7 @@ public class ClusterControllerConfig extends AbstractConfigProducer<ClusterContr builder.min_node_ratio_per_group(minNodeRatioPerGroup); } builder.enable_cluster_feed_block(enableClusterFeedBlock); - setDefaultClusterFeedBlockLimits(builder); + resourceLimits.getConfig(builder); } - private static void setDefaultClusterFeedBlockLimits(FleetcontrollerConfig.Builder builder) { - // TODO: Override these based on resource-limits in services.xml (if they are specified). - // TODO: Choose other defaults when this is default enabled. - // Note: The resource categories must match the ones used in host info reporting - // between content nodes and cluster controller: - // storage/src/vespa/storage/persistence/filestorage/service_layer_host_info_reporter.cpp - builder.cluster_feed_block_limit.put("memory", 0.79); - builder.cluster_feed_block_limit.put("disk", 0.79); - builder.cluster_feed_block_limit.put("attribute-enum-store", 0.89); - builder.cluster_feed_block_limit.put("attribute-multi-value", 0.89); - } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterResourceLimits.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterResourceLimits.java new file mode 100644 index 00000000000..c7d56c9b4b5 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterResourceLimits.java @@ -0,0 +1,122 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.content; + +import com.yahoo.vespa.model.builder.xml.dom.ModelElement; +import com.yahoo.vespa.model.content.cluster.DomResourceLimitsBuilder; + +import java.util.Optional; +import java.util.function.Consumer; + +/** + * Class tracking the feed block resource limits for a content cluster. + * + * This includes the limits used by the cluster controller and the content nodes (proton). + * + * @author geirst + */ +public class ClusterResourceLimits { + + private final ResourceLimits clusterControllerLimits; + private final ResourceLimits contentNodeLimits; + + private ClusterResourceLimits(Builder builder) { + clusterControllerLimits = builder.ctrlBuilder.build(); + contentNodeLimits = builder.nodeBuilder.build(); + } + + public ResourceLimits getClusterControllerLimits() { + return clusterControllerLimits; + } + + public ResourceLimits getContentNodeLimits() { + return contentNodeLimits; + } + + public static class Builder { + + private final boolean enableFeedBlockInDistributor; + private ResourceLimits.Builder ctrlBuilder = new ResourceLimits.Builder(); + private ResourceLimits.Builder nodeBuilder = new ResourceLimits.Builder(); + + public Builder(boolean enableFeedBlockInDistributor) { + this.enableFeedBlockInDistributor = enableFeedBlockInDistributor; + } + + public ClusterResourceLimits build(ModelElement clusterElem) { + ModelElement tuningElem = clusterElem.childByPath("tuning"); + if (tuningElem != null) { + ctrlBuilder = DomResourceLimitsBuilder.createBuilder(tuningElem); + } + + ModelElement protonElem = clusterElem.childByPath("engine.proton"); + if (protonElem != null) { + nodeBuilder = DomResourceLimitsBuilder.createBuilder(protonElem); + } + + deriveLimits(); + return new ClusterResourceLimits(this); + } + + public void setClusterControllerBuilder(ResourceLimits.Builder builder) { + ctrlBuilder = builder; + } + + public void setContentNodeBuilder(ResourceLimits.Builder builder) { + nodeBuilder = builder; + } + + public ClusterResourceLimits build() { + deriveLimits(); + return new ClusterResourceLimits(this); + } + + private void deriveLimits() { + if (enableFeedBlockInDistributor) { + // This also ensures that content nodes limits are derived according to the formula in calcContentNodeLimit(). + considerSettingDefaultClusterControllerLimit(ctrlBuilder.getDiskLimit(), nodeBuilder.getDiskLimit(), ctrlBuilder::setDiskLimit); + considerSettingDefaultClusterControllerLimit(ctrlBuilder.getMemoryLimit(), nodeBuilder.getMemoryLimit(), ctrlBuilder::setMemoryLimit); + } + + deriveClusterControllerLimit(ctrlBuilder.getDiskLimit(), nodeBuilder.getDiskLimit(), ctrlBuilder::setDiskLimit); + deriveClusterControllerLimit(ctrlBuilder.getMemoryLimit(), nodeBuilder.getMemoryLimit(), ctrlBuilder::setMemoryLimit); + + deriveContentNodeLimit(nodeBuilder.getDiskLimit(), ctrlBuilder.getDiskLimit(), nodeBuilder::setDiskLimit); + deriveContentNodeLimit(nodeBuilder.getMemoryLimit(), ctrlBuilder.getMemoryLimit(), nodeBuilder::setMemoryLimit); + } + + private void considerSettingDefaultClusterControllerLimit(Optional<Double> clusterControllerLimit, + Optional<Double> contentNodeLimit, + Consumer<Double> setter) { + // TODO: remove this when feed block in distributor is default enabled. + if (!clusterControllerLimit.isPresent() && !contentNodeLimit.isPresent()) { + setter.accept(0.8); + } + } + + private void deriveClusterControllerLimit(Optional<Double> clusterControllerLimit, + Optional<Double> contentNodeLimit, + Consumer<Double> setter) { + if (!clusterControllerLimit.isPresent()) { + contentNodeLimit.ifPresent(limit -> + // TODO: emit warning when feed block in distributor is default enabled. + setter.accept(Double.max(0.0, (limit - 0.01)))); + } + } + + private void deriveContentNodeLimit(Optional<Double> contentNodeLimit, + Optional<Double> clusterControllerLimit, + Consumer<Double> setter) { + if (!contentNodeLimit.isPresent()) { + clusterControllerLimit.ifPresent(limit -> + setter.accept(calcContentNodeLimit(limit))); + } + } + + private double calcContentNodeLimit(double clusterControllerLimit) { + // Note that validation in the range [0.0-1.0] is handled by the rnc schema. + return clusterControllerLimit + ((1.0 - clusterControllerLimit) / 2); + } + + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java b/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java index babfe3603b0..b93f4405333 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java @@ -298,7 +298,7 @@ public class Content extends ConfigModel { HostResource host = searchNode.getHostResource(); if (!processedHosts.contains(host)) { String containerName = String.valueOf(searchNode.getDistributionKey()); - ApplicationContainer docprocService = new ApplicationContainer(indexingCluster, modelContext.featureFlags(), containerName, index, + ApplicationContainer docprocService = new ApplicationContainer(indexingCluster, containerName, index, modelContext.getDeployState().isHosted()); index++; docprocService.useDynamicPorts(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index dd29df61f35..0fdd39e70cf 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -66,6 +66,7 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> private final ProtonConfig.Indexing.Optimize.Enum feedSequencerType; private final double defaultFeedConcurrency; private final boolean useBucketExecutorForLidSpaceCompact; + private final double defaultMaxDeadBytesRatio; /** Whether the nodes of this cluster also hosts a container cluster in a hosted system */ private final boolean combined; @@ -79,13 +80,15 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> private final Map<String, NewDocumentType> documentDefinitions; private final Set<NewDocumentType> globallyDistributedDocuments; private final boolean combined; + private final ResourceLimits resourceLimits; public Builder(Map<String, NewDocumentType> documentDefinitions, Set<NewDocumentType> globallyDistributedDocuments, - boolean combined) { + boolean combined, ResourceLimits resourceLimits) { this.documentDefinitions = documentDefinitions; this.globallyDistributedDocuments = globallyDistributedDocuments; this.combined = combined; + this.resourceLimits = resourceLimits; } @Override @@ -106,10 +109,7 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> if (tuning != null) { search.setTuning(new DomSearchTuningBuilder().build(deployState, search, tuning.getXml())); } - ModelElement protonElem = clusterElem.childByPath("engine.proton"); - if (protonElem != null) { - search.setResourceLimits(DomResourceLimitsBuilder.build(protonElem)); - } + search.setResourceLimits(resourceLimits); buildAllStreamingSearchClusters(deployState, clusterElem, clusterName, search); buildIndexedSearchCluster(deployState, clusterElem, clusterName, search); @@ -190,7 +190,7 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> } } - private ContentSearchCluster(AbstractConfigProducer parent, + private ContentSearchCluster(AbstractConfigProducer<?> parent, String clusterName, ModelContext.FeatureFlags featureFlags, Map<String, NewDocumentType> documentDefinitions, @@ -207,6 +207,7 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> feedSequencerType = convertFeedSequencerType(featureFlags.feedSequencerType()); defaultFeedConcurrency = featureFlags.feedConcurrency(); useBucketExecutorForLidSpaceCompact = featureFlags.useBucketExecutorForLidSpaceCompact(); + defaultMaxDeadBytesRatio = featureFlags.maxDeadBytesRatio(); } public void setVisibilityDelay(double delay) { @@ -233,8 +234,8 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> NamedSchema searchDefinition = schemaDefinitionXMLHandler.getResponsibleSearchDefinition(deployState.getSchemas()); if (searchDefinition == null) - throw new RuntimeException("Search definition parsing error or file does not exist: '" + - schemaDefinitionXMLHandler.getName() + "'"); + throw new RuntimeException("Schema '" + schemaDefinitionXMLHandler.getName() + "' referenced in " + + this + " does not exist"); // TODO: remove explicit building of user configs when the complete content model is built using builders. sc.getLocalSDS().add(new AbstractSearchCluster.SchemaSpec(searchDefinition, @@ -377,6 +378,7 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> .configid(getConfigId()) .visibilitydelay(visibilityDelay) .global(globalDocType); + ddbB.allocation.max_dead_bytes_ratio(defaultMaxDeadBytesRatio); if (hasIndexingModeStreaming(type)) { hasAnyNonIndexedCluster = true; @@ -445,4 +447,8 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> public IndexedSearchCluster getIndexed() { return indexedCluster; } public boolean hasIndexedCluster() { return indexedCluster != null; } public String getClusterName() { return clusterName; } + + @Override + public String toString() { return "content cluster '" + clusterName + "'"; } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ResourceLimits.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ResourceLimits.java index 28e8c36d202..a12e183b409 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ResourceLimits.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ResourceLimits.java @@ -1,16 +1,17 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.content; +import com.yahoo.vespa.config.content.FleetcontrollerConfig; import com.yahoo.vespa.config.search.core.ProtonConfig; import java.util.Optional; /** - * Class tracking resource limits for a content cluster with engine proton. + * Class tracking feed block resource limits used by a component in a content cluster (e.g. cluster controller or content node). * * @author geirst */ -public class ResourceLimits implements ProtonConfig.Producer { +public class ResourceLimits implements FleetcontrollerConfig.Producer, ProtonConfig.Producer { private final Optional<Double> diskLimit; private final Optional<Double> memoryLimit; @@ -20,6 +21,26 @@ public class ResourceLimits implements ProtonConfig.Producer { this.memoryLimit = builder.memoryLimit; } + public Optional<Double> getDiskLimit() { + return diskLimit; + } + + public Optional<Double> getMemoryLimit() { + return memoryLimit; + } + + @Override + public void getConfig(FleetcontrollerConfig.Builder builder) { + // TODO: Choose other defaults when this is default enabled. + // Note: The resource categories must match the ones used in host info reporting + // between content nodes and cluster controller: + // storage/src/vespa/storage/persistence/filestorage/service_layer_host_info_reporter.cpp + builder.cluster_feed_block_limit.put("memory", memoryLimit.orElse(0.8)); + builder.cluster_feed_block_limit.put("disk", diskLimit.orElse(0.8)); + builder.cluster_feed_block_limit.put("attribute-enum-store", 0.89); + builder.cluster_feed_block_limit.put("attribute-multi-value", 0.89); + } + @Override public void getConfig(ProtonConfig.Builder builder) { if (diskLimit.isPresent()) { @@ -39,11 +60,19 @@ public class ResourceLimits implements ProtonConfig.Producer { return new ResourceLimits(this); } + public Optional<Double> getDiskLimit() { + return diskLimit; + } + public Builder setDiskLimit(double diskLimit) { this.diskLimit = Optional.of(diskLimit); return this; } + public Optional<Double> getMemoryLimit() { + return memoryLimit; + } + public Builder setMemoryLimit(double memoryLimit) { this.memoryLimit = Optional.of(memoryLimit); return this; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index a627e030156..f27c84d19c3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -22,7 +22,6 @@ import com.yahoo.vespa.config.content.StorDistributionConfig; import com.yahoo.vespa.config.content.core.BucketspacesConfig; import com.yahoo.vespa.config.content.core.StorDistributormanagerConfig; import com.yahoo.vespa.model.HostResource; -import com.yahoo.vespa.model.Service; import com.yahoo.vespa.model.admin.Admin; import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerCluster; import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerComponent; @@ -33,11 +32,10 @@ import com.yahoo.vespa.model.admin.clustercontroller.ReindexingContext; import com.yahoo.vespa.model.admin.monitoring.Monitoring; import com.yahoo.vespa.model.builder.xml.dom.ModelElement; import com.yahoo.vespa.model.builder.xml.dom.NodesSpecification; -import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.Container; -import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.ContainerModel; import com.yahoo.vespa.model.content.ClusterControllerConfig; +import com.yahoo.vespa.model.content.ClusterResourceLimits; import com.yahoo.vespa.model.content.ContentSearch; import com.yahoo.vespa.model.content.ContentSearchCluster; import com.yahoo.vespa.model.content.DistributionBitCalculator; @@ -57,8 +55,6 @@ import org.w3c.dom.Element; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; -import java.util.Comparator; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -68,8 +64,6 @@ import java.util.TreeMap; import java.util.logging.Level; import java.util.stream.Collectors; -import static java.util.stream.Collectors.toList; - /** * A content cluster. * @@ -134,11 +128,15 @@ public class ContentCluster extends AbstractConfigProducer implements ContentCluster c = new ContentCluster(context.getParentProducer(), getClusterId(contentElement), documentDefinitions, globallyDistributedDocuments, routingSelection, deployState.zone(), deployState.isHosted()); - c.clusterControllerConfig = new ClusterControllerConfig.Builder(getClusterId(contentElement), contentElement).build(deployState, c, contentElement.getXml()); + boolean enableFeedBlockInDistributor = deployState.getProperties().featureFlags().enableFeedBlockInDistributor(); + var resourceLimits = new ClusterResourceLimits.Builder(enableFeedBlockInDistributor).build(contentElement); + c.clusterControllerConfig = new ClusterControllerConfig.Builder(getClusterId(contentElement), + contentElement, + resourceLimits.getClusterControllerLimits()).build(deployState, c, contentElement.getXml()); c.search = new ContentSearchCluster.Builder(documentDefinitions, - globallyDistributedDocuments, - isCombined(getClusterId(contentElement), containers)) - .build(deployState, c, contentElement.getXml()); + globallyDistributedDocuments, + isCombined(getClusterId(contentElement), containers), + resourceLimits.getContentNodeLimits()).build(deployState, c, contentElement.getXml()); c.persistenceFactory = new EngineFactoryBuilder().build(contentElement, c); c.storageNodes = new StorageCluster.Builder().build(deployState, c, w3cContentElement); c.distributorNodes = new DistributorCluster.Builder(c).build(deployState, c, w3cContentElement); @@ -168,7 +166,7 @@ public class ContentCluster extends AbstractConfigProducer implements if (context.getParentProducer().getRoot() == null) return c; - addClusterControllers(containers, context, c.rootGroup, contentElement, c.clusterId, c); + addClusterControllers(context, c.rootGroup, contentElement, c.clusterId, c); return c; } @@ -281,9 +279,11 @@ public class ContentCluster extends AbstractConfigProducer implements } } - private void addClusterControllers(Collection<ContainerModel> containers, ConfigModelContext context, - StorageGroup rootGroup, ModelElement contentElement, - String contentClusterName, ContentCluster contentCluster) { + private void addClusterControllers(ConfigModelContext context, + StorageGroup rootGroup, + ModelElement contentElement, + String contentClusterName, + ContentCluster contentCluster) { if (admin == null) return; // only in tests if (contentCluster.getPersistence() == null) return; @@ -304,7 +304,7 @@ public class ContentCluster extends AbstractConfigProducer implements .orElse(NodesSpecification.nonDedicated(3, context)); Collection<HostResource> hosts = nodesSpecification.isDedicated() ? getControllerHosts(nodesSpecification, admin, clusterName, context) : - drawControllerHosts(nodesSpecification.minResources().nodes(), rootGroup, containers); + drawControllerHosts(nodesSpecification.minResources().nodes(), rootGroup); clusterControllers = createClusterControllers(new ClusterControllerCluster(contentCluster, "standalone"), hosts, clusterName, @@ -350,9 +350,9 @@ public class ContentCluster extends AbstractConfigProducer implements return nodesSpecification.provision(admin.hostSystem(), ClusterSpec.Type.admin, ClusterSpec.Id.from(clusterName), context.getDeployLogger(), false).keySet(); } - private List<HostResource> drawControllerHosts(int count, StorageGroup rootGroup, Collection<ContainerModel> containers) { - List<HostResource> hosts = drawControllerHosts(count, false, rootGroup, containers); - List<HostResource> retiredHosts = drawControllerHosts(count, true, rootGroup, containers); + private List<HostResource> drawControllerHosts(int count, StorageGroup rootGroup) { + List<HostResource> hosts = drawControllerHosts(count, false, rootGroup); + List<HostResource> retiredHosts = drawControllerHosts(count, true, rootGroup); // preserve the cluster state in case all pre-existing controllers are on retired nodes List<HostResource> all = new ArrayList<>(hosts); @@ -360,78 +360,14 @@ public class ContentCluster extends AbstractConfigProducer implements return all; } - private List<HostResource> drawControllerHosts(int count, boolean retired, StorageGroup rootGroup, Collection<ContainerModel> containers) { + private List<HostResource> drawControllerHosts(int count, boolean retired, StorageGroup rootGroup) { List<HostResource> hosts = drawContentHostsRecursively(count, retired, rootGroup); - // if (hosts.size() < count) // supply with containers TODO: Currently disabled due to leading to topology change problems - // hosts.addAll(drawContainerHosts(count - hosts.size(), containers, new HashSet<>(hosts))); if (hosts.size() % 2 == 0 && ! hosts.isEmpty()) // ZK clusters of even sizes are less available (even in the size=2 case) hosts = hosts.subList(0, hosts.size()-1); return hosts; } /** - * Draws <code>count</code> container nodes to use as cluster controllers, or as many as possible - * if less than <code>count</code> are available. - * - * This will draw the same nodes each time it is - * invoked if cluster names and node indexes are unchanged. - */ - // DO NOT DELETE - see above - private List<HostResource> drawContainerHosts(int count, Collection<ContainerModel> containerClusters, - Set<HostResource> usedHosts) { - if (containerClusters.isEmpty()) return Collections.emptyList(); - - List<HostResource> allHosts = new ArrayList<>(); - for (ApplicationContainerCluster cluster : clustersSortedByName(containerClusters)) - allHosts.addAll(hostResourcesSortedByIndex(cluster)); - - // Don't use hosts already selected to be assigned a cluster controllers as part of building this, - // and don't use hosts which already have one. One physical host may have many roles but can only - // have one cluster controller - List<HostResource> uniqueHostsWithoutClusterController = allHosts.stream() - .filter(h -> ! usedHosts.contains(h)) - .filter(h -> ! hostHasClusterController(h.getHostname(), allHosts)) - .distinct() - .collect(Collectors.toList()); - - return uniqueHostsWithoutClusterController.subList(0, Math.min(uniqueHostsWithoutClusterController.size(), count)); - } - - private List<ApplicationContainerCluster> clustersSortedByName(Collection<ContainerModel> containerModels) { - return containerModels.stream() - .map(ContainerModel::getCluster) - .filter(cluster -> cluster instanceof ApplicationContainerCluster) - .map(cluster -> (ApplicationContainerCluster) cluster) - .sorted(Comparator.comparing(ContainerCluster::getName)) - .collect(Collectors.toList()); - } - - private List<HostResource> hostResourcesSortedByIndex(ApplicationContainerCluster cluster) { - return cluster.getContainers().stream() - .sorted(Comparator.comparing(Container::index)) - .map(Container::getHostResource) - .collect(Collectors.toList()); - } - - /** Returns whether any host having the given hostname has a cluster controller */ - private boolean hostHasClusterController(String hostname, List<HostResource> hosts) { - for (HostResource host : hosts) { - if ( ! host.getHostname().equals(hostname)) continue; - - if (hasClusterController(host)) - return true; - } - return false; - } - - private boolean hasClusterController(HostResource host) { - for (Service service : host.getServices()) - if (service instanceof ClusterControllerContainer) - return true; - return false; - } - - /** * Draw <code>count</code> nodes from as many different content groups below this as possible. * This will only achieve maximum spread in the case where the groups are balanced and never on the same * physical node. It will not achieve maximum spread over all levels in a multilevel group hierarchy. @@ -457,7 +393,7 @@ public class ContentCluster extends AbstractConfigProducer implements return sortedHosts; } - private ClusterControllerContainerCluster createClusterControllers(AbstractConfigProducer parent, + private ClusterControllerContainerCluster createClusterControllers(AbstractConfigProducer<?> parent, Collection<HostResource> hosts, String name, boolean multitenant, @@ -502,7 +438,7 @@ public class ContentCluster extends AbstractConfigProducer implements } - private ContentCluster(AbstractConfigProducer parent, String clusterId, + private ContentCluster(AbstractConfigProducer<?> parent, String clusterId, Map<String, NewDocumentType> documentDefinitions, Set<NewDocumentType> globallyDistributedDocuments, String routingSelection, Zone zone, boolean isHosted) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomResourceLimitsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomResourceLimitsBuilder.java index 8e91f14238e..210f062f9b2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomResourceLimitsBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomResourceLimitsBuilder.java @@ -5,17 +5,17 @@ import com.yahoo.vespa.model.builder.xml.dom.ModelElement; import com.yahoo.vespa.model.content.ResourceLimits; /** - * Builder for resource limits for a content cluster with engine proton. + * Builder for feed block resource limits. * * @author geirst */ public class DomResourceLimitsBuilder { - public static ResourceLimits build(ModelElement contentXml) { + public static ResourceLimits.Builder createBuilder(ModelElement contentXml) { ResourceLimits.Builder builder = new ResourceLimits.Builder(); ModelElement resourceLimits = contentXml.child("resource-limits"); if (resourceLimits == null) { - return builder.build(); + return builder; } if (resourceLimits.child("disk") != null) { builder.setDiskLimit(resourceLimits.childAsDouble("disk")); @@ -23,7 +23,7 @@ public class DomResourceLimitsBuilder { if (resourceLimits.child("memory") != null) { builder.setMemoryLimit(resourceLimits.childAsDouble("memory")); } - return builder.build(); + return builder; } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java index ab5dab4fbb9..57292b32a35 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java @@ -47,7 +47,6 @@ public class FileStorProducer implements StorFilestorConfig.Producer { private final int reponseNumThreads; private final StorFilestorConfig.Response_sequencer_type.Enum responseSequencerType; private final boolean useAsyncMessageHandlingOnSchedule; - private final int mergeChunkSize; private static StorFilestorConfig.Response_sequencer_type.Enum convertResponseSequencerType(String sequencerType) { try { @@ -56,17 +55,13 @@ public class FileStorProducer implements StorFilestorConfig.Producer { return StorFilestorConfig.Response_sequencer_type.Enum.ADAPTIVE; } } - private static int alignUp2MiB(int value) { - final int twoMB = 0x200000; - return ((value + twoMB - 1)/twoMB) * twoMB; - } + public FileStorProducer(ModelContext.FeatureFlags featureFlags, ContentCluster parent, Integer numThreads) { this.numThreads = numThreads; this.cluster = parent; this.reponseNumThreads = featureFlags.defaultNumResponseThreads(); this.responseSequencerType = convertResponseSequencerType(featureFlags.responseSequencerType()); useAsyncMessageHandlingOnSchedule = featureFlags.useAsyncMessageHandlingOnSchedule(); - mergeChunkSize = alignUp2MiB(featureFlags.mergeChunkSize()); // Align up to default huge page size. } @Override @@ -78,7 +73,6 @@ public class FileStorProducer implements StorFilestorConfig.Producer { builder.num_response_threads(reponseNumThreads); builder.response_sequencer_type(responseSequencerType); builder.use_async_message_handling_on_schedule(useAsyncMessageHandlingOnSchedule); - builder.bucket_merge_chunk_size(mergeChunkSize); } } |