diff options
278 files changed, 3689 insertions, 4859 deletions
diff --git a/client/go/internal/admin/jvm/env.go b/client/go/internal/admin/jvm/env.go index 7b1ce97a40a..1cbdb46648f 100644 --- a/client/go/internal/admin/jvm/env.go +++ b/client/go/internal/admin/jvm/env.go @@ -5,10 +5,12 @@ package jvm import ( "fmt" + "strings" "github.com/vespa-engine/vespa/client/go/internal/admin/defaults" "github.com/vespa-engine/vespa/client/go/internal/admin/envvars" "github.com/vespa-engine/vespa/client/go/internal/admin/prog" + "github.com/vespa-engine/vespa/client/go/internal/admin/trace" "github.com/vespa-engine/vespa/client/go/internal/util" ) @@ -29,8 +31,19 @@ func (opts *Options) exportEnvSettings(ps *prog.Spec) { ps.Setenv(envvars.LD_LIBRARY_PATH, dlp) ps.Setenv(envvars.MALLOC_ARENA_MAX, "1") if preload := ps.Getenv(envvars.PRELOAD); preload != "" { - ps.Setenv(envvars.JAVAVM_LD_PRELOAD, preload) - ps.Setenv(envvars.LD_PRELOAD, preload) + checked := []string{} + for _, fileName := range strings.Split(preload, ":") { + if util.PathExists(fileName) { + checked = append(checked, fileName) + } else { + trace.Info("File in PRELOAD missing, skipped:", fileName) + } + } + if len(checked) > 0 { + preload := strings.Join(checked, ":") + ps.Setenv(envvars.JAVAVM_LD_PRELOAD, preload) + ps.Setenv(envvars.LD_PRELOAD, preload) + } } util.OptionallyReduceTimerFrequency() c.exportExtraEnv(ps) diff --git a/client/src/main/java/ai/vespa/client/dsl/NearestNeighbor.java b/client/src/main/java/ai/vespa/client/dsl/NearestNeighbor.java index 1ae7f5cdfde..7dd45153353 100644 --- a/client/src/main/java/ai/vespa/client/dsl/NearestNeighbor.java +++ b/client/src/main/java/ai/vespa/client/dsl/NearestNeighbor.java @@ -14,7 +14,7 @@ public class NearestNeighbor extends QueryChain { this.nonEmpty = true; } - NearestNeighbor annotate(Annotation annotation) { + public NearestNeighbor annotate(Annotation annotation) { this.annotation = annotation; return this; } diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index b55b9322463..1ab3cc30db7 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -108,24 +108,14 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"hmusum"}) default Architecture adminClusterArchitecture() { return Architecture.getDefault(); } @ModelFeatureFlag(owners = {"tokle"}) default boolean enableProxyProtocolMixedMode() { return true; } @ModelFeatureFlag(owners = {"arnej"}) default String logFileCompressionAlgorithm(String defVal) { return defVal; } - @ModelFeatureFlag(owners = {"vekterli"}, removeAfter = "8.110") default boolean useTwoPhaseDocumentGc() { return true; } @ModelFeatureFlag(owners = {"tokle"}) default boolean useRestrictedDataPlaneBindings() { return false; } - @ModelFeatureFlag(owners = {"arnej","baldersheim"}, removeAfter = "8.110") default boolean useOldJdiscContainerStartup() { return false; } - @ModelFeatureFlag(owners = {"tokle, bjorncs"}, removeAfter = "8.108") default boolean enableDataPlaneFilter() { return true; } @ModelFeatureFlag(owners = {"arnej, bjorncs"}) default boolean enableGlobalPhase() { return true; } @ModelFeatureFlag(owners = {"baldersheim"}, comment = "Select summary decode type") default String summaryDecodePolicy() { return "eager"; } @ModelFeatureFlag(owners = {"hmusum"}) default boolean allowMoreThanOneContentGroupDown(ClusterSpec.Id id) { return false; } - @ModelFeatureFlag(owners = {"vekterli", "havardpe"}) default boolean enableConditionalPutRemoveWriteRepair() { return false; } + @ModelFeatureFlag(owners = {"vekterli", "havardpe"}, removeAfter = "8.207") default boolean enableConditionalPutRemoveWriteRepair() { return true; } @ModelFeatureFlag(owners = {"mortent", "olaa"}) default boolean enableDataplaneProxy() { return false; } @ModelFeatureFlag(owners = {"baldersheim"}) default boolean enableNestedMultivalueGrouping() { return false; } @ModelFeatureFlag(owners = {"jonmv"}) default boolean useReconfigurableDispatcher() { return false; } - - // Below are all flags that must be kept until 7 is out of the door and implementations and/or default flag values are in sync with what is defined here. - @ModelFeatureFlag(owners = {"arnej"}, removeAfter="7.last") default boolean ignoreThreadStackSizes() { return false; } - @ModelFeatureFlag(owners = {"baldersheim"}, removeAfter="7.last") default boolean skipCommunicationManagerThread() { return true; } - @ModelFeatureFlag(owners = {"baldersheim"}, removeAfter="7.last") default boolean skipMbusRequestThread() { return true; } - @ModelFeatureFlag(owners = {"baldersheim"}, removeAfter="7.last") default boolean skipMbusReplyThread() { return true; } - @ModelFeatureFlag(owners = {"arnej"}, removeAfter="7.last") default boolean avoidRenamingSummaryFeatures() { return false; } } /** Warning: As elsewhere in this package, do not make backwards incompatible changes that will break old config models! */ diff --git a/config-model/CMakeLists.txt b/config-model/CMakeLists.txt index 0d9a05160d9..39c8e01f478 100644 --- a/config-model/CMakeLists.txt +++ b/config-model/CMakeLists.txt @@ -3,5 +3,3 @@ install_jar(config-model-jar-with-dependencies.jar) install(DIRECTORY src/main/resources/schema DESTINATION share/vespa PATTERN ".gitignore" EXCLUDE PATTERN "version" EXCLUDE) install(DIRECTORY src/main/resources/schema DESTINATION share/vespa/schema/version/8.x PATTERN ".gitignore" EXCLUDE PATTERN "version" EXCLUDE) -# TODO: Remove when Vespa > 8 and no apps are left on 7 in hosted Vespa -install(DIRECTORY src/main/resources/schema/version/7.x/ DESTINATION share/vespa/schema/version/7.x/schema PATTERN ".gitignore" EXCLUDE) diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 66a23c79fbb..b06d3572fcb 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -86,7 +86,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private Optional<CloudAccount> cloudAccount = Optional.empty(); private boolean allowUserFilters = true; private boolean allowMoreThanOneContentGroupDown = false; - private boolean enableConditionalPutRemoveWriteRepair = false; private List<DataplaneToken> dataplaneTokens; private boolean enableDataplaneProxy; @@ -147,7 +146,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public boolean allowUserFilters() { return allowUserFilters; } @Override public boolean enableGlobalPhase() { return true; } // Enable global-phase by default for unit tests only @Override public boolean allowMoreThanOneContentGroupDown(ClusterSpec.Id id) { return allowMoreThanOneContentGroupDown; } - @Override public boolean enableConditionalPutRemoveWriteRepair() { return enableConditionalPutRemoveWriteRepair; } @Override public List<DataplaneToken> dataplaneTokens() { return dataplaneTokens; } @Override public boolean enableDataplaneProxy() { return enableDataplaneProxy; } @@ -383,11 +381,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } - public TestProperties setEnableConditionalPutRemoveWriteRepair(boolean enable) { - this.enableConditionalPutRemoveWriteRepair = enable; - return this; - } - public TestProperties setAllowUserFilters(boolean b) { this.allowUserFilters = b; return this; } public TestProperties setDataplaneTokens(Collection<DataplaneToken> tokens) { diff --git a/config-model/src/main/java/com/yahoo/schema/derived/AttributeFields.java b/config-model/src/main/java/com/yahoo/schema/derived/AttributeFields.java index 12ca67bf2c9..c3531d03d3f 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/AttributeFields.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/AttributeFields.java @@ -51,9 +51,9 @@ public class AttributeFields extends Derived implements AttributesConfig.Produce if (unsupportedFieldType(field)) { return; // Ignore complex struct and map fields for indexed search (only supported for streaming search) } - if (isArrayOfSimpleStruct(field, false)) { + if (isArrayOfSimpleStruct(field)) { deriveArrayOfSimpleStruct(field); - } else if (isMapOfSimpleStruct(field, false)) { + } else if (isMapOfSimpleStruct(field)) { deriveMapOfSimpleStruct(field); } else if (isMapOfPrimitiveType(field)) { deriveMapOfPrimitiveType(field); diff --git a/config-model/src/main/java/com/yahoo/schema/derived/ImportedFields.java b/config-model/src/main/java/com/yahoo/schema/derived/ImportedFields.java index 122048d02b9..fa3f49f06d5 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/ImportedFields.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/ImportedFields.java @@ -61,9 +61,9 @@ public class ImportedFields extends Derived implements ImportedFieldsConfig.Prod ImmutableSDField targetField = field.targetField(); if (GeoPos.isAnyPos(targetField)) { // no action needed - } else if (isArrayOfSimpleStruct(targetField, false)) { + } else if (isArrayOfSimpleStruct(targetField)) { considerNestedFields(builder, field); - } else if (isMapOfSimpleStruct(targetField, false)) { + } else if (isMapOfSimpleStruct(targetField)) { considerSimpleField(builder, field.getNestedField("key")); considerNestedFields(builder, field.getNestedField("value")); } else if (isMapOfPrimitiveType(targetField)) { diff --git a/config-model/src/main/java/com/yahoo/schema/document/ComplexAttributeFieldUtils.java b/config-model/src/main/java/com/yahoo/schema/document/ComplexAttributeFieldUtils.java index 5e4ee6d4b27..ebafd8f1d24 100644 --- a/config-model/src/main/java/com/yahoo/schema/document/ComplexAttributeFieldUtils.java +++ b/config-model/src/main/java/com/yahoo/schema/document/ComplexAttributeFieldUtils.java @@ -22,31 +22,26 @@ import com.yahoo.document.StructDataType; public class ComplexAttributeFieldUtils { public static boolean isSupportedComplexField(ImmutableSDField field) { - return isSupportedComplexField(field, false); - } - - // TODO: Remove the stricterValidation flag when this is changed to being always on. - public static boolean isSupportedComplexField(ImmutableSDField field, boolean stricterValidation) { - return (isArrayOfSimpleStruct(field, stricterValidation) || - isMapOfSimpleStruct(field, stricterValidation) || + return (isArrayOfSimpleStruct(field) || + isMapOfSimpleStruct(field) || isMapOfPrimitiveType(field)); } - public static boolean isArrayOfSimpleStruct(ImmutableSDField field, boolean stricterValidation) { + public static boolean isArrayOfSimpleStruct(ImmutableSDField field) { if (field.getDataType() instanceof ArrayDataType) { ArrayDataType arrayType = (ArrayDataType)field.getDataType(); - return isStructWithPrimitiveStructFieldAttributes(arrayType.getNestedType(), field, stricterValidation); + return isStructWithPrimitiveStructFieldAttributes(arrayType.getNestedType(), field); } else { return false; } } - public static boolean isMapOfSimpleStruct(ImmutableSDField field, boolean stricterValidation) { + public static boolean isMapOfSimpleStruct(ImmutableSDField field) { if (field.getDataType() instanceof MapDataType) { MapDataType mapType = (MapDataType)field.getDataType(); return isPrimitiveType(mapType.getKeyType()) && isStructWithPrimitiveStructFieldAttributes(mapType.getValueType(), - field.getStructField("value"), stricterValidation); + field.getStructField("value")); } else { return false; } @@ -62,7 +57,7 @@ public class ComplexAttributeFieldUtils { } } - private static boolean isStructWithPrimitiveStructFieldAttributes(DataType type, ImmutableSDField field, boolean stricterValidation) { + private static boolean isStructWithPrimitiveStructFieldAttributes(DataType type, ImmutableSDField field) { if (type instanceof StructDataType && ! GeoPos.isPos(type)) { for (ImmutableSDField structField : field.getStructFields()) { Attribute attribute = structField.getAttributes().get(structField.getName()); @@ -75,7 +70,7 @@ public class ComplexAttributeFieldUtils { return false; } } - if (stricterValidation && !structField.isImportedField() && hasStructFieldAttributes(structField)) { + if (!structField.isImportedField() && hasStructFieldAttributes(structField)) { return false; } } @@ -113,9 +108,9 @@ public class ComplexAttributeFieldUtils { } public static boolean isComplexFieldWithOnlyStructFieldAttributes(ImmutableSDField field) { - if (isArrayOfSimpleStruct(field, false)) { + if (isArrayOfSimpleStruct(field)) { return hasOnlyStructFieldAttributes(field); - } else if (isMapOfSimpleStruct(field, false)) { + } else if (isMapOfSimpleStruct(field)) { return (field.getStructField("key").hasSingleAttribute()) && hasOnlyStructFieldAttributes(field.getStructField("value")); } else if (isMapOfPrimitiveType(field)) { diff --git a/config-model/src/main/java/com/yahoo/schema/processing/ImportedFieldsResolver.java b/config-model/src/main/java/com/yahoo/schema/processing/ImportedFieldsResolver.java index 8e44bd026a3..ee465be44f2 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/ImportedFieldsResolver.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/ImportedFieldsResolver.java @@ -52,9 +52,9 @@ public class ImportedFieldsResolver extends Processor { ImmutableSDField targetField = getTargetField(importedField, reference); if (GeoPos.isAnyPos(targetField)) { resolveImportedPositionField(importedField, reference, targetField, validate); - } else if (isArrayOfSimpleStruct(targetField, false)) { + } else if (isArrayOfSimpleStruct(targetField)) { resolveImportedArrayOfStructField(importedField, reference, targetField, validate); - } else if (isMapOfSimpleStruct(targetField, false)) { + } else if (isMapOfSimpleStruct(targetField)) { resolveImportedMapOfStructField(importedField, reference, targetField, validate); } else if (isMapOfPrimitiveType(targetField)) { resolveImportedMapOfPrimitiveField(importedField, reference, targetField, validate); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java index 62cbf61b336..f8837a4530a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin; +import ai.vespa.metrics.set.MetricSet; import com.yahoo.cloud.config.SlobroksConfig; import com.yahoo.cloud.config.ZookeepersConfig; import com.yahoo.cloud.config.log.LogdConfig; @@ -20,7 +21,6 @@ import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerContainer; import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerContainerCluster; import com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainer; import com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster; -import com.yahoo.vespa.model.admin.monitoring.MetricSet; import com.yahoo.vespa.model.admin.monitoring.Monitoring; import com.yahoo.vespa.model.admin.monitoring.builder.Metrics; import com.yahoo.vespa.model.filedistribution.FileDistributionConfigProducer; @@ -31,7 +31,8 @@ import java.util.Collections; import java.util.List; import java.util.Optional; -import static com.yahoo.vespa.model.admin.monitoring.MetricSet.empty; +import static ai.vespa.metrics.set.MetricSet.empty; + /** * This is the admin pseudo-plugin of the Vespa model, responsible for diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/ConsumersConfigGenerator.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/ConsumersConfigGenerator.java index 97a6f27b759..ea3d5e55b07 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/ConsumersConfigGenerator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/ConsumersConfigGenerator.java @@ -2,17 +2,16 @@ package com.yahoo.vespa.model.admin.metricsproxy; +import ai.vespa.metrics.set.Metric; +import ai.vespa.metrics.set.MetricSet; import ai.vespa.metricsproxy.core.ConsumersConfig.Consumer; import com.yahoo.config.provision.SystemName; -import com.yahoo.vespa.model.admin.monitoring.Metric; -import com.yahoo.vespa.model.admin.monitoring.MetricSet; import com.yahoo.vespa.model.admin.monitoring.MetricsConsumer; import java.util.Comparator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; /** * Helper class to generate config for metrics consumers. diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java index 58e624960f9..df6fee1deb7 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.model.admin.metricsproxy; +import ai.vespa.metrics.set.MetricSet; import ai.vespa.metricsproxy.core.ConsumersConfig; import ai.vespa.metricsproxy.core.MetricsConsumers; import ai.vespa.metricsproxy.core.MetricsManager; @@ -27,7 +28,6 @@ import com.yahoo.config.provision.Zone; import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.vespa.model.admin.Admin; -import com.yahoo.vespa.model.admin.monitoring.MetricSet; import com.yahoo.vespa.model.admin.monitoring.MetricsConsumer; import com.yahoo.vespa.model.admin.monitoring.Monitoring; import com.yahoo.vespa.model.container.ContainerCluster; @@ -45,6 +45,7 @@ import java.util.logging.Logger; import java.util.stream.Collectors; import java.util.stream.Stream; +import static ai.vespa.metrics.set.MetricSet.empty; import static com.yahoo.vespa.model.admin.metricsproxy.ConsumersConfigGenerator.addMetrics; import static com.yahoo.vespa.model.admin.metricsproxy.ConsumersConfigGenerator.generateConsumers; import static com.yahoo.vespa.model.admin.metricsproxy.ConsumersConfigGenerator.toConsumerBuilder; @@ -53,7 +54,6 @@ import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerClus import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.AppDimensionNames.LEGACY_APPLICATION; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.AppDimensionNames.SYSTEM; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.AppDimensionNames.TENANT; -import static com.yahoo.vespa.model.admin.monitoring.MetricSet.empty; /** * Container cluster for metrics proxy containers. diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricsConsumer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricsConsumer.java index c6bd4c6e295..e18156876cd 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricsConsumer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricsConsumer.java @@ -1,21 +1,21 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin.monitoring; +import ai.vespa.metrics.set.Metric; +import ai.vespa.metrics.set.MetricSet; import ai.vespa.metricsproxy.core.VespaMetrics; import ai.vespa.metricsproxy.http.ValuesFetcher; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Objects; -import static com.yahoo.vespa.model.admin.monitoring.AutoscalingMetrics.autoscalingMetricSet; -import static com.yahoo.vespa.model.admin.monitoring.DefaultMetrics.defaultMetricSet; -import static com.yahoo.vespa.model.admin.monitoring.NetworkMetrics.networkMetricSet; -import static com.yahoo.vespa.model.admin.monitoring.SystemMetrics.systemMetricSet; -import static com.yahoo.vespa.model.admin.monitoring.VespaMetricSet.vespaMetricSet; -import static java.util.Collections.unmodifiableList; +import static ai.vespa.metrics.set.AutoscalingMetrics.autoscalingMetricSet; +import static ai.vespa.metrics.set.DefaultMetrics.defaultMetricSet; +import static ai.vespa.metrics.set.NetworkMetrics.networkMetricSet; +import static ai.vespa.metrics.set.SystemMetrics.systemMetricSet; +import static ai.vespa.metrics.set.VespaMetricSet.vespaMetricSet; /** * A metric consumer is a set of metrics given an id that can be requested at runtime. diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/PredefinedMetricSets.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/PredefinedMetricSets.java index d0a5b1bbe43..3af2b4f8732 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/PredefinedMetricSets.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/PredefinedMetricSets.java @@ -1,19 +1,19 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin.monitoring.builder; -import com.yahoo.vespa.model.admin.monitoring.MetricSet; +import ai.vespa.metrics.set.MetricSet; import java.util.Collections; import java.util.LinkedHashMap; import java.util.Map; -import static com.yahoo.vespa.model.admin.monitoring.AutoscalingMetrics.autoscalingMetricSet; -import static com.yahoo.vespa.model.admin.monitoring.DefaultMetrics.defaultMetricSet; -import static com.yahoo.vespa.model.admin.monitoring.NetworkMetrics.networkMetricSet; -import static com.yahoo.vespa.model.admin.monitoring.SystemMetrics.systemMetricSet; -import static com.yahoo.vespa.model.admin.monitoring.DefaultVespaMetrics.defaultVespaMetricSet; -import static com.yahoo.vespa.model.admin.monitoring.VespaMetricSet.vespaMetricSet; -import static com.yahoo.vespa.model.admin.monitoring.InfrastructureMetricSet.infrastructureMetricSet; +import static ai.vespa.metrics.set.AutoscalingMetrics.autoscalingMetricSet; +import static ai.vespa.metrics.set.DefaultMetrics.defaultMetricSet; +import static ai.vespa.metrics.set.DefaultVespaMetrics.defaultVespaMetricSet; +import static ai.vespa.metrics.set.InfrastructureMetricSet.infrastructureMetricSet; +import static ai.vespa.metrics.set.NetworkMetrics.networkMetricSet; +import static ai.vespa.metrics.set.SystemMetrics.systemMetricSet; +import static ai.vespa.metrics.set.VespaMetricSet.vespaMetricSet; /** * A data object for predefined metric sets. diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/MetricsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/MetricsBuilder.java index f75b2a864f9..a00fe47eb21 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/MetricsBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/MetricsBuilder.java @@ -1,10 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin.monitoring.builder.xml; +import ai.vespa.metrics.set.Metric; +import ai.vespa.metrics.set.MetricSet; import com.yahoo.config.model.ConfigModelContext.ApplicationType; import com.yahoo.text.XML; -import com.yahoo.vespa.model.admin.monitoring.Metric; -import com.yahoo.vespa.model.admin.monitoring.MetricSet; import com.yahoo.vespa.model.admin.monitoring.MetricsConsumer; import com.yahoo.vespa.model.admin.monitoring.builder.Metrics; import org.w3c.dom.Element; @@ -14,8 +14,9 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; -import static com.yahoo.vespa.model.admin.monitoring.DefaultVespaMetrics.defaultVespaMetricSet; -import static com.yahoo.vespa.model.admin.monitoring.SystemMetrics.systemMetricSet; +import static ai.vespa.metrics.set.DefaultVespaMetrics.defaultVespaMetricSet; +import static ai.vespa.metrics.set.SystemMetrics.systemMetricSet; + /** * @author gjoranv diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexFieldsWithStructFieldAttributesValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexFieldsWithStructFieldAttributesValidator.java index d2999a24775..1d67b0c023f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexFieldsWithStructFieldAttributesValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexFieldsWithStructFieldAttributesValidator.java @@ -39,19 +39,15 @@ public class ComplexFieldsWithStructFieldAttributesValidator extends Validator { } private static void validateComplexFields(String clusterName, Schema schema, DeployLogger logger) { - String unsupportedFields = validateComplexFields(clusterName, schema, false); + String unsupportedFields = validateComplexFields(schema); if (!unsupportedFields.isEmpty()) { throw new IllegalArgumentException(getErrorMessage(clusterName, schema, unsupportedFields)); } - unsupportedFields = validateComplexFields(clusterName, schema, true); - if (!unsupportedFields.isEmpty()) { - logger.logApplicationPackage(Level.WARNING, getErrorMessage(clusterName, schema, unsupportedFields)); - } } - private static String validateComplexFields(String clusterName, Schema schema, boolean stricterValidation) { + private static String validateComplexFields(Schema schema) { return schema.allFields() - .filter(field -> isUnsupportedComplexField(field, stricterValidation)) + .filter(field -> isUnsupportedComplexField(field)) .map(ComplexFieldsWithStructFieldAttributesValidator::toString) .collect(Collectors.joining(", ")); } @@ -63,14 +59,14 @@ public class ComplexFieldsWithStructFieldAttributesValidator extends Validator { clusterName, schema.getName(), unsupportedFields); } - private static boolean isUnsupportedComplexField(ImmutableSDField field, boolean stricterValidation) { + private static boolean isUnsupportedComplexField(ImmutableSDField field) { return (field.usesStructOrMap() && - !isSupportedComplexField(field, stricterValidation) && + !isSupportedComplexField(field) && hasStructFieldAttributes(field.getStructFields())); } - private static boolean isSupportedComplexField(ImmutableSDField field, boolean stricterValidation) { - return (ComplexAttributeFieldUtils.isSupportedComplexField(field, stricterValidation) || + private static boolean isSupportedComplexField(ImmutableSDField field) { + return (ComplexAttributeFieldUtils.isSupportedComplexField(field) || GeoPos.isAnyPos(field)); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java index f7d4fe28c6e..9d1b27d4bfe 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java @@ -377,8 +377,7 @@ public abstract class Container extends AbstractService implements @Override public void getConfig(ContainerHttpConfig.Builder builder) { - if (hostResponseHeaderKey.isPresent()) - builder.hostResponseHeaderKey(hostResponseHeaderKey.get()); + hostResponseHeaderKey.ifPresent(builder::hostResponseHeaderKey); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index bf8c96db614..8dc58a33e04 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -35,7 +35,6 @@ import java.util.Optional; import java.util.Objects; import java.util.TreeMap; import java.util.function.Predicate; -import java.util.stream.Collectors; /** * Encapsulates the various options for search in a content model. @@ -44,7 +43,8 @@ import java.util.stream.Collectors; public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> implements ProtonConfig.Producer, DispatchNodesConfig.Producer, - DispatchConfig.Producer + DispatchConfig.Producer, + Redundancy.Provider { private static final int DEFAULT_DOC_STORE_COMPRESSION_LEVEL = 3; @@ -303,7 +303,7 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> if (element == null) { searchNode = SearchNode.create(parent, "" + node.getDistributionKey(), node.getDistributionKey(), spec, clusterName, node, flushOnShutdown, tuning, resourceLimits, deployState.isHosted(), - fractionOfMemoryReserved, redundancy, deployState.featureFlags()); + fractionOfMemoryReserved, this, deployState.featureFlags()); searchNode.setHostResource(node.getHostResource()); searchNode.initService(deployState); @@ -312,7 +312,7 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> tls.initService(deployState); } else { searchNode = new SearchNode.Builder(""+node.getDistributionKey(), spec, clusterName, node, flushOnShutdown, - tuning, resourceLimits, fractionOfMemoryReserved, redundancy) + tuning, resourceLimits, fractionOfMemoryReserved, this) .build(deployState, parent, element.getXml()); tls = new TransactionLogServer.Builder(clusterName, syncTransactionLog).build(deployState, searchNode, element.getXml()); } @@ -352,7 +352,6 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> if (hasIndexedCluster()) { // Important: these must all be the normalized "within a single leaf group" values, // _not_ the cluster-wide, cross-group values. - indexedCluster.setSearchableCopies(redundancy.readyCopies()); indexedCluster.setRedundancy(redundancy.finalRedundancy()); } this.redundancy = redundancy; @@ -495,4 +494,6 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> @Override public String toString() { return "content cluster '" + clusterName + "'"; } + public Redundancy redundancy() { return redundancy; } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java index 6f0a03bab60..8503eafa713 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java @@ -34,7 +34,6 @@ public class DistributorCluster extends TreeConfigProducer<Distributor> implemen private final GcOptions gc; private final boolean hasIndexedDocumentType; private final int maxActivationInhibitedOutOfSyncGroups; - private final boolean enableConditionalPutRemoveWriteRepair; public static class Builder extends VespaDomBuilder.DomConfigProducerBuilderBase<DistributorCluster> { ContentCluster parent; @@ -95,20 +94,17 @@ public class DistributorCluster extends TreeConfigProducer<Distributor> implemen final GcOptions gc = parseGcOptions(documentsNode); final boolean hasIndexedDocumentType = clusterContainsIndexedDocumentType(documentsNode); int maxInhibitedGroups = deployState.getProperties().featureFlags().maxActivationInhibitedOutOfSyncGroups(); - boolean enableConditionalPutRemoveWriteRepair = deployState.getProperties().featureFlags().enableConditionalPutRemoveWriteRepair(); return new DistributorCluster(parent, new BucketSplitting.Builder().build(new ModelElement(producerSpec)), gc, hasIndexedDocumentType, - maxInhibitedGroups, - enableConditionalPutRemoveWriteRepair); + maxInhibitedGroups); } } private DistributorCluster(ContentCluster parent, BucketSplitting bucketSplitting, GcOptions gc, boolean hasIndexedDocumentType, - int maxActivationInhibitedOutOfSyncGroups, - boolean enableConditionalPutRemoveWriteRepair) + int maxActivationInhibitedOutOfSyncGroups) { super(parent, "distributor"); this.parent = parent; @@ -116,7 +112,6 @@ public class DistributorCluster extends TreeConfigProducer<Distributor> implemen this.gc = gc; this.hasIndexedDocumentType = hasIndexedDocumentType; this.maxActivationInhibitedOutOfSyncGroups = maxActivationInhibitedOutOfSyncGroups; - this.enableConditionalPutRemoveWriteRepair = enableConditionalPutRemoveWriteRepair; } @Override @@ -129,7 +124,7 @@ public class DistributorCluster extends TreeConfigProducer<Distributor> implemen builder.enable_revert(parent.getPersistence().supportRevert()); builder.disable_bucket_activation(!hasIndexedDocumentType); builder.max_activation_inhibited_out_of_sync_groups(maxActivationInhibitedOutOfSyncGroups); - builder.enable_condition_probing(enableConditionalPutRemoveWriteRepair); + builder.enable_condition_probing(true); bucketSplitting.getConfig(builder); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/Redundancy.java b/config-model/src/main/java/com/yahoo/vespa/model/content/Redundancy.java index 616dcbbc760..e883e87d36c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/Redundancy.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/Redundancy.java @@ -64,4 +64,8 @@ public class Redundancy implements StorDistributionConfig.Producer, ProtonConfig builder.distribution(distBuilder); } + public interface Provider { + Redundancy redundancy(); + } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java index 6078215f9b6..74c1b7400c1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.content; -import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.ConfigModelContext; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.provision.ClusterMembership; @@ -132,8 +131,7 @@ public class StorageGroup { public void getConfig(StorDistributionConfig.Group.Builder builder) { builder.index(index == null ? "invalid" : index); builder.name(name == null ? "invalid" : name); - if (partitions.isPresent()) - builder.partitions(partitions.get()); + partitions.ifPresent(builder::partitions); for (StorageNode node : nodes) { StorDistributionConfig.Group.Nodes.Builder nb = new StorDistributionConfig.Group.Nodes.Builder(); nb.index(node.getDistributionKey()); @@ -384,11 +382,7 @@ public class StorageGroup { Map<Optional<ClusterSpec.Group>, Map<HostResource, ClusterMembership>> hostsPerGroup = new LinkedHashMap<>(); for (Map.Entry<HostResource, ClusterMembership> entry : hostMapping.entrySet()) { Optional<ClusterSpec.Group> group = entry.getValue().cluster().group(); - Map<HostResource, ClusterMembership> hostsInGroup = hostsPerGroup.get(group); - if (hostsInGroup == null) { - hostsInGroup = new LinkedHashMap<>(); - hostsPerGroup.put(group, hostsInGroup); - } + Map<HostResource, ClusterMembership> hostsInGroup = hostsPerGroup.computeIfAbsent(group, k -> new LinkedHashMap<>()); hostsInGroup.put(entry.getKey(), entry.getValue()); } return hostsPerGroup; @@ -396,23 +390,15 @@ public class StorageGroup { } - private static class XmlNodeBuilder { - - private final ModelElement clusterElement; - private final ModelElement element; - - private XmlNodeBuilder(ModelElement clusterElement, ModelElement element) { - this.clusterElement = clusterElement; - this.element = element; - } + private record XmlNodeBuilder(ModelElement clusterElement, ModelElement element) { public StorageNode build(DeployState deployState, ContentCluster parent, StorageGroup storageGroup) { - StorageNode sNode = new StorageNode.Builder().build(deployState, parent.getStorageCluster(), element.getXml()); - PersistenceEngine provider = parent.getPersistence().create(deployState, sNode, storageGroup, element); - new Distributor.Builder(clusterElement, provider).build(deployState, parent.getDistributorNodes(), element.getXml()); - return sNode; - } - } + StorageNode sNode = new StorageNode.Builder().build(deployState, parent.getStorageCluster(), element.getXml()); + PersistenceEngine provider = parent.getPersistence().create(deployState, sNode, storageGroup, element); + new Distributor.Builder(clusterElement, provider).build(deployState, parent.getDistributorNodes(), element.getXml()); + return sNode; + } + } /** * Creates a content group builder from a group and/or nodes element. @@ -441,14 +427,13 @@ public class StorageGroup { childAsString(groupElement, VespaDomBuilder.VESPAMALLOC_DEBUG), childAsString(groupElement, VespaDomBuilder.VESPAMALLOC_DEBUG_STACKTRACE)); - List<GroupBuilder> subGroups = groupElement.isPresent() ? collectSubGroups(isHosted, group, groupElement.get()) - : List.of(); + List<GroupBuilder> subGroups = groupElement.map(modelElement -> collectSubGroups(isHosted, group, modelElement)).orElseGet(List::of); List<XmlNodeBuilder> explicitNodes = new ArrayList<>(); explicitNodes.addAll(collectExplicitNodes(groupElement)); explicitNodes.addAll(collectExplicitNodes(nodesElement)); - if (subGroups.size() > 0 && nodesElement.isPresent()) + if (!subGroups.isEmpty() && nodesElement.isPresent()) throw new IllegalArgumentException("A group can contain either explicit subgroups or a nodes specification, but not both."); Optional<NodesSpecification> nodeRequirement; @@ -474,12 +459,10 @@ public class StorageGroup { return Optional.ofNullable(element.get().childAsString(childTagName)); } private Optional<Long> childAsLong(Optional<ModelElement> element, String childTagName) { - if (element.isEmpty()) return Optional.empty(); - return Optional.ofNullable(element.get().childAsLong(childTagName)); + return element.map(modelElement -> modelElement.childAsLong(childTagName)); } private Optional<Boolean> childAsBoolean(Optional<ModelElement> element, String childTagName) { - if (element.isEmpty()) return Optional.empty(); - return Optional.ofNullable(element.get().childAsBoolean(childTagName)); + return element.map(modelElement -> modelElement.childAsBoolean(childTagName)); } private boolean booleanAttributeOr(Optional<ModelElement> element, String attributeName, boolean defaultValue) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/EngineFactoryBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/EngineFactoryBuilder.java index 65e329ab622..8600b40d5a6 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/EngineFactoryBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/EngineFactoryBuilder.java @@ -2,7 +2,9 @@ package com.yahoo.vespa.model.content.cluster; import com.yahoo.vespa.model.builder.xml.dom.ModelElement; -import com.yahoo.vespa.model.content.engines.*; +import com.yahoo.vespa.model.content.engines.DummyPersistence; +import com.yahoo.vespa.model.content.engines.PersistenceEngine; +import com.yahoo.vespa.model.content.engines.ProtonEngine; /** * Creates the correct engine factory from XML. @@ -18,7 +20,7 @@ public class EngineFactoryBuilder { if (persistence.child("proton") != null) { return new ProtonEngine.Factory(c.getSearch()); } else if (persistence.child("dummy") != null) { - return new com.yahoo.vespa.model.content.engines.DummyPersistence.Factory(); + return new DummyPersistence.Factory(); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/DispatchGroup.java b/config-model/src/main/java/com/yahoo/vespa/model/search/DispatchGroup.java deleted file mode 100644 index 643a305f369..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/DispatchGroup.java +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.search; - -import java.util.Map; -import java.util.TreeMap; - -/** - * Class representing a group of @link{SearchInterface} nodes and a set of @link{Dispatch} nodes. - * - * Each @link{Dispatch} has a reference to an instance of this class and use it when producing config. - * - * @author baldersheim - */ -public class DispatchGroup { - - private final Map<Integer, Map<Integer, SearchInterface>> searchers = new TreeMap<>(); - - final private IndexedSearchCluster sc; - - public DispatchGroup(IndexedSearchCluster sc) { - this.sc = sc; - } - - DispatchGroup addSearcher(SearchInterface search) { - Map<Integer, SearchInterface> rows = searchers.get(search.getNodeSpec().partitionId()); - if (rows == null) { - rows = new TreeMap<>(); - rows.put(search.getNodeSpec().groupIndex(), search); - searchers.put(search.getNodeSpec().partitionId(), rows); - } else { - if (rows.containsKey(search.getNodeSpec().groupIndex())) { - throw new IllegalArgumentException("Already contains a search node with row id '" + search.getNodeSpec().groupIndex() + "'"); - } - rows.put(search.getNodeSpec().groupIndex(), search); - } - return this; - } - - public Iterable getSearchersIterable() { - return new Iterable(searchers); - } - - public int getRowBits() { - return sc.getRowBits(); - } - - public int getNumPartitions() { - return searchers.size(); - } - - public boolean useFixedRowInDispatch() { - return sc.useFixedRowInDispatch(); - } - - public int getSearchableCopies() { return sc.getSearchableCopies(); } - - public int getRedundancy() { return sc.getRedundancy(); } - - static class Iterator implements java.util.Iterator<SearchInterface> { - - private java.util.Iterator<Map<Integer, SearchInterface>> it1; - private java.util.Iterator<SearchInterface> it2; - - Iterator(Map<Integer, Map<Integer, SearchInterface> > s) { - it1 = s.values().iterator(); - if (it1.hasNext()) { - it2 = it1.next().values().iterator(); - } - } - - @Override - public boolean hasNext() { - if (it2 == null) { - return false; - } - while (!it2.hasNext() && it1.hasNext()) { - it2 = it1.next().values().iterator(); - } - return it2.hasNext(); - } - - @Override - public SearchInterface next() { - return it2.next(); - } - - @Override - public void remove() { - throw new IllegalStateException("'remove' not implemented"); - } - } - - public static class Iterable implements java.lang.Iterable<SearchInterface> { - final Map<Integer, Map<Integer, SearchInterface> > searchers; - Iterable(Map<Integer, Map<Integer, SearchInterface> > searchers) { this.searchers = searchers; } - @Override - public java.util.Iterator<SearchInterface> iterator() { - return new Iterator(searchers); - } - } - -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java index a71e72c0ef8..65a667feb4f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java @@ -50,10 +50,8 @@ public class IndexedSearchCluster extends SearchCluster private final List<DocumentDatabase> documentDbs = new LinkedList<>(); private final MultipleDocumentDatabasesConfigProducer documentDbsConfigProducer; - private int searchableCopies = 1; private int redundancy = 1; - private final DispatchGroup rootDispatch; private final List<SearchNode> searchNodes = new ArrayList<>(); private final DispatchTuning.DispatchPolicy defaultDispatchPolicy; private final double dispatchWarmup; @@ -72,7 +70,6 @@ public class IndexedSearchCluster extends SearchCluster super(parent, clusterName, index); indexingDocproc = new IndexingDocproc(); documentDbsConfigProducer = new MultipleDocumentDatabasesConfigProducer(this, documentDbs); - rootDispatch = new DispatchGroup(this); defaultDispatchPolicy = DispatchTuning.Builder.toDispatchPolicy(featureFlags.queryDispatchPolicy()); dispatchWarmup = featureFlags.queryDispatchWarmup(); summaryDecodePolicy = featureFlags.summaryDecodePolicy(); @@ -83,11 +80,9 @@ public class IndexedSearchCluster extends SearchCluster public IndexingDocproc getIndexingDocproc() { return indexingDocproc; } - public DispatchGroup getRootDispatch() { return rootDispatch; } public void addSearcher(SearchNode searcher) { searchNodes.add(searcher); - rootDispatch.addSearcher(searcher); } public List<SearchNode> getSearchNodes() { return Collections.unmodifiableList(searchNodes); } @@ -198,27 +193,6 @@ public class IndexedSearchCluster extends SearchCluster documentDbsConfigProducer.getConfig(builder); } - boolean useFixedRowInDispatch() { - for (SearchNode node : getSearchNodes()) { - if (node.getNodeSpec().groupIndex() > 0) { - return true; - } - } - return false; - } - - public int getSearchableCopies() { - return searchableCopies; - } - - public void setSearchableCopies(int searchableCopies) { - this.searchableCopies = searchableCopies; - } - - public int getRedundancy() { - return redundancy; - } - public void setRedundancy(int redundancy) { this.redundancy = redundancy; } @@ -258,7 +232,7 @@ public class IndexedSearchCluster extends SearchCluster if (tuning.dispatch.getMaxHitsPerPartition() != null) builder.maxHitsPerNode(tuning.dispatch.getMaxHitsPerPartition()); - builder.redundancy(rootDispatch.getRedundancy()); + builder.redundancy(redundancy); if (searchCoverage != null) { if (searchCoverage.getMinimum() != null) builder.minSearchCoverage(searchCoverage.getMinimum() * 100.0); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java index f238d30308b..53da4d31488 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java @@ -70,7 +70,7 @@ public class SearchNode extends AbstractService implements private final Optional<Tuning> tuning; private final Optional<ResourceLimits> resourceLimits; private final double fractionOfMemoryReserved; - private final Redundancy redundancy; + private final Redundancy.Provider redundancyProvider; public static class Builder extends VespaDomBuilder.DomConfigProducerBuilderBase<SearchNode> { @@ -82,11 +82,11 @@ public class SearchNode extends AbstractService implements private final Optional<Tuning> tuning; private final Optional<ResourceLimits> resourceLimits; private final double fractionOfMemoryReserved; - private final Redundancy redundancy; + private final Redundancy.Provider redundancyProvider; public Builder(String name, NodeSpec nodeSpec, String clusterName, ContentNode node, boolean flushOnShutdown, Optional<Tuning> tuning, Optional<ResourceLimits> resourceLimits, - double fractionOfMemoryReserved, Redundancy redundancy) { + double fractionOfMemoryReserved, Redundancy.Provider redundancyprovider) { this.name = name; this.nodeSpec = nodeSpec; this.clusterName = clusterName; @@ -95,7 +95,7 @@ public class SearchNode extends AbstractService implements this.tuning = tuning; this.resourceLimits = resourceLimits; this.fractionOfMemoryReserved = fractionOfMemoryReserved; - this.redundancy = redundancy; + this.redundancyProvider = redundancyprovider; } @Override @@ -103,7 +103,7 @@ public class SearchNode extends AbstractService implements Element producerSpec) { return SearchNode.create(ancestor, name, contentNode.getDistributionKey(), nodeSpec, clusterName, contentNode, flushOnShutdown, tuning, resourceLimits, deployState.isHosted(), - fractionOfMemoryReserved, redundancy, deployState.featureFlags()); + fractionOfMemoryReserved, redundancyProvider, deployState.featureFlags()); } } @@ -111,10 +111,10 @@ public class SearchNode extends AbstractService implements public static SearchNode create(TreeConfigProducer<?> parent, String name, int distributionKey, NodeSpec nodeSpec, String clusterName, AbstractService serviceLayerService, boolean flushOnShutdown, Optional<Tuning> tuning, Optional<ResourceLimits> resourceLimits, - boolean isHostedVespa, double fractionOfMemoryReserved, Redundancy redundancy, + boolean isHostedVespa, double fractionOfMemoryReserved, Redundancy.Provider redundancyProvider, ModelContext.FeatureFlags featureFlags) { SearchNode node = new SearchNode(parent, name, distributionKey, nodeSpec, clusterName, serviceLayerService, flushOnShutdown, - tuning, resourceLimits, isHostedVespa, fractionOfMemoryReserved, redundancy); + tuning, resourceLimits, isHostedVespa, fractionOfMemoryReserved, redundancyProvider); if (featureFlags.loadCodeAsHugePages()) { node.addEnvironmentVariable("VESPA_LOAD_CODE_AS_HUGEPAGES", true); } @@ -127,7 +127,7 @@ public class SearchNode extends AbstractService implements private SearchNode(TreeConfigProducer<?> parent, String name, int distributionKey, NodeSpec nodeSpec, String clusterName, AbstractService serviceLayerService, boolean flushOnShutdown, Optional<Tuning> tuning, Optional<ResourceLimits> resourceLimits, boolean isHostedVespa, - double fractionOfMemoryReserved, Redundancy redundancy) { + double fractionOfMemoryReserved, Redundancy.Provider redundancyProvider) { super(parent, name); this.distributionKey = distributionKey; this.serviceLayerService = serviceLayerService; @@ -144,7 +144,7 @@ public class SearchNode extends AbstractService implements // Properties are set in DomSearchBuilder this.tuning = tuning; this.resourceLimits = resourceLimits; - this.redundancy = redundancy; + this.redundancyProvider = redundancyProvider; setPropertiesElastic(clusterName, distributionKey); addEnvironmentVariable("OMP_NUM_THREADS", 1); } @@ -286,7 +286,7 @@ public class SearchNode extends AbstractService implements if (nodeResources.isPresent()) { var nodeResourcesTuning = new NodeResourcesTuning(nodeResources.get(), tuning.map(Tuning::threadsPerSearch).orElse(1), - fractionOfMemoryReserved, redundancy); + fractionOfMemoryReserved, redundancyProvider.redundancy()); nodeResourcesTuning.getConfig(builder); tuning.ifPresent(t -> t.getConfig(builder)); diff --git a/config-model/src/main/resources/schema/version/7.x/.gitignore b/config-model/src/main/resources/schema/version/7.x/.gitignore deleted file mode 100644 index e8bf39e2289..00000000000 --- a/config-model/src/main/resources/schema/version/7.x/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -*.xsd -*.rng diff --git a/config-model/src/main/resources/schema/version/7.x/admin.rnc b/config-model/src/main/resources/schema/version/7.x/admin.rnc deleted file mode 100644 index 2b9b414374c..00000000000 --- a/config-model/src/main/resources/schema/version/7.x/admin.rnc +++ /dev/null @@ -1,115 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -Admin = AdminV2 | AdminV3 | AdminV4 - -AdminV2 = - element admin { - attribute version { "2.0" } & - element adminserver { service.attlist } & - GenericConfig* & - LogServer? & - (ConfigServer | ConfigServers)? & - AdminSlobroks? & - AdminMonitoring? & - Metrics? & - ClusterControllers? & - LogForwarding? - } - -AdminV3 = - element admin { - attribute version { "3.0" } & - GenericConfig* & - Nodes - } - -AdminV4 = - element admin { - attribute version { "4.0" } & - AdminV4Slobroks? & - AdminV4LogServers? & - GenericConfig* & - AdminMonitoring? & - Metrics? & - LogForwarding? - } - -AdminV4Slobroks = - element slobroks { - OptionalDedicatedNodes - } - -AdminV4LogServers = - element logservers { - OptionalDedicatedNodes - } - -AdminSlobroks = - element slobroks { - element slobrok { - service.attlist & - attribute index { xsd:nonNegativeInteger }? - }+ - } - -AdminMonitoring = - element monitoring { - attribute interval { xsd:int }?, - attribute systemname { xsd:string }? - } - -ConfigServer = element configserver { - service.attlist -} - -ConfigServers = element configservers { - ConfigServer+ -} - -LogServer = element logserver { - service.attlist -} - -Metrics = element metrics { - element consumer { - attribute id { xsd:Name } & - element metric-set { attribute id { xsd:Name } }* & - element metric { - attribute id { xsd:Name } & - attribute display-name { xsd:Name }? - }* & - Cloudwatch? - }+ -} - -Cloudwatch = element cloudwatch { - attribute region { xsd:Name } & - attribute namespace { xsd:string { pattern = "[\w_\-/#:\.]+" } } & - ( - element credentials { - attribute access-key-name { xsd:Name } & - attribute secret-key-name { xsd:Name } - } - | - element shared-credentials { - attribute file { string } & - attribute profile { xsd:Name }? - } - )? -} - -ClusterControllers = element cluster-controllers { - attribute standalone-zookeeper { xsd:string }? & - element cluster-controller { - service.attlist - }+ -} - -LogForwarding = element logforwarding { - attribute include-admin { xsd:boolean }? & - element splunk { - attribute splunk-home { xsd:string }? & - attribute deployment-server { xsd:string } & - attribute client-name { xsd:string } & - attribute phone-home-interval { xsd:positiveInteger }? - } -} diff --git a/config-model/src/main/resources/schema/version/7.x/clients-v2.rnc b/config-model/src/main/resources/schema/version/7.x/clients-v2.rnc deleted file mode 100644 index 5a9353321c8..00000000000 --- a/config-model/src/main/resources/schema/version/7.x/clients-v2.rnc +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -Clients20 = - attribute version { "2.0" } & - LoadTypes? - -LoadTypes = element load-types { - element type { - attribute name { text } & - attribute default-priority { text }? - }* -} - -FeederOptions20 = element feederoptions { - FeederOptionsOpts20 & - DocProcChain? -} - -FeederOptionsOpts20 = element abortondocumenterror { xsd:boolean }? & - element retryenabled { text }? & - element route { text }? & - element maxpendingdocs { xsd:positiveInteger }? & - element maxpendingbytes { xsd:positiveInteger }? & - element retrydelay { xsd:double { minInclusive = "0.0" } }? & - element timeout { xsd:double { minInclusive = "0.0" } }? & - element tracelevel { xsd:positiveInteger }? & - element mbusport { xsd:positiveInteger }? - - -DocProcChain = element docprocchain { - text -} diff --git a/config-model/src/main/resources/schema/version/7.x/clients.rnc b/config-model/src/main/resources/schema/version/7.x/clients.rnc deleted file mode 100644 index 1e1dffc90b9..00000000000 --- a/config-model/src/main/resources/schema/version/7.x/clients.rnc +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -include "clients-v2.rnc" - -Clients = element clients { - Clients20 -} diff --git a/config-model/src/main/resources/schema/version/7.x/common.rnc b/config-model/src/main/resources/schema/version/7.x/common.rnc deleted file mode 100644 index 45183b0657c..00000000000 --- a/config-model/src/main/resources/schema/version/7.x/common.rnc +++ /dev/null @@ -1,73 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -service.attlist &= attribute hostalias { xsd:NCName } -service.attlist &= attribute baseport { xsd:unsignedShort }? -service.attlist &= attribute jvmargs { text }? -service.attlist &= attribute jvm-options { text }? -service.attlist &= attribute jvm-gc-options { text }? -# preload is for internal use only -service.attlist &= attribute preload { text }? - -anyElement = element * { - (attribute * { text } - | text - | anyElement)* -} - -# Valid qualified java class name. See http://docs.oracle.com/javase/specs/jls/se8/html/jls-3.html#jls-3.8 -JavaId = xsd:string { pattern = "([a-zA-Z_$][a-zA-Z\d_$]*\.)*[a-zA-Z_$][a-zA-Z\d_$]*" } - -Nodes = element nodes { - attribute count { xsd:positiveInteger | xsd:string } & - attribute flavor { xsd:string }? & - attribute docker-image { xsd:string }? & - Resources? -} - -Resources = element resources { - attribute vcpu { xsd:double { minExclusive = "0.0" } | xsd:string } & - attribute memory { xsd:string } & - attribute disk { xsd:string } & - attribute disk-speed { xsd:string }? & - attribute storage-type { xsd:string }? & - attribute architecture { xsd:string }? -} - -OptionalDedicatedNodes = element nodes { - attribute count { xsd:positiveInteger | xsd:string } & - attribute flavor { xsd:string }? & - attribute required { xsd:boolean }? & - attribute docker-image { xsd:string }? & - attribute dedicated { xsd:boolean }? & - attribute exclusive { xsd:boolean }? & - Resources? -} - -GenericConfig = element config { - attribute name { text }, - attribute namespace { text }?, - attribute version { text }?, - anyElement* -} - -ComponentSpec = - ( attribute id { xsd:Name | JavaId } | attribute idref { xsd:Name } | attribute ident { xsd:Name } ) - -ComponentId = - ComponentSpec - -BundleSpec = - attribute class { xsd:Name | JavaId }? & - attribute bundle { xsd:Name }? - -Component = element component { - ComponentDefinition -} - -ComponentDefinition = - ComponentId & - BundleSpec & - GenericConfig* & - Component* - - - diff --git a/config-model/src/main/resources/schema/version/7.x/container-include.rnc b/config-model/src/main/resources/schema/version/7.x/container-include.rnc deleted file mode 100644 index 8f6a8a3bada..00000000000 --- a/config-model/src/main/resources/schema/version/7.x/container-include.rnc +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -include "containercluster.rnc" -include "common.rnc" -include "container.rnc" -include "docproc.rnc" -include "searchchains.rnc" - -start = SearchInContainer | DocprocInContainer | ProcessingInContainer | Components diff --git a/config-model/src/main/resources/schema/version/7.x/container.rnc b/config-model/src/main/resources/schema/version/7.x/container.rnc deleted file mode 100644 index f7f63829281..00000000000 --- a/config-model/src/main/resources/schema/version/7.x/container.rnc +++ /dev/null @@ -1,52 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -# Schema for common container options - -include "processing.rnc" - -Handler = element handler { - ComponentDefinition & - ServerBindings & - element clientBinding {text}* -} - -Binding = element binding {text} -ServerBindings = Binding* - -Client = element client { - ComponentDefinition & - element binding {text}* & - element serverBinding {text}* -} - -Server = element server { - ComponentDefinition -} - -AccessControl = element access-control { - attribute domain { xsd:NCName }? & - attribute read { string "true" | string "false" }? & - attribute write { string "true" | string "false" }? & - attribute tls-handshake-client-auth {string "want" | string "need"}? & - element vespa-domain { xsd:NCName }? & - element exclude { - Binding+ - }? -} - -HttpFilterChain = - HttpFilter* & - ChainBaseContent & - ServerBindings - -HttpFilter = element filter { - ComponentDefinition & - FilterConfig? -} - -FilterConfig = element filter-config { - anyElement* -} - -Renderer = element renderer { - ComponentDefinition -} diff --git a/config-model/src/main/resources/schema/version/7.x/containercluster.rnc b/config-model/src/main/resources/schema/version/7.x/containercluster.rnc deleted file mode 100644 index 3fdbff84f6d..00000000000 --- a/config-model/src/main/resources/schema/version/7.x/containercluster.rnc +++ /dev/null @@ -1,283 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -ContainerCluster = element container | jdisc { - attribute version { "1.0" } & - attribute id { xsd:NCName }? & - attribute jetty { xsd:boolean }? & - Include* & - ContainerServices & - DocumentBinding* & - Aliases? & - NodesOfContainerCluster? & - ClientAuthorize? -} - -ContainerServices = - SearchInContainer? & - DocprocInContainer? & - ProcessingInContainer? & - ModelEvaluation? & - DocumentApi? & - Components* & - Component* & - Embedder* & - Handler* & - Client* & - Server* & - Http? & - AccessLog* & - SecretStore? & - ZooKeeper? & - GenericConfig* - -# TODO(ogronnesby): Change this configuration syntax -ClientAuthorize = element client-authorize { empty } - -Components = element components { - Include* & - Component* -} - -Aliases = element aliases { - element service-alias { xsd:NCName }* & - element endpoint-alias { xsd:NCName }* -} - -Include = element \include { - attribute dir { text } -} - -Http = element http { - (Filtering & HttpServer+) | - HttpServer+ | - empty -} - -Filtering = element filtering { - attribute strict-mode { xsd:boolean }? & - HttpFilter* & - AccessControl? & - element request-chain { - HttpFilterChain - }* & - element response-chain { - HttpFilterChain - }* -} - -HttpServer = element server { - attribute port { xsd:nonNegativeInteger }? & - attribute required { xsd:boolean }? & - attribute default-request-chain { xsd:string }? & - attribute default-response-chain { xsd:string }? & - ComponentId & - (Ssl | SslProvider)? & - GenericConfig* -} - -AccessLog = element accesslog { - attribute type { string "yapache" | string "vespa" | string "json" | string "disabled" }? & - attribute fileNamePattern { string }? & - attribute compressOnRotation { xsd:boolean }? & - attribute symlinkName { string }? & - attribute compressionType { string "gzip" | string "zstd" }? & - attribute queueSize { xsd:nonNegativeInteger }? & - attribute bufferSize { xsd:nonNegativeInteger }? & - attribute rotationInterval { string }? -} - -SecretStore = element secret-store { - attribute type { string "oath-ckms" | string "cloud" } & - element group { - attribute name { string } & - attribute environment { string "alpha" | string "corp" | string "prod" | string "aws" | string "aws_stage" } - } * & - element store { - attribute id { string } & - element aws-parameter-store { - attribute account { string } & - attribute aws-region { string } - } * - }? -} - -ZooKeeper = element zookeeper { - empty -} - -Embedder = element embedder { - attribute id { string }? & - attribute class { xsd:Name | JavaId }? & - attribute bundle { xsd:Name }? & - attribute def { xsd:Name }? & - anyElement* -} - -ModelEvaluation = element model-evaluation { - element onnx { - element models { - element model { - attribute name { string } & - element intraop-threads { xsd:nonNegativeInteger }? & - element interop-threads { xsd:nonNegativeInteger }? & - element execution-mode { string "sequential" | string "parallel" }? - }* - }? - }? -} - -Ssl = element ssl { - element private-key-file { string } & - element certificate-file { string } & - element ca-certificates-file { string }? & - element client-authentication { string "disabled" | string "want" | string "need" }? & - element cipher-suites { string }? & - element protocols { string }? -} - -SslProvider = element ssl-provider { - BundleSpec -} - -Threadpool = element threadpool { - element max-threads { xsd:nonNegativeInteger } & - element min-threads { xsd:nonNegativeInteger } & - element queue-size { xsd:nonNegativeInteger } -} - -# SEARCH: - -SearchInContainer = element search { - Include* & - ServerBindings? & - Searcher* & - SearchChain* & - Provider* & - Renderer* & - GenericConfig* & - Threadpool? -} - -SearchChain = element chain { - GenericSearchChain -} - -GenericSearchChain = - ComponentId & - SearchChainInheritance & - attribute searchers { text }? & - Searcher* & - Phase* & - GenericConfig* - -SearchChainInheritance = - attribute inherits { text }? & - attribute excludes { text }? & - element inherits { - element chain { ComponentSpec }* & - element exclude { ComponentSpec }* - }? - - -# DOCPROC: - -DocprocInContainer = element document-processing { - Include* & - DocprocClusterAttributes? & - DocumentProcessorV3* & - ChainInDocprocInContainerCluster* & - GenericConfig* -} -ChainInDocprocInContainerCluster = element chain { - DocprocChainV3Contents -} - - - -# PROCESSING: - -ProcessingInContainer = element processing { - Include* & - ServerBindings? & - Processor* & - Chain* & - ProcessingRenderer* & - GenericConfig* -} - - - -# DOCUMENT API: - -DocumentApi = element document-api { - ServerBindings & - GenericConfig* & - element abortondocumenterror { xsd:boolean }? & - element retryenabled { xsd:boolean }? & - element route { text }? & - element maxpendingdocs { xsd:positiveInteger }? & - element maxpendingbytes { xsd:positiveInteger }? & - element retrydelay { xsd:double { minInclusive = "0.0" } }? & - element timeout { xsd:double { minInclusive = "0.0" } }? & - element tracelevel { xsd:positiveInteger }? & - element mbusport { xsd:positiveInteger }? & - DocumentRestApi? & - HttpClientApi? -} - -DocumentRestApi = element rest-api { empty } - -HttpClientApi = element http-client-api { - Threadpool? -} - -# NODES: - -NodesOfContainerCluster = element nodes { - attribute jvmargs { text }? & - attribute jvm-options { text }? & - attribute jvm-gc-options { text }? & - attribute preload { text }? & - attribute allocated-memory { text }? & - attribute cpu-socket-affinity { xsd:boolean }? & - element jvm { - attribute options { text }? & - attribute gc-options { text }? & - attribute allocated-memory { text }? - } ? & - Resources? & - element environment-variables { - anyElement + - } ? & - ( - ( - attribute of { xsd:string } & - attribute required { xsd:boolean }? - ) - | - attribute type { xsd:string } - | - ( - attribute count { xsd:positiveInteger | xsd:string } & - attribute flavor { xsd:string }? & - attribute required { xsd:boolean }? & - attribute exclusive { xsd:boolean }? & - attribute docker-image { xsd:string }? - ) - | - element node { - attribute hostalias { xsd:NCName } & - attribute cpu-socket { xsd:positiveInteger }? & - GenericConfig* - }+ - ) -} - - - -#DOCUMENT BINDINGS: - -DocumentBinding = element document { - attribute type { xsd:NCName } & - attribute class { xsd:NCName } & - attribute bundle { xsd:NCName } -} diff --git a/config-model/src/main/resources/schema/version/7.x/content.rnc b/config-model/src/main/resources/schema/version/7.x/content.rnc deleted file mode 100644 index a2b1bd908ec..00000000000 --- a/config-model/src/main/resources/schema/version/7.x/content.rnc +++ /dev/null @@ -1,395 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace a = "http://relaxng.org/ns/compatibility/annotations/1.0" - -include "container.rnc" -include "searchchains.rnc" - -Redundancy = element redundancy { - attribute reply-after { xsd:nonNegativeInteger }? & - xsd:nonNegativeInteger -} - -DistributionType = element distribution { - attribute type { string "strict" | string "loose" | string "legacy" } -} - -BucketSplitting = element bucket-splitting { - attribute max-documents { xsd:nonNegativeInteger }? & - attribute max-size { xsd:nonNegativeInteger }? & - attribute minimum-bits { xsd:nonNegativeInteger }? -} - -MergeTuning = element merges { - attribute max-per-node { xsd:nonNegativeInteger }? & - attribute max-queue-size { xsd:nonNegativeInteger }? & - attribute max-nodes-per-merge { xsd:nonNegativeInteger { - minInclusive = "2" maxInclusive = "16" } }? -} - -VisitorMaxConcurrent = element max-concurrent { - attribute fixed { xsd:nonNegativeInteger }? & - attribute variable { xsd:nonNegativeInteger }? -} - -VisitorTuning = element visitors { - attribute thread-count { xsd:nonNegativeInteger }? & - attribute max-queue-size { xsd:nonNegativeInteger }? & - VisitorMaxConcurrent? -} - -Maintenance = element maintenance { - attribute start { xsd:string { pattern = "[0-9]{2}:[0-9]{2}" } }, - attribute stop { xsd:string { pattern = "[0-9]{2}:[0-9]{2}" } }, - attribute high { string "monday" | string "tuesday" | string "wednesday" | - string "thursday" | string "friday" | string "saturday" | - string "sunday" } -} - -PersistenceThread = element thread { - ## The lowest priority this thread should handle. - attribute lowest-priority { string "HIGHEST" | string "VERY_HIGH" | string "HIGH_1" | - string "HIGH_2" | string "HIGH_3" | string "NORMAL_1" | string "NORMAL_2" | - string "NORMAL_3" | string "NORMAL_4" | string "NORMAL_5" | string "NORMAL_6" | - string "LOW_1" | string "LOW_2" | string "LOW_3" | string "VERY_LOW" }? & - ## The number of threads of this type to create - attribute count { xsd:integer }? -} - -## Declare which storage threads each disk should have. -PersistenceThreads = element persistence-threads { - ## The number of threads to create - attribute count { xsd:integer }? & - ## All of the below settings are deprecated. - ## Operations with priority worse than this can be blocked - attribute highest-priority-to-block { xsd:string } ? & - ## Operations with priority better than this can block others - attribute lowest-priority-to-block-others { xsd:string } ? & - Thread* -} - -MinNodeRatioPerGroup = element min-node-ratio-per-group { - xsd:double { minInclusive = "0" maxInclusive = "1" } -} - -ClusterControllerTuning = element cluster-controller { - element init-progress-time { xsd:string { pattern = "([0-9\.]+)\s*([a-z]+)?" } }? & - element transition-time { xsd:string { pattern = "([0-9\.]+)\s*([a-z]+)?" } }? & - element max-premature-crashes { xsd:nonNegativeInteger }? & - element stable-state-period { xsd:string { pattern = "([0-9\.]+)\s*([a-z]+)?" } }? & - element min-distributor-up-ratio { xsd:double }? & - element min-storage-up-ratio { xsd:double }? -} - -DispatchTuning = element dispatch { - element max-hits-per-partition { xsd:nonNegativeInteger }? & - element dispatch-policy { string "round-robin" | string "adaptive" | string "random" }? & - element min-group-coverage { xsd:double }? & - element min-active-docs-coverage { xsd:double }? & - element top-k-probability { xsd:double }? & - element use-local-node { string "true" | string "false" }? -} - -ClusterTuning = element tuning { - DispatchTuning? & - DistributionType? & - BucketSplitting? & - MergeTuning? & - VisitorTuning? & - ClusterControllerTuning? & - Maintenance? & - PersistenceThreads? & - MinNodeRatioPerGroup? & - ResourceLimits? -} - -Content = element content { - attribute version { "1.0" } & - attribute id { xsd:NCName }? & - attribute distributor-base-port { xsd:unsignedShort }? & - # Mandatory - Redundancy & - ContentSearch? & - Dispatch? & - ClusterTuning? & - # Can be used for caches and feedbatching. - GenericConfig* & - Engine? & - # Here you can add document definitions that you also want to handle. - # Search might want to know of them in advance. - Documents? & - ContentNodes? & - TopGroup? & - Controllers? - # Contains experimental feature switches - #Experimental? -} - -Controllers = - element controllers { - OptionalDedicatedNodes - } - -ContentSearch = element search { - element query-timeout { xsd:double { minInclusive = "0" } }? & - element visibility-delay { xsd:double { minInclusive = "0" } }? & - SearchCoverage? -} - -SearchCoverage = element coverage { - element minimum { xsd:double { minInclusive = "0" maxInclusive = "1" } }? & - element min-wait-after-coverage-factor { xsd:double { minInclusive = "0" maxInclusive = "1" } }? & - element max-wait-after-coverage-factor { xsd:double { minInclusive = "0" maxInclusive = "1" } }? -} - -Dispatch = element dispatch { - element num-dispatch-groups { xsd:nonNegativeInteger }? & - DispatchGroup* -} - -DispatchGroup = element group { - DispatchNode+ -} - -DispatchNode = element node { - attribute distribution-key { xsd:nonNegativeInteger } -} - -## Specification of what document processing should be done for indexing. -DocumentProcessing = element document-processing { - attribute cluster { text }? & - attribute chain { text }? -} - -## Config for the persistence providers. -Engine = element engine { - (Proton | Dummy) -} - -Proton = element proton { - element flush-on-shutdown { xsd:string }? & - element sync-transactionlog { xsd:string }? & - element visibility-delay { xsd:double { minInclusive = "0.0" } }? & - element query-timeout { xsd:double { minInclusive = "0.0" } }? & - element searchable-copies { xsd:integer { minInclusive = "0" } }? & - ResourceLimits? & - Tuning? -} - -ResourceLimits = element resource-limits { - element disk { xsd:double { minInclusive = "0.0" maxInclusive = "1.0" } }? & - element memory { xsd:double { minInclusive = "0.0" maxInclusive = "1.0" } }? -} - -Dummy = element dummy { - text -} - -Documents = element documents { - attribute selection { xsd:string }? & - attribute garbage-collection { xsd:string }? & - attribute garbage-collection-interval { xsd:nonNegativeInteger }? & - DocumentProcessing? & - - element document { - attribute type { xsd:string } & - attribute selection { xsd:string }? & - attribute mode { string "index" | string "streaming" | string "store-only" } & - attribute global { xsd:boolean }? - }+ -} - -ContentNode = element node { - GenericConfig* & - service.attlist & - attribute distribution-key { xsd:nonNegativeInteger } & - attribute capacity { xsd:double { minExclusive = "0.0" } }? & - attribute mmap-core-limit { xsd:nonNegativeInteger }? & - attribute core-on-oom { xsd:boolean }? & - attribute no-vespamalloc { xsd:string }? & - attribute vespamalloc { xsd:string }? & - attribute vespamalloc-debug { xsd:string }? & - attribute vespamalloc-debug-stacktrace { xsd:string }? & - attribute cpu-socket { xsd:nonNegativeInteger }? -} - -ContentNodes = element nodes { - Resources? & - attribute cpu-socket-affinity { xsd:string }? & - attribute mmap-core-limit { xsd:nonNegativeInteger }? & - attribute core-on-oom { xsd:boolean }? & - attribute no-vespamalloc { xsd:string }? & - attribute vespamalloc { xsd:string }? & - attribute vespamalloc-debug { xsd:string }? & - attribute vespamalloc-debug-stacktrace { xsd:string }? & - ( - ( - attribute count { xsd:positiveInteger | xsd:string } & - attribute flavor { xsd:string }? & - attribute required { xsd:boolean }? & - attribute exclusive { xsd:boolean }? & - attribute docker-image { xsd:string }? & - attribute groups { xsd:positiveInteger | xsd:string }? - ) - | - ContentNode + - ) -} - -TopGroup = element group { - # Neither name nor distribution key makes any sense for the top group. There has to be - # a top group so it never needs referring to, and it's only one group to choose from so - # it has no use of a distribution key. Leaving it allowed to set them for now to not - # break all system tests and backward compatibility. - attribute name { xsd:string }? & - attribute cpu-socket-affinity { xsd:string }? & - attribute mmap-core-limit { xsd:nonNegativeInteger }? & - attribute core-on-oom { xsd:boolean }? & - attribute no-vespamalloc { xsd:string }? & - attribute vespamalloc { xsd:string }? & - attribute vespamalloc-debug { xsd:string }? & - attribute vespamalloc-debug-stacktrace { xsd:string }? & - attribute distribution-key { xsd:nonNegativeInteger }? & - ( - ContentNode + - | - ( - element distribution { - attribute partitions { xsd:string } - } & - Group + - ) - ) -} - -Group = element group { - attribute distribution-key { xsd:nonNegativeInteger } & - attribute name { xsd:string } & - ( - ContentNode + - | - ( - element nodes { - attribute count { xsd:positiveInteger | xsd:string } & - attribute flavor { xsd:string }? & - attribute required { xsd:boolean }? & - attribute exclusive { xsd:boolean }? & - attribute docker-image { xsd:string }? & - attribute groups { xsd:positiveInteger | xsd:string }? - } - ) - | - ( - element distribution { - attribute partitions { xsd:string } - } & - Group + - ) - ) -} - -Tuning = element tuning { - element dispatch { - element max-hits-per-partition { xsd:nonNegativeInteger }? - }? & - element searchnode { - element requestthreads { - element search { xsd:nonNegativeInteger }? & - element persearch { xsd:nonNegativeInteger }? & - element summary { xsd:nonNegativeInteger }? - }? & - element flushstrategy { - element native { - element total { - element maxmemorygain { xsd:nonNegativeInteger }? & - element diskbloatfactor { xsd:double { minInclusive = "0.0" } }? - }? & - element component { - element maxmemorygain { xsd:nonNegativeInteger }? & - element diskbloatfactor { xsd:double { minInclusive = "0.0" } }? & - element maxage { xsd:nonNegativeInteger }? - }? & - element transactionlog { - element maxsize { xsd:nonNegativeInteger }? - }? & - element conservative { - element memory-limit-factor { xsd:double { minInclusive = "0.0" maxInclusive = "1.0" } }? & - element disk-limit-factor { xsd:double { minInclusive = "0.0" maxInclusive = "1.0" } }? - }? - }? - }? & - element resizing { - element initialdocumentcount { xsd:nonNegativeInteger }? & - element amortize-count { xsd:nonNegativeInteger }? - }? & - element index { - element io { - element write { TuningIoOptionsLight }? & - element read { TuningIoOptionsLight }? & - element search { TuningIoOptionsSearch }? - }? & - element warmup { - element time { xsd:double { minInclusive = "0.0" } }? & - element unpack { xsd:boolean }? - }? - }? & - element attribute { - element io { - element write { TuningIoOptionsLight }? - } - }? & - element summary { - element io { - element write { TuningIoOptionsLight }? & - element read { TuningIoOptionsFull }? - }? & - element store { - element cache { - element maxsize { xsd:nonNegativeInteger }? & - element maxsize-percent { xsd:double { minInclusive = "0.0" maxInclusive = "50.0" } }? & - element initialentries { xsd:nonNegativeInteger }? & - TuningCompression? - }? & - element logstore { - element maxfilesize { xsd:nonNegativeInteger }? & - element minfilesizefactor { xsd:double { minInclusive = "0.10" maxInclusive = "1.0" } }? & - element chunk { - element maxsize { xsd:nonNegativeInteger }? & - TuningCompression? - }? - }? - }? - }? & - element initialize { - element threads { xsd:nonNegativeInteger }? - }? & - element feeding { - element concurrency { xsd:double { minInclusive = "0.0" maxInclusive = "1.0" } }? - }? & - element removed-db { - element prune { - element age { xsd:double { minInclusive = "0.0" } }? & - element interval { xsd:double { minInclusive = "60.0" } }? - }? - }? - }? -} - -TuningIoOptionsLight = string "normal" | string "directio" -TuningIoOptionsFull = string "normal" | string "directio" | string "mmap" | string "populate" -TuningIoOptionsSearch = string "mmap" | string "populate" - -TuningCompression = element compression { - element type { string "none" | string "lz4" | string "zstd" }? & - element level { xsd:nonNegativeInteger }? -} - -#Experimental = element experimental { -# Put experimental flags here -#} - -Thread = element thread { - ## The lowest priority this thread should handle. - attribute lowest-priority { xsd:string}? & - ## The number of threads of this type to create - attribute count { xsd:integer }? -} diff --git a/config-model/src/main/resources/schema/version/7.x/deployment.rnc b/config-model/src/main/resources/schema/version/7.x/deployment.rnc deleted file mode 100644 index 3abced8e04a..00000000000 --- a/config-model/src/main/resources/schema/version/7.x/deployment.rnc +++ /dev/null @@ -1,144 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -# RELAX NG Compact Syntax -# Vespa Deployment file - -start = element deployment { - attribute version { "1.0" } & - attribute major-version { text }? & - attribute athenz-domain { xsd:string }? & - attribute athenz-service { xsd:string }? & - attribute cloud-account { xsd:string }? & - Step -} - -Step = - StepExceptInstance & - Instance* - -StepExceptInstance = - Delay* & - ParallelInstances* & - Upgrade? & - BlockChange* & - Notifications? & - Endpoints? & - Test? & - Staging? & - Prod* - -PrimitiveStep = - Delay* & - Region* & - ProdTest* - -Instance = element instance { - attribute id { xsd:string } & - attribute athenz-service { xsd:string }? & - attribute cloud-account { xsd:string }? & - StepExceptInstance -} - -ParallelSteps = element parallel { - SerialSteps* & - PrimitiveStep* -} - -SerialSteps = element steps { - ParallelSteps* & - PrimitiveStep* -} - -ParallelInstances = element parallel { - Instance* -} - -Upgrade = element upgrade { - attribute policy { xsd:string }? & - attribute revision-target { xsd:string }? & - attribute revision-change { xsd:string }? & - attribute min-risk { xsd:long }? & - attribute max-risk { xsd:long }? & - attribute max-idle-hours { xsd:long }? & - attribute rollout { xsd:string }? -} - -BlockChange = element block-change { - attribute revision { xsd:boolean }? & - attribute version { xsd:boolean }? & - attribute days { xsd:string }? & - attribute hours { xsd:string }? & - attribute from-date { xsd:string }? & - attribute to-date { xsd:string }? & - attribute time-zone { xsd:string }? -} - -Notifications = element notifications { - attribute when { xsd:string }? & - Email* -} - -Email = element email { - attribute address { xsd:string }? & - attribute role { xsd:string }? & - attribute when { xsd:string }? -} - -Test = element test { - attribute athenz-service { xsd:string }? & - attribute tester-flavor { xsd:string }? & - text -} - -Staging = element staging { - attribute athenz-service { xsd:string }? & - attribute tester-flavor { xsd:string }? & - text -} - -Prod = element prod { - attribute global-service-id { text }? & - attribute athenz-service { xsd:string }? & - attribute tester-flavor { xsd:string }? & - Region* & - Delay* & - ProdTest* & - ParallelSteps* -} - -ProdTest = element test { - text -} - -Region = element region { - attribute active { xsd:boolean }? & - attribute athenz-service { xsd:string }? & - attribute cloud-account { xsd:string }? & - text -} - -Delay = element delay { - attribute hours { xsd:long }? & - attribute minutes { xsd:long }? & - attribute seconds { xsd:long }? -} - -EndpointRegion = element region { - text -} - -EndpointInstance = element instance { - attribute weight { xsd:long } & - text -} - -Endpoint = element endpoint { - attribute id { xsd:string }? & - attribute container-id { xsd:string } & - attribute region { xsd:string }? & - EndpointRegion* & - EndpointInstance* -} - -Endpoints = element endpoints { - Endpoint+ -} diff --git a/config-model/src/main/resources/schema/version/7.x/docproc-standalone.rnc b/config-model/src/main/resources/schema/version/7.x/docproc-standalone.rnc deleted file mode 100644 index caba5327e25..00000000000 --- a/config-model/src/main/resources/schema/version/7.x/docproc-standalone.rnc +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -include "common.rnc" -include "container.rnc" -include "searchchains.rnc" -include "docproc.rnc" -start = DocprocChainsV3 diff --git a/config-model/src/main/resources/schema/version/7.x/docproc.rnc b/config-model/src/main/resources/schema/version/7.x/docproc.rnc deleted file mode 100644 index 1e7e28b2002..00000000000 --- a/config-model/src/main/resources/schema/version/7.x/docproc.rnc +++ /dev/null @@ -1,99 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -DocProc = element docproc { - DocProcV3 -} - -SchemaMapping = element map { - element field { - attribute doctype { text }?, - attribute in-document { text }, - attribute in-processor { text } - }+ -} - - - - -#Version 3 config: - -DocProcV3 = attribute version { "3.0" }, - (ClusterV3* & - OuterDocprocChainsV3? & - GenericConfig* -) - -DocprocClusterAttributes = attribute compressdocuments { xsd:boolean }? & - attribute numnodesperclient { xsd:positiveInteger }? & - attribute preferlocalnode { xsd:boolean }? & - attribute maxmessagesinqueue { xsd:nonNegativeInteger }? & - attribute maxqueuebytesize { xsd:string { minLength = "1" } }? & - attribute maxqueuewait { xsd:positiveInteger }? & - attribute maxconcurrentfactor { xsd:double { minExclusive = "0.0" maxExclusive = "1.0" } }? & - attribute documentexpansionfactor { xsd:double { minExclusive = "0.0" } }? & - attribute containercorememory { xsd:nonNegativeInteger }? - -ClusterV3 = element cluster { - attribute name { xsd:NCName } & - DocprocClusterAttributes? & - - element nodes { - Resources? & - attribute jvmargs { text }? & - attribute preload { text }? & - element node { - GenericConfig* & - service.attlist & - attribute maxmessagesinqueue { xsd:nonNegativeInteger }? & - attribute maxqueuebytesize { xsd:string { minLength = "1" } }? & - attribute maxqueuewait { xsd:positiveInteger }? - }+ - } & - GenericConfig* & - SchemaMapping? & - Component* & - Handler* & - DocprocChainsV3? -} - -DocprocChainsV3 = - element docprocchains { - DocumentProcessorV3* & - DocprocChainV3* & - GenericConfig* - } - -OuterDocprocChainsV3 = - element docprocchains { - DocumentProcessorV3* & - DocprocChainV3* - } - -DocprocChainV3 = - element docprocchain { - DocprocChainV3Contents - } - -DocprocChainV3Contents = attribute name { xsd:NCName }? & - ComponentId & - SchemaMapping? & - DocprocChainInheritance & - attribute documentprocessors { text }? & - DocumentProcessorV3* & - Phase* & - GenericConfig* - - -DocprocChainInheritance = - attribute inherits { text }? & - attribute excludes { text }? & - element inherits { - element docprocchain { ComponentSpec }* & - element exclude { ComponentSpec }* - }? - -DocumentProcessorV3 = - element documentprocessor { - BundleSpec & - SchemaMapping? & - GenericSearcherOrDocumentProcessor - } diff --git a/config-model/src/main/resources/schema/version/7.x/federation.rnc b/config-model/src/main/resources/schema/version/7.x/federation.rnc deleted file mode 100644 index 8e341fa7a9c..00000000000 --- a/config-model/src/main/resources/schema/version/7.x/federation.rnc +++ /dev/null @@ -1,74 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -# Schema for federation configuration inside the searchchains section. - -GenericSource = - GenericSearchChainInQrservers & - FederationOptions? - -Source = - element source { - GenericSource - } - -Provider = - element provider { - GenericSource & - attribute cachesize { xsd:string { pattern = "\d+(\.\d*)?\s*[kmgKMG]?" } }? & - attribute type { xsd:string }? & - attribute cluster { xsd:string }? & - - attribute yca-application-id { xsd:string }? & - attribute yca-cache-ttl { xsd:string { pattern = "\d+(\.\d*)?\s*m?s" } }? & - attribute yca-cache-retry-wait { xsd:string { pattern = "\d+(\.\d*)?\s*m?s" } }? & - YcaProxy? & - HttpProviderSearcherOptions & - - Source* - } - -YcaProxy = - element yca-proxy { - GenericNode? - } - -GenericNode = - attribute host { xsd:string } & - attribute port { xsd:int } - - -HttpProviderSearcherOptions = - attribute cacheweight { xsd:float { minInclusive = "0" } }? & - attribute path { xsd:string }? & - attribute readtimeout { xsd:string { pattern = "\d+(\.\d*)?\s*m?s" } }? & - attribute connectiontimeout { xsd:string { pattern = "\d+(\.\d*)?\s*m?s" } }? & - attribute connectionpooltimeout { xsd:string { pattern = "\d+(\.\d*)?\s*m?s" } }? & - attribute retries { xsd:int }? & - element nodes { - element node { - attribute host { xsd:string } & - attribute port { xsd:int } - }+ - } ? - -FederationOptions = - element federationoptions { - attribute optional { xsd:boolean }? & - attribute timeout { xsd:string { pattern = "\d+(\.\d*)?\s*m?s" } }? & - attribute requestTimeout { xsd:string { pattern = "\d+(\.\d*)?\s*m?s" } }? & - attribute default { xsd:boolean }? - } - -FederationSearcher = - element federation { - GenericSearcherOrDocumentProcessor & - element source { - ComponentSpec & - FederationOptions? - }* & - element target-selector { - ComponentDefinition - }? & - element source-set { - attribute inherits { string "default" } - }? - } diff --git a/config-model/src/main/resources/schema/version/7.x/genericcluster.rnc b/config-model/src/main/resources/schema/version/7.x/genericcluster.rnc deleted file mode 100644 index a749a592c09..00000000000 --- a/config-model/src/main/resources/schema/version/7.x/genericcluster.rnc +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -# Generic, application-specific service cluster -# -include "genericmodule.rnc" - -GenericCluster = element service { - attribute version { "1.0" } & - attribute id { xsd:NCName }? & - attribute name { text } & - attribute command { text } & - attribute hostservice { text }? & - attribute num-hosts { text }? & - GenericConfig* & - GenericModule* & - element node { - service.attlist & - attribute name { text }? & - GenericModule* & - GenericConfig* - }* -} - diff --git a/config-model/src/main/resources/schema/version/7.x/genericmodule.rnc b/config-model/src/main/resources/schema/version/7.x/genericmodule.rnc deleted file mode 100644 index 5f8ac3f7dda..00000000000 --- a/config-model/src/main/resources/schema/version/7.x/genericmodule.rnc +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -# Generic, nestable module - -GenericModule = element module { - attribute name { text } & - GenericConfig* & - GenericModule* -} diff --git a/config-model/src/main/resources/schema/version/7.x/hosts.rnc b/config-model/src/main/resources/schema/version/7.x/hosts.rnc deleted file mode 100644 index d089b23804e..00000000000 --- a/config-model/src/main/resources/schema/version/7.x/hosts.rnc +++ /dev/null @@ -1,10 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -# RELAX NG Compact Syntax -# Vespa Hosts file - -element hosts { - element host { - attribute name { text }, - element alias { text }* - }+ -} diff --git a/config-model/src/main/resources/schema/version/7.x/legacygenericmodule.rnc b/config-model/src/main/resources/schema/version/7.x/legacygenericmodule.rnc deleted file mode 100644 index a54f7fd9afc..00000000000 --- a/config-model/src/main/resources/schema/version/7.x/legacygenericmodule.rnc +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -# Generic, nestable module - -LegacyGenericModule = element module { - attribute name { text } & - GenericConfig* & - LegacyGenericModule* -} diff --git a/config-model/src/main/resources/schema/version/7.x/processing.rnc b/config-model/src/main/resources/schema/version/7.x/processing.rnc deleted file mode 100644 index a753de70265..00000000000 --- a/config-model/src/main/resources/schema/version/7.x/processing.rnc +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -# Schema for processing components and chains - -ProcessingRenderer = element renderer { - ComponentDefinition -} - -Processing = - element processing { - Processor* & - Chain* & - ProcessingRenderer* & - GenericConfig* - } - -ChainBaseContent = - ComponentId & - ChainInheritance & - GenericConfig* - -Chain = element chain { - ChainBaseContent & - Processor* & - Phase* - } - -ChainInheritance = - attribute inherits { text }? & - attribute excludes { text }? & - element inherits { - element chain { ComponentSpec }* & - element exclude { ComponentSpec }* - }? - -Processor = - element processor { - BundleSpec & - GenericSearcherOrDocumentProcessor - } diff --git a/config-model/src/main/resources/schema/version/7.x/routing-standalone.rnc b/config-model/src/main/resources/schema/version/7.x/routing-standalone.rnc deleted file mode 100644 index e95369fd192..00000000000 --- a/config-model/src/main/resources/schema/version/7.x/routing-standalone.rnc +++ /dev/null @@ -1,4 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -include "common.rnc" -include "routing.rnc" -start = Routing diff --git a/config-model/src/main/resources/schema/version/7.x/routing.rnc b/config-model/src/main/resources/schema/version/7.x/routing.rnc deleted file mode 100644 index 5ca033b2fd7..00000000000 --- a/config-model/src/main/resources/schema/version/7.x/routing.rnc +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -# RELAX NG compact syntax pattern -# for Vespa MessageBus explicit routing config -Routing = element routing { - attribute version { "1.0" } & - element routingtable { - attribute protocol { string "document" } & - attribute verify { xsd:boolean }? & - element hop { - attribute name { text } & - attribute selector { text } & - attribute ignore-result { xsd:boolean }? & - element recipient { - attribute session { text } - }* - }* & - element route { - attribute name { text } & - attribute hops { text } - }* - }* & - element services { - attribute protocol { string "document" } & - element service { - attribute name { text } - }* - }* -} diff --git a/config-model/src/main/resources/schema/version/7.x/schemas.xml b/config-model/src/main/resources/schema/version/7.x/schemas.xml deleted file mode 100644 index c3f8e2be448..00000000000 --- a/config-model/src/main/resources/schema/version/7.x/schemas.xml +++ /dev/null @@ -1,7 +0,0 @@ -<?xml version="1.0"?> -<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<locatingRules xmlns="http://thaiopensource.com/ns/locating-rules/1.0"> - <documentElement localName="hosts" uri="hosts.rnc"/> - <documentElement localName="services" uri="services.rnc"/> - <documentElement localName="deployment" uri="deployment.rnc"/> -</locatingRules> diff --git a/config-model/src/main/resources/schema/version/7.x/searchchains-standalone.rnc b/config-model/src/main/resources/schema/version/7.x/searchchains-standalone.rnc deleted file mode 100644 index 6725627143d..00000000000 --- a/config-model/src/main/resources/schema/version/7.x/searchchains-standalone.rnc +++ /dev/null @@ -1,4 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -include "common.rnc" -include "searchchains.rnc" -start = SearchChains diff --git a/config-model/src/main/resources/schema/version/7.x/searchchains.rnc b/config-model/src/main/resources/schema/version/7.x/searchchains.rnc deleted file mode 100644 index d4c9e8f4f98..00000000000 --- a/config-model/src/main/resources/schema/version/7.x/searchchains.rnc +++ /dev/null @@ -1,72 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#Schema for search chains and searchers inside the searchchains section. - -include "federation.rnc" - -SearchChains = - element searchchains { - Searcher* & - SearchChainInQrservers* & - GenericConfig* - } - -OuterSearchChains = - element searchchains { - Searcher* & - SearchChainInQrservers* - } - -SearchChainInQrservers = - element searchchain { - GenericSearchChainInQrservers - } | - Provider - -GenericSearchChainInQrservers = - ComponentId & - SearchChainInheritanceInQrservers & - attribute searchers { text }? & - Searcher* & - Phase* & - GenericConfig* - -SearchChainInheritanceInQrservers = - attribute inherits { text }? & - attribute excludes { text }? & - element inherits { - element searchchain { ComponentSpec }* & - element exclude { ComponentSpec }* - }? - -Searcher = - RegularSearcher | - FederationSearcher - -RegularSearcher = - element searcher { - BundleSpec & - GenericSearcherOrDocumentProcessor - } - - -GenericSearcherOrDocumentProcessor = - ComponentId & - SearcherOrDocumentProcessorDependencies & - GenericConfig* - -SearcherOrDocumentProcessorDependencies = - Dependencies & - attribute provides { text }? & - element provides { xsd:Name }* - -Dependencies = - attribute before { text }? & - attribute after { text }? & - element before { xsd:Name }* & - element after { xsd:Name }* - -Phase = - element phase { - ComponentId & - Dependencies - } diff --git a/config-model/src/main/resources/schema/version/7.x/services.rnc b/config-model/src/main/resources/schema/version/7.x/services.rnc deleted file mode 100644 index 3a8ffe30563..00000000000 --- a/config-model/src/main/resources/schema/version/7.x/services.rnc +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -include "common.rnc" -include "admin.rnc" -include "clients.rnc" -include "content.rnc" -include "docproc.rnc" -include "routing.rnc" -include "containercluster.rnc" -include "genericcluster.rnc" - -start = element services { - attribute version { "1.0" }? & - attribute application-type { "hosted-infrastructure" }? & - element legacy { element v7-geo-positions { xsd:boolean } }? & - GenericCluster* & - GenericConfig* & - Admin? & - Clients? & - Content* & - ContainerCluster* & - Routing? -} -| GenericCluster -| Admin -| Clients -| Content -| ContainerCluster - diff --git a/config-model/src/main/resources/schema/version/7.x/validation-overrides.rnc b/config-model/src/main/resources/schema/version/7.x/validation-overrides.rnc deleted file mode 100644 index a0caa10fc60..00000000000 --- a/config-model/src/main/resources/schema/version/7.x/validation-overrides.rnc +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -# RELAX NG Compact Syntax -# Vespa validation overrides - -start = element validation-overrides { - Allow* -} - -Allow = element allow { - attribute until { xsd:string } & - attribute comment { xsd:string }? & - text -} diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java index c128b9af6e0..9126c678171 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java @@ -1327,7 +1327,7 @@ public class ModelProvisioningTest { assertEquals(4, cluster.getRedundancy().effectiveInitialRedundancy()); assertEquals(4, cluster.getRedundancy().effectiveFinalRedundancy()); assertEquals(4, cluster.getRedundancy().effectiveReadyCopies()); - assertEquals(4, cluster.getSearch().getIndexed().getSearchableCopies()); + assertEquals(4, cluster.getRedundancy().readyCopies()); assertFalse(cluster.getRootGroup().getPartitions().isPresent()); assertEquals(4, cluster.getRootGroup().getNodes().size()); assertEquals(0, cluster.getRootGroup().getSubgroups().size()); diff --git a/config-model/src/test/java/com/yahoo/schema/document/ComplexAttributeFieldUtilsTestCase.java b/config-model/src/test/java/com/yahoo/schema/document/ComplexAttributeFieldUtilsTestCase.java index 310ede6bae2..7a89f52268f 100644 --- a/config-model/src/test/java/com/yahoo/schema/document/ComplexAttributeFieldUtilsTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/document/ComplexAttributeFieldUtilsTestCase.java @@ -30,11 +30,11 @@ public class ComplexAttributeFieldUtilsTestCase { } boolean isArrayOfSimpleStruct() { - return ComplexAttributeFieldUtils.isArrayOfSimpleStruct(field(), false); + return ComplexAttributeFieldUtils.isArrayOfSimpleStruct(field()); } boolean isMapOfSimpleStruct() { - return ComplexAttributeFieldUtils.isMapOfSimpleStruct(field(), false); + return ComplexAttributeFieldUtils.isMapOfSimpleStruct(field()); } boolean isMapOfPrimitiveType() { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/DedicatedAdminV4Test.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/DedicatedAdminV4Test.java index b809f25ced2..383027438fc 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/DedicatedAdminV4Test.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/DedicatedAdminV4Test.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin; +import ai.vespa.metrics.set.Metric; import com.yahoo.cloud.config.LogforwarderConfig; import com.yahoo.cloud.config.SentinelConfig; import com.yahoo.config.application.api.ApplicationPackage; @@ -15,7 +16,6 @@ import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.model.VespaModel; -import com.yahoo.vespa.model.admin.monitoring.Metric; import com.yahoo.vespa.model.admin.monitoring.MetricsConsumer; import com.yahoo.vespa.model.admin.monitoring.Monitoring; import org.junit.jupiter.api.Test; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsConsumersTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsConsumersTest.java index 0a1791b3766..d5ce26bafe8 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsConsumersTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsConsumersTest.java @@ -1,13 +1,18 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin.metricsproxy; +import ai.vespa.metrics.set.Metric; +import ai.vespa.metrics.set.MetricSet; import ai.vespa.metricsproxy.core.ConsumersConfig; import com.yahoo.vespa.model.VespaModel; -import com.yahoo.vespa.model.admin.monitoring.Metric; -import com.yahoo.vespa.model.admin.monitoring.MetricSet; import com.yahoo.vespa.model.admin.monitoring.MetricsConsumer; import org.junit.jupiter.api.Test; +import static ai.vespa.metrics.set.DefaultMetrics.defaultMetricSet; +import static ai.vespa.metrics.set.DefaultVespaMetrics.defaultVespaMetricSet; +import static ai.vespa.metrics.set.NetworkMetrics.networkMetricSet; +import static ai.vespa.metrics.set.SystemMetrics.systemMetricSet; +import static ai.vespa.metrics.set.VespaMetricSet.vespaMetricSet; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.TestMode.hosted; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.TestMode.self_hosted; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.checkMetric; @@ -16,11 +21,6 @@ import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.c import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getCustomConsumer; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getModel; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.servicesWithAdminOnly; -import static com.yahoo.vespa.model.admin.monitoring.DefaultMetrics.defaultMetricSet; -import static com.yahoo.vespa.model.admin.monitoring.DefaultVespaMetrics.defaultVespaMetricSet; -import static com.yahoo.vespa.model.admin.monitoring.NetworkMetrics.networkMetricSet; -import static com.yahoo.vespa.model.admin.monitoring.SystemMetrics.systemMetricSet; -import static com.yahoo.vespa.model.admin.monitoring.VespaMetricSet.vespaMetricSet; import static java.util.Collections.singleton; import static org.junit.jupiter.api.Assertions.*; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java index 2c9293473ed..fe5b6950d45 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.model.admin.metricsproxy; +import ai.vespa.metrics.set.Metric; import ai.vespa.metricsproxy.core.ConsumersConfig; import ai.vespa.metricsproxy.http.application.MetricsNodesConfig; import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensionsConfig; @@ -11,7 +12,6 @@ import ai.vespa.metricsproxy.service.VespaServicesConfig; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.search.config.QrStartConfig; import com.yahoo.vespa.model.VespaModel; -import com.yahoo.vespa.model.admin.monitoring.Metric; import com.yahoo.vespa.model.admin.monitoring.MetricsConsumer; import com.yahoo.vespa.model.test.VespaModelTester; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/monitoring/MetricSetTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/monitoring/MetricSetTest.java index 7eec4450f33..8235f45aaec 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/monitoring/MetricSetTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/monitoring/MetricSetTest.java @@ -1,6 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin.monitoring; +import ai.vespa.metrics.set.Metric; +import ai.vespa.metrics.set.MetricSet; import com.google.common.collect.Sets; import org.junit.jupiter.api.Test; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/monitoring/MetricTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/monitoring/MetricTest.java index e87bb90c36b..f07b8c59322 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/monitoring/MetricTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/monitoring/MetricTest.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin.monitoring; +import ai.vespa.metrics.set.Metric; import com.google.common.collect.ImmutableMap; import org.junit.jupiter.api.Test; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ComplexFieldsValidatorTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ComplexFieldsValidatorTestCase.java index 04abd4e4836..8f8918b5140 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ComplexFieldsValidatorTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ComplexFieldsValidatorTestCase.java @@ -71,9 +71,9 @@ public class ComplexFieldsValidatorTestCase { } @Test - void logs_warning_when_struct_field_inside_nested_struct_array_is_specified_as_attribute() throws IOException, SAXException { - var logger = new MyLogger(); - createModelAndValidate(joinLines( + void throws_exception_when_struct_field_inside_nested_struct_array_is_specified_as_attribute() throws IOException, SAXException { + Throwable exception = assertThrows(IllegalArgumentException.class, () -> { + createModelAndValidate(joinLines( "schema test {", "document test {", "struct item {", @@ -92,8 +92,10 @@ public class ComplexFieldsValidatorTestCase { "}", "}", "}", - "}"), logger); - assertTrue(logger.message.toString().contains(getExpectedMessage("cabinet (cabinet.value.items.name, cabinet.value.items.color)"))); + "}")); + + }); + assertTrue(exception.getMessage().contains(getExpectedMessage("cabinet (cabinet.value.items.name, cabinet.value.items.color)"))); } private String getExpectedMessage(String unsupportedFields) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java index b9909214dfd..d9632b62fb2 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java @@ -1309,17 +1309,6 @@ public class ContentClusterTest extends ContentBaseTest { assertEquals(2, resolveMaxInhibitedGroupsConfigWithFeatureFlag(2)); } - private boolean resolveConditionProbingFromWriteRepairFeatureFlag(boolean enable) throws Exception { - var cfg = resolveStorDistributormanagerConfig(new TestProperties().setEnableConditionalPutRemoveWriteRepair(enable)); - return cfg.enable_condition_probing(); - } - - @Test - void distributor_condition_probing_is_controlled_by_write_repair_feature_flag() throws Exception { - assertFalse(resolveConditionProbingFromWriteRepairFeatureFlag(false)); - assertTrue(resolveConditionProbingFromWriteRepairFeatureFlag(true)); - } - private int resolveNumDistributorStripesConfig(Optional<Flavor> flavor) throws Exception { var cc = createOneNodeCluster(new TestProperties(), flavor); var builder = new StorDistributormanagerConfig.Builder(); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/DispatchTuningTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/DispatchTuningTest.java index af547965749..15fba6a7dc9 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/DispatchTuningTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/DispatchTuningTest.java @@ -15,13 +15,13 @@ public class DispatchTuningTest { void requireThatAccessorWork() { DispatchTuning dispatch = new DispatchTuning.Builder() .setMaxHitsPerPartition(69) - .setDispatchPolicy("round-robin") + .setDispatchPolicy("best-of-random-2") .setMinActiveDocsCoverage(12.5) .setTopKProbability(18.3) .build(); assertEquals(69, dispatch.getMaxHitsPerPartition().intValue()); assertEquals(12.5, dispatch.getMinActiveDocsCoverage(), 0.0); - assertEquals(DispatchTuning.DispatchPolicy.ROUNDROBIN, dispatch.getDispatchPolicy()); + assertEquals(DispatchTuning.DispatchPolicy.BEST_OF_RANDOM_2, dispatch.getDispatchPolicy()); assertEquals(18.3, dispatch.getTopkProbability(), 0.0); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedHierarchicDistributionTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedHierarchicDistributionTest.java index 7fba5ba12e9..ab147f22e8b 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedHierarchicDistributionTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedHierarchicDistributionTest.java @@ -3,13 +3,10 @@ package com.yahoo.vespa.model.content; import com.yahoo.vespa.config.content.StorDistributionConfig; import com.yahoo.vespa.model.content.cluster.ContentCluster; -import com.yahoo.vespa.model.search.DispatchGroup; -import com.yahoo.vespa.model.search.SearchInterface; import com.yahoo.vespa.model.search.SearchNode; import com.yahoo.yolean.Exceptions; import org.junit.jupiter.api.Test; -import java.util.ArrayList; import java.util.List; import java.util.Optional; @@ -147,30 +144,6 @@ public class IndexedHierarchicDistributionTest { "</tuning>"); } - private String getRandomDispatchXml() { - return joinLines("<tuning>", - " <dispatch>", - " <dispatch-policy>random</dispatch-policy>", - " </dispatch>", - "</tuning>"); - } - - private ContentCluster getOddGroupsCluster() throws Exception { - String groupXml = joinLines(" <group>", - " <distribution partitions='2|*'/>", - " <group distribution-key='0' name='group0'>", - " <node distribution-key='0' hostalias='mockhost'/>", - " <node distribution-key='1' hostalias='mockhost'/>", - " </group>", - " <group distribution-key='1' name='group1'>", - " <node distribution-key='3' hostalias='mockhost'/>", - " <node distribution-key='4' hostalias='mockhost'/>", - " <node distribution-key='5' hostalias='mockhost'/>", - " </group>", - " </group>", ""); - return createCluster(createClusterXml(groupXml, Optional.of(getRandomDispatchXml()), 4, 4)); - } - @Test void requireThatWeMustHaveOnlyOneGroupLevel() { try { @@ -193,30 +166,6 @@ public class IndexedHierarchicDistributionTest { } @Test - void requireThatLeafGroupsCanHaveUnequalNumberOfNodesIfRandomPolicy() throws Exception { - ContentCluster c = getOddGroupsCluster(); - DispatchGroup dg = c.getSearch().getIndexed().getRootDispatch(); - assertEquals(8, dg.getRowBits()); - assertEquals(3, dg.getNumPartitions()); - assertTrue(dg.useFixedRowInDispatch()); - ArrayList<SearchInterface> list = new ArrayList<>(); - for (SearchInterface si : dg.getSearchersIterable()) { - list.add(si); - } - assertEquals(5, list.size()); - assertEquals(0, list.get(0).getNodeSpec().partitionId()); - assertEquals(0, list.get(0).getNodeSpec().groupIndex()); - assertEquals(0, list.get(1).getNodeSpec().partitionId()); - assertEquals(1, list.get(1).getNodeSpec().groupIndex()); - assertEquals(1, list.get(2).getNodeSpec().partitionId()); - assertEquals(0, list.get(2).getNodeSpec().groupIndex()); - assertEquals(1, list.get(3).getNodeSpec().partitionId()); - assertEquals(1, list.get(3).getNodeSpec().groupIndex()); - assertEquals(2, list.get(4).getNodeSpec().partitionId()); - assertEquals(1, list.get(4).getNodeSpec().groupIndex()); - } - - @Test void requireThatLeafGroupsCountMustBeAFactorOfRedundancy() { try { getTwoGroupsCluster(3, 3, "2|*"); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java index a6ea6cb8132..0800f26d6e8 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java @@ -72,7 +72,7 @@ public class ClusterTest { "", joinLines( "<max-hits-per-partition>77</max-hits-per-partition>", - "<dispatch-policy>round-robin</dispatch-policy>", + "<dispatch-policy>best-of-random-2</dispatch-policy>", "<min-active-docs-coverage>93</min-active-docs-coverage>", "<top-k-probability>0.777</top-k-probability>"), false); @@ -81,7 +81,7 @@ public class ClusterTest { DispatchConfig config = new DispatchConfig(builder); assertEquals(3, config.redundancy()); assertEquals(93.0, config.minActivedocsPercentage(), DELTA); - assertEquals(DispatchConfig.DistributionPolicy.ROUNDROBIN, config.distributionPolicy()); + assertEquals(DispatchConfig.DistributionPolicy.BEST_OF_RANDOM_2, config.distributionPolicy()); assertEquals(77, config.maxHitsPerNode()); assertEquals(0.777, config.topKProbability(), DELTA); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchNodeTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchNodeTest.java index b357a3e6718..191fc9e63bb 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchNodeTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchNodeTest.java @@ -50,7 +50,7 @@ public class SearchNodeTest { private static SearchNode createSearchNode(MockRoot root, String name, int distributionKey, NodeSpec nodeSpec, boolean flushOnShutDown, boolean isHosted, ModelContext.FeatureFlags featureFlags) { return SearchNode.create(root, name, distributionKey, nodeSpec, "mycluster", null, flushOnShutDown, - Optional.empty(), Optional.empty(), isHosted, 0.0, new Redundancy(1,1,1,1,1), featureFlags); + Optional.empty(), Optional.empty(), isHosted, 0.0, () -> new Redundancy(1,1,1,1,1), featureFlags); } private static SearchNode createSearchNode(MockRoot root) { diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/SystemName.java b/config-provisioning/src/main/java/com/yahoo/config/provision/SystemName.java index e2349f6f63f..c6eecf1e705 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/SystemName.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/SystemName.java @@ -75,4 +75,6 @@ public enum SystemName { return Stream.of(values()).filter(predicate).collect(Collectors.toUnmodifiableSet()); } + public static Set<SystemName> hostedVespa() { return EnumSet.of(main, cd, Public, PublicCd); } + } diff --git a/configdefinitions/src/vespa/orchestrator.def b/configdefinitions/src/vespa/orchestrator.def index cba221e445f..459e8502236 100644 --- a/configdefinitions/src/vespa/orchestrator.def +++ b/configdefinitions/src/vespa/orchestrator.def @@ -6,3 +6,11 @@ serviceMonitorConvergenceLatencySeconds int default=0 # The number of services in the routing/container cluster of hosted-vespa.routing.default. numProxies int default=0 + +# The number of of the services that are allowed to be down for the routing container cluster +# in the hosted-vespa.routing.default infrastructure application. +numProxiesAllowedDown int default=1 + +# See numProxiesAllowedDown for background. Allow up to this ratio of services to be down, +# unless already allowed by numProxiesAllowedDown. +numProxiesAllowedDownRatio double default=0.1 diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index e35520d5381..d815ea3328a 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -167,9 +167,6 @@ public class ModelContextImpl implements ModelContext { private final String feedSequencer; private final String responseSequencer; private final int numResponseThreads; - private final boolean skipCommunicationManagerThread; - private final boolean skipMbusRequestThread; - private final boolean skipMbusReplyThread; private final boolean useAsyncMessageHandlingOnSchedule; private final double feedConcurrency; private final double feedNiceness; @@ -184,11 +181,9 @@ public class ModelContextImpl implements ModelContext { private final double containerShutdownTimeout; private final int maxUnCommittedMemory; private final boolean forwardIssuesAsErrors; - private final boolean ignoreThreadStackSizes; private final boolean useV8GeoPositions; private final int maxCompactBuffers; private final List<String> ignoredHttpUserAgents; - private final boolean avoidRenamingSummaryFeatures; private final Architecture adminClusterArchitecture; private final boolean enableProxyProtocolMixedMode; private final boolean sharedStringRepoNoReclaim; @@ -205,7 +200,6 @@ public class ModelContextImpl implements ModelContext { private final boolean enableGlobalPhase; private final String summaryDecodePolicy; private final Predicate<ClusterSpec.Id> allowMoreThanOneContentGroupDown; - private final boolean enableConditionalPutRemoveWriteRepair; private final boolean enableDataplaneProxy; private final boolean enableNestedMultivalueGrouping; private final boolean useReconfigurableDispatcher; @@ -215,9 +209,6 @@ public class ModelContextImpl implements ModelContext { this.feedSequencer = flagValue(source, appId, version, Flags.FEED_SEQUENCER_TYPE); this.responseSequencer = flagValue(source, appId, version, Flags.RESPONSE_SEQUENCER_TYPE); this.numResponseThreads = flagValue(source, appId, version, Flags.RESPONSE_NUM_THREADS); - this.skipCommunicationManagerThread = flagValue(source, appId, version, Flags.SKIP_COMMUNICATIONMANAGER_THREAD); - this.skipMbusRequestThread = flagValue(source, appId, version, Flags.SKIP_MBUS_REQUEST_THREAD); - this.skipMbusReplyThread = flagValue(source, appId, version, Flags.SKIP_MBUS_REPLY_THREAD); this.useAsyncMessageHandlingOnSchedule = flagValue(source, appId, version, Flags.USE_ASYNC_MESSAGE_HANDLING_ON_SCHEDULE); this.feedConcurrency = flagValue(source, appId, version, Flags.FEED_CONCURRENCY); this.feedNiceness = flagValue(source, appId, version, Flags.FEED_NICENESS); @@ -233,11 +224,9 @@ public class ModelContextImpl implements ModelContext { this.containerShutdownTimeout = flagValue(source, appId, version, Flags.CONTAINER_SHUTDOWN_TIMEOUT); this.maxUnCommittedMemory = flagValue(source, appId, version, Flags.MAX_UNCOMMITTED_MEMORY); this.forwardIssuesAsErrors = flagValue(source, appId, version, PermanentFlags.FORWARD_ISSUES_AS_ERRORS); - this.ignoreThreadStackSizes = flagValue(source, appId, version, Flags.IGNORE_THREAD_STACK_SIZES); this.useV8GeoPositions = flagValue(source, appId, version, Flags.USE_V8_GEO_POSITIONS); this.maxCompactBuffers = flagValue(source, appId, version, Flags.MAX_COMPACT_BUFFERS); this.ignoredHttpUserAgents = flagValue(source, appId, version, PermanentFlags.IGNORED_HTTP_USER_AGENTS); - this.avoidRenamingSummaryFeatures = flagValue(source, appId, version, Flags.AVOID_RENAMING_SUMMARY_FEATURES); this.adminClusterArchitecture = Architecture.valueOf(flagValue(source, appId, version, PermanentFlags.ADMIN_CLUSTER_NODE_ARCHITECTURE)); this.enableProxyProtocolMixedMode = flagValue(source, appId, version, Flags.ENABLE_PROXY_PROTOCOL_MIXED_MODE); this.sharedStringRepoNoReclaim = flagValue(source, appId, version, Flags.SHARED_STRING_REPO_NO_RECLAIM); @@ -255,7 +244,6 @@ public class ModelContextImpl implements ModelContext { this.enableGlobalPhase = flagValue(source, appId, version, Flags.ENABLE_GLOBAL_PHASE); this.summaryDecodePolicy = flagValue(source, appId, version, Flags.SUMMARY_DECODE_POLICY); this.allowMoreThanOneContentGroupDown = clusterId -> flagValue(source, appId, version, clusterId, Flags.ALLOW_MORE_THAN_ONE_CONTENT_GROUP_DOWN); - this.enableConditionalPutRemoveWriteRepair = flagValue(source, appId, version, Flags.ENABLE_CONDITIONAL_PUT_REMOVE_WRITE_REPAIR); this.enableDataplaneProxy = flagValue(source, appId, version, Flags.ENABLE_DATAPLANE_PROXY); this.enableNestedMultivalueGrouping = flagValue(source, appId, version, Flags.ENABLE_NESTED_MULTIVALUE_GROUPING); this.useReconfigurableDispatcher = flagValue(source, appId, version, Flags.USE_RECONFIGURABLE_DISPATCHER); @@ -269,9 +257,6 @@ public class ModelContextImpl implements ModelContext { @Override public String feedSequencerType() { return feedSequencer; } @Override public String responseSequencerType() { return responseSequencer; } @Override public int defaultNumResponseThreads() { return numResponseThreads; } - @Override public boolean skipCommunicationManagerThread() { return skipCommunicationManagerThread; } - @Override public boolean skipMbusRequestThread() { return skipMbusRequestThread; } - @Override public boolean skipMbusReplyThread() { return skipMbusReplyThread; } @Override public boolean useAsyncMessageHandlingOnSchedule() { return useAsyncMessageHandlingOnSchedule; } @Override public double feedConcurrency() { return feedConcurrency; } @Override public double feedNiceness() { return feedNiceness; } @@ -289,11 +274,9 @@ public class ModelContextImpl implements ModelContext { @Override public boolean loadCodeAsHugePages() { return loadCodeAsHugePages; } @Override public int maxUnCommittedMemory() { return maxUnCommittedMemory; } @Override public boolean forwardIssuesAsErrors() { return forwardIssuesAsErrors; } - @Override public boolean ignoreThreadStackSizes() { return ignoreThreadStackSizes; } @Override public boolean useV8GeoPositions() { return useV8GeoPositions; } @Override public int maxCompactBuffers() { return maxCompactBuffers; } @Override public List<String> ignoredHttpUserAgents() { return ignoredHttpUserAgents; } - @Override public boolean avoidRenamingSummaryFeatures() { return avoidRenamingSummaryFeatures; } @Override public Architecture adminClusterArchitecture() { return adminClusterArchitecture; } @Override public boolean enableProxyProtocolMixedMode() { return enableProxyProtocolMixedMode; } @Override public boolean sharedStringRepoNoReclaim() { return sharedStringRepoNoReclaim; } @@ -313,7 +296,6 @@ public class ModelContextImpl implements ModelContext { @Override public boolean useRestrictedDataPlaneBindings() { return useRestrictedDataPlaneBindings; } @Override public boolean enableGlobalPhase() { return enableGlobalPhase; } @Override public boolean allowMoreThanOneContentGroupDown(ClusterSpec.Id id) { return allowMoreThanOneContentGroupDown.test(id); } - @Override public boolean enableConditionalPutRemoveWriteRepair() { return enableConditionalPutRemoveWriteRepair; } @Override public boolean enableDataplaneProxy() { return enableDataplaneProxy; } @Override public boolean enableNestedMultivalueGrouping() { return enableNestedMultivalueGrouping; } @Override public boolean useReconfigurableDispatcher() { return useReconfigurableDispatcher; } diff --git a/configserver/src/main/resources/configserver-app/services.xml b/configserver/src/main/resources/configserver-app/services.xml index b6904467893..02481291213 100644 --- a/configserver/src/main/resources/configserver-app/services.xml +++ b/configserver/src/main/resources/configserver-app/services.xml @@ -141,7 +141,12 @@ </handler> <http> - <server port="19071" id="configserver" /> + <server port="19071" id="configserver"> + <config name="jdisc.http.connector"> + <!-- Limit max request content size accepted --> + <maxContentSize>2000000000</maxContentSize> <!-- ~2GB --> + </config> + </server> <preprocess:include file='http-server.xml' required='false' /> </http> diff --git a/container-core/abi-spec.json b/container-core/abi-spec.json index 6d7e3c86351..2b5e4386e94 100644 --- a/container-core/abi-spec.json +++ b/container-core/abi-spec.json @@ -3280,27 +3280,27 @@ "public final boolean getBoolean(java.lang.String)", "public final boolean getBoolean(com.yahoo.processing.request.CompoundName, boolean)", "public final boolean getBoolean(java.lang.String, boolean)", - "protected final boolean asBoolean(java.lang.Object, boolean)", + "protected static boolean asBoolean(java.lang.Object, boolean)", "public final java.lang.String getString(com.yahoo.processing.request.CompoundName)", "public final java.lang.String getString(java.lang.String)", "public final java.lang.String getString(com.yahoo.processing.request.CompoundName, java.lang.String)", "public final java.lang.String getString(java.lang.String, java.lang.String)", - "protected final java.lang.String asString(java.lang.Object, java.lang.String)", + "protected static java.lang.String asString(java.lang.Object, java.lang.String)", "public final java.lang.Integer getInteger(com.yahoo.processing.request.CompoundName)", "public final java.lang.Integer getInteger(java.lang.String)", "public final java.lang.Integer getInteger(com.yahoo.processing.request.CompoundName, java.lang.Integer)", "public final java.lang.Integer getInteger(java.lang.String, java.lang.Integer)", - "protected final java.lang.Integer asInteger(java.lang.Object, java.lang.Integer)", + "protected static java.lang.Integer asInteger(java.lang.Object, java.lang.Integer)", "public final java.lang.Long getLong(com.yahoo.processing.request.CompoundName)", "public final java.lang.Long getLong(java.lang.String)", "public final java.lang.Long getLong(com.yahoo.processing.request.CompoundName, java.lang.Long)", "public final java.lang.Long getLong(java.lang.String, java.lang.Long)", - "protected final java.lang.Long asLong(java.lang.Object, java.lang.Long)", + "protected static java.lang.Long asLong(java.lang.Object, java.lang.Long)", "public final java.lang.Double getDouble(com.yahoo.processing.request.CompoundName)", "public final java.lang.Double getDouble(java.lang.String)", "public final java.lang.Double getDouble(com.yahoo.processing.request.CompoundName, java.lang.Double)", "public final java.lang.Double getDouble(java.lang.String, java.lang.Double)", - "protected final java.lang.Double asDouble(java.lang.Object, java.lang.Double)", + "protected static java.lang.Double asDouble(java.lang.Object, java.lang.Double)", "public com.yahoo.processing.request.Properties clone()", "public static java.util.Map cloneMap(java.util.Map)", "public static java.lang.Object clone(java.lang.Object)", diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/state/MetricsPacketsHandler.java b/container-core/src/main/java/com/yahoo/container/jdisc/state/MetricsPacketsHandler.java index e06c7c8aa32..b8beeb5c1d1 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/state/MetricsPacketsHandler.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/state/MetricsPacketsHandler.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.jdisc.state; +import ai.vespa.metrics.set.InfrastructureMetricSet; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; @@ -22,11 +23,15 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import static com.yahoo.container.jdisc.state.JsonUtil.sanitizeDouble; import static com.yahoo.container.jdisc.state.StateHandler.getSnapshotProviderOrThrow; @@ -60,6 +65,7 @@ public class MetricsPacketsHandler extends AbstractRequestHandler { private final SnapshotProvider snapshotProvider; private final String applicationName; private final String hostDimension; + private final Map<String, Set<String>> metricSets; @Inject public MetricsPacketsHandler(Timer timer, @@ -69,6 +75,7 @@ public class MetricsPacketsHandler extends AbstractRequestHandler { snapshotProvider = getSnapshotProviderOrThrow(snapshotProviders); applicationName = config.application(); hostDimension = config.hostname(); + metricSets = getMetricSets(); } @@ -93,14 +100,19 @@ public class MetricsPacketsHandler extends AbstractRequestHandler { private byte[] buildMetricOutput(String query) { try { - if (query != null && query.equals("array-formatted")) { - return getMetricsArray(); + var queryMap = parseQuery(query); + var metricSetId = queryMap.get("metric-set"); + var format = queryMap.get("format"); + + // TODO: Remove "array-formatted" + if ("array".equals(format) || queryMap.containsKey("array-formatted")) { + return getMetricsArray(metricSetId); } - if ("format=prometheus".equals(query)) { + if ("prometheus".equals(format)) { return buildPrometheusOutput(); } - String output = getAllMetricsPackets() + "\n"; + String output = getAllMetricsPackets(metricSetId) + "\n"; return output.getBytes(StandardCharsets.UTF_8); } catch (JsonProcessingException e) { throw new RuntimeException("Bad JSON construction.", e); @@ -109,10 +121,10 @@ public class MetricsPacketsHandler extends AbstractRequestHandler { } } - private byte[] getMetricsArray() throws JsonProcessingException { + private byte[] getMetricsArray(String metricSetId) throws JsonProcessingException { ObjectNode root = jsonMapper.createObjectNode(); ArrayNode jsonArray = jsonMapper.createArrayNode(); - getPacketsForSnapshot(getSnapshot(), applicationName, timer.currentTimeMillis()) + getPacketsForSnapshot(getSnapshot(), metricSetId, applicationName, timer.currentTimeMillis()) .forEach(jsonArray::add); MetricGatherer.getAdditionalMetrics().forEach(jsonArray::add); root.set("metrics", jsonArray); @@ -132,9 +144,9 @@ public class MetricsPacketsHandler extends AbstractRequestHandler { .writeValueAsString(jsonObject); } - private String getAllMetricsPackets() throws JsonProcessingException { + private String getAllMetricsPackets(String metricSetId) throws JsonProcessingException { StringBuilder ret = new StringBuilder(); - List<JsonNode> metricsPackets = getPacketsForSnapshot(getSnapshot(), applicationName, timer.currentTimeMillis()); + List<JsonNode> metricsPackets = getPacketsForSnapshot(getSnapshot(), metricSetId, applicationName, timer.currentTimeMillis()); String delimiter = ""; for (JsonNode packet : metricsPackets) { ret.append(delimiter); // For legibility and parsing in unit tests @@ -166,6 +178,29 @@ public class MetricsPacketsHandler extends AbstractRequestHandler { return packets; } + private List<JsonNode> getPacketsForSnapshot(MetricSnapshot metricSnapshot, String metricSetId, String application, long timestamp) { + if (metricSnapshot == null) return Collections.emptyList(); + if (metricSetId == null) return getPacketsForSnapshot(metricSnapshot, application, timestamp); + Set<String> configuredMetrics = metricSets.getOrDefault(metricSetId, Collections.emptySet()); + List<JsonNode> packets = new ArrayList<>(); + + for (Map.Entry<MetricDimensions, MetricSet> snapshotEntry : metricSnapshot) { + MetricDimensions metricDimensions = snapshotEntry.getKey(); + MetricSet metricSet = snapshotEntry.getValue(); + + ObjectNode packet = jsonMapper.createObjectNode(); + addMetaData(timestamp, application, packet); + addDimensions(metricDimensions, packet); + var metrics = getMetrics(metricSet); + metrics.keySet().retainAll(configuredMetrics); + if (!metrics.isEmpty()) { + addMetrics(metrics, packet); + packets.add(packet); + } + } + return packets; + } + private void addMetaData(long timestamp, String application, ObjectNode packet) { packet.put(APPLICATION_KEY, application); packet.put(TIMESTAMP_KEY, TimeUnit.MILLISECONDS.toSeconds(timestamp)); @@ -196,7 +231,10 @@ public class MetricsPacketsHandler extends AbstractRequestHandler { GaugeMetric gauge = (GaugeMetric) value; metrics.put(name + ".average", sanitizeDouble(gauge.getAverage())) .put(name + ".last", sanitizeDouble(gauge.getLast())) - .put(name + ".max", sanitizeDouble(gauge.getMax())); + .put(name + ".max", sanitizeDouble(gauge.getMax())) + .put(name + ".min", sanitizeDouble(gauge.getMin())) + .put(name + ".sum", sanitizeDouble(gauge.getSum())) + .put(name + ".count", gauge.getCount()); if (gauge.getPercentiles().isPresent()) { for (Tuple2<String, Double> prefixAndValue : gauge.getPercentiles().get()) { metrics.put(name + "." + prefixAndValue.first + "percentile", prefixAndValue.second.doubleValue()); @@ -208,6 +246,42 @@ public class MetricsPacketsHandler extends AbstractRequestHandler { } } + private Map<String, Number> getMetrics(MetricSet metricSet) { + var metrics = new HashMap<String, Number>(); + for (Map.Entry<String, MetricValue> metric : metricSet) { + String name = metric.getKey(); + MetricValue value = metric.getValue(); + if (value instanceof CountMetric) { + metrics.put(name + ".count", ((CountMetric) value).getCount()); + } else if (value instanceof GaugeMetric) { + GaugeMetric gauge = (GaugeMetric) value; + metrics.put(name + ".average", sanitizeDouble(gauge.getAverage())); + metrics.put(name + ".last", sanitizeDouble(gauge.getLast())); + metrics.put(name + ".max", sanitizeDouble(gauge.getMax())); + metrics.put(name + ".min", sanitizeDouble(gauge.getMin())); + metrics.put(name + ".sum", sanitizeDouble(gauge.getSum())); + metrics.put(name + ".count", gauge.getCount()); + if (gauge.getPercentiles().isPresent()) { + for (Tuple2<String, Double> prefixAndValue : gauge.getPercentiles().get()) { + metrics.put(name + "." + prefixAndValue.first + "percentile", prefixAndValue.second.doubleValue()); + } + } + } else { + throw new UnsupportedOperationException("Unknown metric class: " + value.getClass().getName()); + } + } + return metrics; + } + + private void addMetrics(Map<String, Number> metrics, ObjectNode packet) { + ObjectNode metricsObject = jsonMapper.createObjectNode(); + packet.set(METRICS_KEY, metricsObject); + metrics.forEach((name, value) -> { + if (value instanceof Double) metricsObject.put(name, (Double) value); + else metricsObject.put(name, (Long) value); + }); + } + private String getContentType(String query) { if ("format=prometheus".equals(query)) { return "text/plain;charset=utf-8"; @@ -215,4 +289,17 @@ public class MetricsPacketsHandler extends AbstractRequestHandler { return "application/json"; } + private Map<String, String> parseQuery(String query) { + if (query == null) return Map.of(); + return Arrays.stream(query.split("&")) + .map(s -> s.split("=")) + .collect(Collectors.toMap(s -> s[0], s -> s.length < 2 ? "" : s[1])); + } + + private Map<String, Set<String>> getMetricSets() { + // For now - single infrastructure metric set + return Map.of( + InfrastructureMetricSet.infrastructureMetricSet.getId(), InfrastructureMetricSet.infrastructureMetricSet.getMetrics().keySet() + ); + } } diff --git a/container-core/src/main/java/com/yahoo/processing/request/CompoundName.java b/container-core/src/main/java/com/yahoo/processing/request/CompoundName.java index 0edff9162b5..2a346305e2f 100644 --- a/container-core/src/main/java/com/yahoo/processing/request/CompoundName.java +++ b/container-core/src/main/java/com/yahoo/processing/request/CompoundName.java @@ -63,7 +63,7 @@ public final class CompoundName { } private CompoundName(String [] compounds) { - this(toCompoundString(compounds), compounds, false); + this(toCompoundString(compounds), compounds, true); } /** diff --git a/container-core/src/main/java/com/yahoo/processing/request/Properties.java b/container-core/src/main/java/com/yahoo/processing/request/Properties.java index ac43f99472e..2880b734e4e 100644 --- a/container-core/src/main/java/com/yahoo/processing/request/Properties.java +++ b/container-core/src/main/java/com/yahoo/processing/request/Properties.java @@ -316,7 +316,7 @@ public class Properties implements Cloneable { * Converts a value to boolean - this will be true only if the value is either the empty string, * or any Object which has a toString which is case-insensitive equal to "true" */ - protected final boolean asBoolean(Object value, boolean defaultValue) { + protected static boolean asBoolean(Object value, boolean defaultValue) { if (value == null) return defaultValue; String s = value.toString(); @@ -371,7 +371,7 @@ public class Properties implements Cloneable { return asString(get(key), defaultValue); } - protected final String asString(Object value, String defaultValue) { + protected static String asString(Object value, String defaultValue) { if (value == null) return defaultValue; return value.toString(); } @@ -424,7 +424,7 @@ public class Properties implements Cloneable { return asInteger(get(name), defaultValue); } - protected final Integer asInteger(Object value, Integer defaultValue) { + protected static Integer asInteger(Object value, Integer defaultValue) { try { if (value == null) return defaultValue; @@ -490,7 +490,7 @@ public class Properties implements Cloneable { return asLong(get(name), defaultValue); } - protected final Long asLong(Object value, Long defaultValue) { + protected static Long asLong(Object value, Long defaultValue) { try { if (value == null) return defaultValue; @@ -556,7 +556,7 @@ public class Properties implements Cloneable { return asDouble(get(name), defaultValue); } - protected final Double asDouble(Object value, Double defaultValue) { + protected static Double asDouble(Object value, Double defaultValue) { try { if (value == null) return defaultValue; diff --git a/container-core/src/test/java/com/yahoo/container/jdisc/state/MetricsPacketsHandlerTest.java b/container-core/src/test/java/com/yahoo/container/jdisc/state/MetricsPacketsHandlerTest.java index 3f5c31e5e7f..1aa4ee93ab6 100644 --- a/container-core/src/test/java/com/yahoo/container/jdisc/state/MetricsPacketsHandlerTest.java +++ b/container-core/src/test/java/com/yahoo/container/jdisc/state/MetricsPacketsHandlerTest.java @@ -163,6 +163,67 @@ public class MetricsPacketsHandlerTest extends StateHandlerTestBase { """; assertEquals(expectedResponse, response); } + + @Test + public void test_metric_filtering() { + var context = StateMetricContext.newInstance(Map.of("dim-1", "value1")); + var snapshot = new MetricSnapshot(); + snapshot.set(context, "gauge.metric", 0.2); + snapshot.add(context, "counter.metric", 5); + snapshot.add(context, "configserver.requests", 120); + // Infrastructure set only contains max and average + snapshot.set(context, "lockAttempt.lockedLoad", 500); + + // Without filtering + snapshotProvider.setSnapshot(snapshot); + var response = requestAsString("http://localhost/metrics-packets"); + var expectedResponse = """ + { + "application" : "state-handler-test-base", + "timestamp" : 0, + "dimensions" : { + "dim-1" : "value1", + "host" : "some-hostname" + }, + "metrics" : { + "gauge.metric.average" : 0.2, + "gauge.metric.last" : 0.2, + "gauge.metric.max" : 0.2, + "gauge.metric.min" : 0.2, + "gauge.metric.sum" : 0.2, + "gauge.metric.count" : 1, + "configserver.requests.count" : 120, + "lockAttempt.lockedLoad.average" : 500.0, + "lockAttempt.lockedLoad.last" : 500.0, + "lockAttempt.lockedLoad.max" : 500.0, + "lockAttempt.lockedLoad.min" : 500.0, + "lockAttempt.lockedLoad.sum" : 500.0, + "lockAttempt.lockedLoad.count" : 1, + "counter.metric.count" : 5 + } + } + """; + assertEquals(expectedResponse, response); + + // With filtering + response = requestAsString("http://localhost/metrics-packets?metric-set=infrastructure"); + expectedResponse = """ + { + "application" : "state-handler-test-base", + "timestamp" : 0, + "dimensions" : { + "dim-1" : "value1", + "host" : "some-hostname" + }, + "metrics" : { + "configserver.requests.count" : 120, + "lockAttempt.lockedLoad.average" : 500.0, + "lockAttempt.lockedLoad.max" : 500.0 + } + } + """; + assertEquals(expectedResponse, response); + } private List<JsonNode> incrementTimeAndGetJsonPackets() throws Exception { advanceToNextSnapshot(); diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java index f73ed52246c..7b9fe7da7a2 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java @@ -108,7 +108,7 @@ public class QueryProfileProperties extends Properties { /** * Sets a value in this query profile * - * @throws IllegalArgumentException if this property cannot be set in the wrapped query profile + * @throws IllegalInputException if this property cannot be set in the wrapped query profile */ @Override public void set(CompoundName name, Object value, Map<String, String> context) { diff --git a/container-search/src/main/java/com/yahoo/search/query/profiling/Profiling.java b/container-search/src/main/java/com/yahoo/search/query/profiling/Profiling.java index 8541b9c3930..1dbda8143a0 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profiling/Profiling.java +++ b/container-search/src/main/java/com/yahoo/search/query/profiling/Profiling.java @@ -33,9 +33,9 @@ public class Profiling implements Cloneable { public static QueryProfileType getArgumentType() { return argumentType; } - private ProfilingParams matching = new ProfilingParams(); - private ProfilingParams firstPhaseRanking = new ProfilingParams(); - private ProfilingParams secondPhaseRanking = new ProfilingParams(); + private final ProfilingParams matching = new ProfilingParams(); + private final ProfilingParams firstPhaseRanking = new ProfilingParams(); + private final ProfilingParams secondPhaseRanking = new ProfilingParams(); public ProfilingParams getMatching() { return matching; diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java index 240da3f123f..800b3a1ba89 100644 --- a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java +++ b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java @@ -24,7 +24,9 @@ import com.yahoo.search.query.ranking.Matching; import com.yahoo.search.query.ranking.SoftTimeout; import com.yahoo.tensor.Tensor; +import java.util.HashMap; import java.util.Map; +import java.util.Set; /** * Maps between the query model and text properties. @@ -35,10 +37,101 @@ import java.util.Map; */ public class QueryProperties extends Properties { + interface GetProperty { + Object get(Query query); + } + interface SetProperty { + void set(Query query, Object value); + } + private Query query; private final CompiledQueryProfileRegistry profileRegistry; private final Map<String, Embedder> embedders; + private static final Set<String> reservedPrefix = Set.of(Model.MODEL, Presentation.PRESENTATION, Select.SELECT, Ranking.RANKING, Trace.TRACE); + + + private record GetterSetter(GetProperty getter, SetProperty setter) { + static GetterSetter of(GetProperty getter, SetProperty setter) { + return new GetterSetter(getter, setter); + } + } + + private static void addDualCasedRM(Map<CompoundName, GetterSetter> map, String last, GetterSetter accessor) { + map.put(CompoundName.fromComponents(Ranking.RANKING, Ranking.MATCHING, last), accessor); + map.put(CompoundName.fromComponents(Ranking.RANKING, Ranking.MATCHING, last.toLowerCase()), accessor); + } + + private static final Map<CompoundName, GetterSetter> properyAccessors = createPropertySetterMap(); + private static Map<CompoundName, GetterSetter> createPropertySetterMap() { + Map<CompoundName, GetterSetter> map = new HashMap<>(); + map.put(CompoundName.fromComponents(Model.MODEL, Model.QUERY_STRING), GetterSetter.of(query -> query.getModel().getQueryString(), (query, value) -> query.getModel().setQueryString(asString(value, "")))); + map.put(CompoundName.fromComponents(Model.MODEL, Model.TYPE), GetterSetter.of(query -> query.getModel().getType(), (query, value) -> query.getModel().setType(asString(value, "ANY")))); + map.put(CompoundName.fromComponents(Model.MODEL, Model.FILTER), GetterSetter.of(query -> query.getModel().getFilter(), (query, value) -> query.getModel().setFilter(asString(value, "")))); + map.put(CompoundName.fromComponents(Model.MODEL, Model.DEFAULT_INDEX), GetterSetter.of(query -> query.getModel().getDefaultIndex(), (query, value) -> query.getModel().setDefaultIndex(asString(value, "")))); + map.put(CompoundName.fromComponents(Model.MODEL, Model.LANGUAGE), GetterSetter.of(query -> query.getModel().getLanguage(), (query, value) -> query.getModel().setLanguage(asString(value, "")))); + map.put(CompoundName.fromComponents(Model.MODEL, Model.LOCALE), GetterSetter.of(query -> query.getModel().getLocale(), (query, value) -> query.getModel().setLocale(asString(value, "")))); + map.put(CompoundName.fromComponents(Model.MODEL, Model.ENCODING), GetterSetter.of(query -> query.getModel().getEncoding(), (query, value) -> query.getModel().setEncoding(asString(value,"")))); + map.put(CompoundName.fromComponents(Model.MODEL, Model.SOURCES), GetterSetter.of(query -> query.getModel().getSources(), (query, value) -> query.getModel().setSources(asString(value,"")))); + map.put(CompoundName.fromComponents(Model.MODEL, Model.SEARCH_PATH), GetterSetter.of(query -> query.getModel().getSearchPath(), (query, value) -> query.getModel().setSearchPath(asString(value,"")))); + map.put(CompoundName.fromComponents(Model.MODEL, Model.RESTRICT), GetterSetter.of(query -> query.getModel().getRestrict(), (query, value) -> query.getModel().setRestrict(asString(value,"")))); + map.put(CompoundName.fromComponents(Ranking.RANKING, Ranking.LOCATION), GetterSetter.of(query -> query.getRanking().getLocation(), (query, value) -> query.getRanking().setLocation(asString(value,"")))); + map.put(CompoundName.fromComponents(Ranking.RANKING, Ranking.PROFILE), GetterSetter.of(query -> query.getRanking().getProfile(), (query, value) -> query.getRanking().setProfile(asString(value,"")))); + map.put(CompoundName.fromComponents(Ranking.RANKING, Ranking.SORTING), GetterSetter.of(query -> query.getRanking().getSorting(), (query, value) -> query.getRanking().setSorting(asString(value,"")))); + map.put(CompoundName.fromComponents(Ranking.RANKING, Ranking.FRESHNESS), GetterSetter.of(query -> query.getRanking().getFreshness(), (query, value) -> query.getRanking().setFreshness(asString(value, "")))); + map.put(CompoundName.fromComponents(Ranking.RANKING, Ranking.QUERYCACHE), GetterSetter.of(query -> query.getRanking().getQueryCache(), (query, value) -> query.getRanking().setQueryCache(asBoolean(value, false)))); + map.put(CompoundName.fromComponents(Ranking.RANKING, Ranking.RERANKCOUNT), GetterSetter.of(query -> query.getRanking().getRerankCount(), (query, value) -> query.getRanking().setRerankCount(asInteger(value, null)))); + map.put(CompoundName.fromComponents(Ranking.RANKING, Ranking.KEEPRANKCOUNT), GetterSetter.of(query -> query.getRanking().getKeepRankCount(), (query, value) -> query.getRanking().setKeepRankCount(asInteger(value, null)))); + map.put(CompoundName.fromComponents(Ranking.RANKING, Ranking.RANKSCOREDROPLIMIT), GetterSetter.of(query -> query.getRanking().getRankScoreDropLimit(), (query, value) -> query.getRanking().setRankScoreDropLimit(asDouble(value, null)))); + map.put(CompoundName.fromComponents(Ranking.RANKING, Ranking.LIST_FEATURES), GetterSetter.of(query -> query.getRanking().getListFeatures(), (query, value) -> query.getRanking().setListFeatures(asBoolean(value,false)))); + + addDualCasedRM(map, Matching.TERMWISELIMIT, GetterSetter.of(query -> query.getRanking().getMatching().getTermwiseLimit(), (query, value) -> query.getRanking().getMatching().setTermwiselimit(asDouble(value, 1.0)))); + addDualCasedRM(map, Matching.NUMTHREADSPERSEARCH, GetterSetter.of(query -> query.getRanking().getMatching().getNumThreadsPerSearch(), (query, value) -> query.getRanking().getMatching().setNumThreadsPerSearch(asInteger(value, 1)))); + addDualCasedRM(map, Matching.NUMSEARCHPARTITIIONS, GetterSetter.of(query -> query.getRanking().getMatching().getNumSearchPartitions(), (query, value) -> query.getRanking().getMatching().setNumSearchPartitions(asInteger(value, 1)))); + addDualCasedRM(map, Matching.MINHITSPERTHREAD, GetterSetter.of(query -> query.getRanking().getMatching().getMinHitsPerThread(), (query, value) -> query.getRanking().getMatching().setMinHitsPerThread(asInteger(value, 0)))); + addDualCasedRM(map, Matching.POST_FILTER_THRESHOLD, GetterSetter.of(query -> query.getRanking().getMatching().getPostFilterThreshold(), (query, value) -> query.getRanking().getMatching().setPostFilterThreshold(asDouble(value, 1.0)))); + addDualCasedRM(map, Matching.APPROXIMATE_THRESHOLD, GetterSetter.of(query -> query.getRanking().getMatching().getApproximateThreshold(), (query, value) -> query.getRanking().getMatching().setApproximateThreshold(asDouble(value, 0.05)))); + + map.put(CompoundName.fromComponents(Ranking.RANKING, Ranking.MATCH_PHASE, MatchPhase.ATTRIBUTE), GetterSetter.of(query -> query.getRanking().getMatchPhase().getAttribute(), (query, value) -> query.getRanking().getMatchPhase().setAttribute(asString(value, null)))); + map.put(CompoundName.fromComponents(Ranking.RANKING, Ranking.MATCH_PHASE, MatchPhase.ASCENDING), GetterSetter.of(query -> query.getRanking().getMatchPhase().getAscending(), (query, value) -> query.getRanking().getMatchPhase().setAscending(asBoolean(value, false)))); + map.put(CompoundName.fromComponents(Ranking.RANKING, Ranking.MATCH_PHASE, MatchPhase.MAX_HITS), GetterSetter.of(query -> query.getRanking().getMatchPhase().getMaxHits(), (query, value) -> query.getRanking().getMatchPhase().setMaxHits(asLong(value, null)))); + map.put(CompoundName.fromComponents(Ranking.RANKING, Ranking.MATCH_PHASE, MatchPhase.MAX_FILTER_COVERAGE), GetterSetter.of(query -> query.getRanking().getMatchPhase().getMaxFilterCoverage(), (query, value) -> query.getRanking().getMatchPhase().setMaxFilterCoverage(asDouble(value, 0.2)))); + map.put(CompoundName.fromComponents(Ranking.RANKING, Ranking.MATCH_PHASE, Ranking.DIVERSITY, Diversity.ATTRIBUTE),GetterSetter.of(query -> query.getRanking().getMatchPhase().getDiversity().getAttribute(), (query, value) -> query.getRanking().getMatchPhase().getDiversity().setAttribute(asString(value, null)))); + map.put(CompoundName.fromComponents(Ranking.RANKING, Ranking.MATCH_PHASE, Ranking.DIVERSITY, Diversity.MINGROUPS), GetterSetter.of(query -> query.getRanking().getMatchPhase().getDiversity().getMinGroups(), (query, value) -> query.getRanking().getMatchPhase().getDiversity().setMinGroups(asLong(value, null)))); + map.put(CompoundName.fromComponents(Ranking.RANKING, Ranking.MATCH_PHASE, Ranking.DIVERSITY, Diversity.CUTOFF, Diversity.FACTOR), GetterSetter.of(query -> query.getRanking().getMatchPhase().getDiversity().getCutoffFactor(), (query, value) -> query.getRanking().getMatchPhase().getDiversity().setCutoffFactor(asDouble(value, 10.0)))); + map.put(CompoundName.fromComponents(Ranking.RANKING, Ranking.MATCH_PHASE, Ranking.DIVERSITY, Diversity.CUTOFF, Diversity.STRATEGY), GetterSetter.of(query -> query.getRanking().getMatchPhase().getDiversity().getCutoffStrategy(), (query, value) -> query.getRanking().getMatchPhase().getDiversity().setCutoffStrategy(asString(value, "loose")))); + map.put(CompoundName.fromComponents(Ranking.RANKING, Ranking.SOFTTIMEOUT, SoftTimeout.ENABLE), GetterSetter.of(query -> query.getRanking().getSoftTimeout().getEnable(), (query, value) -> query.getRanking().getSoftTimeout().setEnable(asBoolean(value, true)))); + map.put(CompoundName.fromComponents(Ranking.RANKING, Ranking.SOFTTIMEOUT, SoftTimeout.FACTOR), GetterSetter.of(query -> query.getRanking().getSoftTimeout().getFactor(), (query, value) -> query.getRanking().getSoftTimeout().setFactor(asDouble(value, null)))); + map.put(CompoundName.fromComponents(Ranking.RANKING, Ranking.SOFTTIMEOUT, SoftTimeout.TAILCOST), GetterSetter.of(query -> query.getRanking().getSoftTimeout().getTailcost(), (query, value) -> query.getRanking().getSoftTimeout().setTailcost(asDouble(value, null)))); + map.put(CompoundName.fromComponents(Select.SELECT), GetterSetter.of(query -> query.getSelect().getGroupingExpressionString(), (query, value) -> query.getSelect().setGroupingExpressionString(asString(value, "")))); + map.put(CompoundName.fromComponents(Select.SELECT, Select.WHERE), GetterSetter.of(query -> query.getSelect().getWhereString(), (query, value) -> query.getSelect().setWhereString(asString(value, "")))); + map.put(CompoundName.fromComponents(Select.SELECT, Select.GROUPING), GetterSetter.of(query -> query.getSelect().getGroupingString(), (query, value) -> query.getSelect().setGroupingString(asString(value, "")))); + map.put(CompoundName.fromComponents(Trace.TRACE, Trace.LEVEL), GetterSetter.of(query -> query.getTrace().getLevel(), (query, value) -> query.getTrace().setLevel(asInteger(value, 0)))); + map.put(CompoundName.fromComponents(Trace.TRACE, Trace.EXPLAIN_LEVEL), GetterSetter.of(query -> query.getTrace().getExplainLevel(), (query, value) -> query.getTrace().setExplainLevel(asInteger(value, 0)))); + map.put(CompoundName.fromComponents(Trace.TRACE, Trace.PROFILE_DEPTH), GetterSetter.of(null, (query, value) -> query.getTrace().setProfileDepth(asInteger(value, 0)))); + map.put(CompoundName.fromComponents(Trace.TRACE, Trace.TIMESTAMPS), GetterSetter.of(query -> query.getTrace().getTimestamps(), (query, value) -> query.getTrace().setTimestamps(asBoolean(value, false)))); + map.put(CompoundName.fromComponents(Trace.TRACE, Trace.QUERY), GetterSetter.of(query -> query.getTrace().getQuery(), (query, value) -> query.getTrace().setQuery(asBoolean(value, true)))); + map.put(CompoundName.fromComponents(Trace.TRACE, Trace.PROFILING, Profiling.MATCHING, ProfilingParams.DEPTH), GetterSetter.of(query -> query.getTrace().getProfiling().getMatching().getDepth(), (query, value) -> query.getTrace().getProfiling().getMatching().setDepth(asInteger(value, 0)))); + map.put(CompoundName.fromComponents(Trace.TRACE, Trace.PROFILING, Profiling.FIRST_PHASE_RANKING, ProfilingParams.DEPTH), GetterSetter.of(query -> query.getTrace().getProfiling().getFirstPhaseRanking().getDepth(), (query, value) -> query.getTrace().getProfiling().getFirstPhaseRanking().setDepth(asInteger(value, 0)))); + map.put(CompoundName.fromComponents(Trace.TRACE, Trace.PROFILING, Profiling.SECOND_PHASE_RANKING, ProfilingParams.DEPTH), GetterSetter.of(query -> query.getTrace().getProfiling().getSecondPhaseRanking().getDepth(), (query, value) -> query.getTrace().getProfiling().getSecondPhaseRanking().setDepth(asInteger(value, 0)))); + map.put(CompoundName.fromComponents(Presentation.PRESENTATION, Presentation.BOLDING), GetterSetter.of(query -> query.getPresentation().getBolding(), (query, value) -> query.getPresentation().setBolding(asBoolean(value, true)))); + map.put(CompoundName.fromComponents(Presentation.PRESENTATION, Presentation.SUMMARY), GetterSetter.of(query -> query.getPresentation().getSummary(), (query, value) -> query.getPresentation().setSummary(asString(value, "")))); + map.put(CompoundName.fromComponents(Presentation.PRESENTATION, Presentation.FORMAT), GetterSetter.of(query -> query.getPresentation().getFormat(), (query, value) -> query.getPresentation().setFormat(asString(value, "")))); + map.put(CompoundName.fromComponents(Presentation.PRESENTATION, Presentation.TIMING), GetterSetter.of(query -> query.getPresentation().getTiming(), (query, value) -> query.getPresentation().setTiming(asBoolean(value, true)))); + map.put(CompoundName.fromComponents(Presentation.PRESENTATION, Presentation.SUMMARY_FIELDS), GetterSetter.of(query -> query.getPresentation().getSummaryFields(), (query, value) -> query.getPresentation().setSummaryFields(asString(value, "")))); + map.put(CompoundName.fromComponents(Presentation.PRESENTATION, Presentation.FORMAT, Presentation.TENSORS), GetterSetter.of(query -> query.getPresentation().getTensorShortForm(), (query, value) -> query.getPresentation().setTensorFormat(asString(value, "short")))); // TODO: Switch default to short-value on Vespa 9); + map.put(Query.HITS, GetterSetter.of(Query::getHits, (query, value) -> query.setHits(asInteger(value,10)))); + map.put(Query.OFFSET, GetterSetter.of(Query::getOffset, (query, value) -> query.setOffset(asInteger(value,0)))); + map.put(Query.TIMEOUT, GetterSetter.of(Query::getTimeout, (query, value) -> query.setTimeout(value.toString()))); + map.put(Query.NO_CACHE, GetterSetter.of(Query::getNoCache, (query, value) -> query.setNoCache(asBoolean(value,false)))); + map.put(Query.GROUPING_SESSION_CACHE, GetterSetter.of(Query::getGroupingSessionCache, (query, value) -> query.setGroupingSessionCache(asBoolean(value, true)))); + + map.put(CompoundName.fromComponents(Model.MODEL), GetterSetter.of(Query::getModel, null)); + map.put(CompoundName.fromComponents(Ranking.RANKING), GetterSetter.of(Query::getRanking, null)); + map.put(CompoundName.fromComponents(Presentation.PRESENTATION), GetterSetter.of(Query::getPresentation, null)); + return map; + } + public QueryProperties(Query query, CompiledQueryProfileRegistry profileRegistry, Map<String, Embedder> embedders) { this.query = query; this.profileRegistry = profileRegistry; @@ -54,321 +147,59 @@ public class QueryProperties extends Properties { public Object get(CompoundName key, Map<String, String> context, com.yahoo.processing.request.Properties substitution) { - if (key.size() == 2 && key.first().equals(Model.MODEL)) { - Model model = query.getModel(); - if (key.last().equals(Model.QUERY_STRING)) return model.getQueryString(); - if (key.last().equals(Model.TYPE)) return model.getType(); - if (key.last().equals(Model.FILTER)) return model.getFilter(); - if (key.last().equals(Model.DEFAULT_INDEX)) return model.getDefaultIndex(); - if (key.last().equals(Model.LANGUAGE)) return model.getLanguage(); - if (key.last().equals(Model.LOCALE)) return model.getLocale(); - if (key.last().equals(Model.ENCODING)) return model.getEncoding(); - if (key.last().equals(Model.SOURCES)) return model.getSources(); - if (key.last().equals(Model.SEARCH_PATH)) return model.getSearchPath(); - if (key.last().equals(Model.RESTRICT)) return model.getRestrict(); - } - else if (key.first().equals(Ranking.RANKING)) { - Ranking ranking = query.getRanking(); - if (key.size() == 2) { - if (key.last().equals(Ranking.LOCATION)) return ranking.getLocation(); - if (key.last().equals(Ranking.PROFILE)) return ranking.getProfile(); - if (key.last().equals(Ranking.SORTING)) return ranking.getSorting(); - if (key.last().equals(Ranking.FRESHNESS)) return ranking.getFreshness(); - if (key.last().equals(Ranking.QUERYCACHE)) return ranking.getQueryCache(); - if (key.last().equals(Ranking.RERANKCOUNT)) return ranking.getRerankCount(); - if (key.last().equals(Ranking.KEEPRANKCOUNT)) return ranking.getKeepRankCount(); - if (key.last().equals(Ranking.RANKSCOREDROPLIMIT)) return ranking.getRankScoreDropLimit(); - if (key.last().equals(Ranking.LIST_FEATURES)) return ranking.getListFeatures(); - } - else if (key.size() >= 3 && key.get(1).equals(Ranking.MATCH_PHASE)) { - if (key.size() == 3) { - MatchPhase matchPhase = ranking.getMatchPhase(); - if (key.last().equals(MatchPhase.ATTRIBUTE)) return matchPhase.getAttribute(); - if (key.last().equals(MatchPhase.ASCENDING)) return matchPhase.getAscending(); - if (key.last().equals(MatchPhase.MAX_HITS)) return matchPhase.getMaxHits(); - if (key.last().equals(MatchPhase.MAX_FILTER_COVERAGE)) return matchPhase.getMaxFilterCoverage(); - } else if (key.size() >= 4 && key.get(2).equals(Ranking.DIVERSITY)) { - Diversity diversity = ranking.getMatchPhase().getDiversity(); - if (key.size() == 4) { - if (key.last().equals(Diversity.ATTRIBUTE)) return diversity.getAttribute(); - if (key.last().equals(Diversity.MINGROUPS)) return diversity.getMinGroups(); - } else if ((key.size() == 5) && key.get(3).equals(Diversity.CUTOFF)) { - if (key.last().equals(Diversity.FACTOR)) return diversity.getCutoffFactor(); - if (key.last().equals(Diversity.STRATEGY)) return diversity.getCutoffStrategy(); - } - } - } - else if (key.size() == 3 && key.get(1).equals(Ranking.SOFTTIMEOUT)) { - SoftTimeout soft = ranking.getSoftTimeout(); - if (key.last().equals(SoftTimeout.ENABLE)) return soft.getEnable(); - if (key.last().equals(SoftTimeout.FACTOR)) return soft.getFactor(); - if (key.last().equals(SoftTimeout.TAILCOST)) return soft.getTailcost(); - } - else if (key.size() == 3 && key.get(1).equals(Ranking.MATCHING)) { - Matching matching = ranking.getMatching(); - if (equalsWithLowerCaseAlias(key.last(), Matching.TERMWISELIMIT)) return matching.getTermwiseLimit(); - if (equalsWithLowerCaseAlias(key.last(), Matching.NUMTHREADSPERSEARCH)) return matching.getNumThreadsPerSearch(); - if (equalsWithLowerCaseAlias(key.last(), Matching.NUMSEARCHPARTITIIONS)) return matching.getNumSearchPartitions(); - if (equalsWithLowerCaseAlias(key.last(), Matching.MINHITSPERTHREAD)) return matching.getMinHitsPerThread(); + GetterSetter propertyAccessor = properyAccessors.get(key); + if (propertyAccessor != null && propertyAccessor.getter != null) return propertyAccessor.getter.get(query); - } - else if (key.size() > 2) { + if (key.first().equals(Ranking.RANKING)) { + Ranking ranking = query.getRanking(); + if (key.size() > 2) { // pass the portion after "ranking.features/properties" down if (key.get(1).equals(Ranking.FEATURES)) return ranking.getFeatures().getObject(key.rest().rest().toString()); if (key.get(1).equals(Ranking.PROPERTIES)) return ranking.getProperties().get(key.rest().rest().toString()); } } - else if (key.first().equals(Select.SELECT)) { - if (key.size() == 1) { - return query.getSelect().getGroupingExpressionString(); - } - else if (key.size() == 2) { - if (key.last().equals(Select.WHERE)) return query.getSelect().getWhereString(); - if (key.last().equals(Select.GROUPING)) return query.getSelect().getGroupingString(); - } + + return super.get(key, context, substitution); + } + + private void setInternal(CompoundName key, Object value, Map<String,String> context) { + GetterSetter propertyAccessor = properyAccessors.get(key); + if (propertyAccessor != null && propertyAccessor.setter != null) { + propertyAccessor.setter.set(query, value); + return; } - else if (key.first().equals(Presentation.PRESENTATION)) { - if (key.size() == 2) { - if (key.last().equals(Presentation.BOLDING)) return query.getPresentation().getBolding(); - if (key.last().equals(Presentation.SUMMARY)) return query.getPresentation().getSummary(); - if (key.last().equals(Presentation.FORMAT)) return query.getPresentation().getFormat(); - if (key.last().equals(Presentation.TIMING)) return query.getPresentation().getTiming(); - if (key.last().equals(Presentation.SUMMARY_FIELDS)) return query.getPresentation().getSummaryFields(); - } else if (key.size() == 3 && key.get(1).equals(Presentation.FORMAT)) { - if (key.last().equals(Presentation.TENSORS)) return query.getPresentation().getTensorShortForm(); + + if (key.first().equals(Ranking.RANKING)) { + if (key.size() > 2) { + String restKey = key.rest().rest().toString(); + chained().requireSettable(key, value, context); + if (key.get(1).equals(Ranking.FEATURES)) { + setRankFeature(query, restKey, toSpecifiedType(restKey, value, + profileRegistry.getTypeRegistry().getComponent("features"), + context)); + return; + } else if (key.get(1).equals(Ranking.PROPERTIES)) { + Ranking ranking = query.getRanking(); + ranking.getProperties().put(restKey, toSpecifiedType(restKey, value, + profileRegistry.getTypeRegistry().getComponent("properties"), + context)); + return; + } } } - else if (key.size() == 2 && key.first().equals(Trace.TRACE)) { - if (key.last().equals(Trace.LEVEL)) return query.getTrace().getLevel(); - if (key.last().equals(Trace.EXPLAIN_LEVEL)) return query.getTrace().getExplainLevel(); - if (key.last().equals(Trace.TIMESTAMPS)) return query.getTrace().getTimestamps(); - if (key.last().equals(Trace.QUERY)) return query.getTrace().getQuery(); + if (reservedPrefix.contains(key.first())) { + // Setting a property under the reserved paths are illegal, while retrieving(get) one is not. + throwIllegalParameter(key.rest().toString(), key.first()); + } else { + super.set(key, value, context); } - else if (key.size() == 1) { - if (key.equals(Query.HITS)) return query.getHits(); - if (key.equals(Query.OFFSET)) return query.getOffset(); - if (key.equals(Query.TIMEOUT)) return query.getTimeout(); - if (key.equals(Query.NO_CACHE)) return query.getNoCache(); - if (key.equals(Query.GROUPING_SESSION_CACHE)) return query.getGroupingSessionCache(); - if (key.toString().equals(Model.MODEL)) return query.getModel(); - if (key.toString().equals(Ranking.RANKING)) return query.getRanking(); - if (key.toString().equals(Presentation.PRESENTATION)) return query.getPresentation(); - } - - return super.get(key, context, substitution); } @Override public void set(CompoundName key, Object value, Map<String,String> context) { // Note: The defaults here are never used try { - if (key.size() == 2 && key.first().equals(Model.MODEL)) { - Model model = query.getModel(); - if (key.last().equals(Model.QUERY_STRING)) - model.setQueryString(asString(value, "")); - else if (key.last().equals(Model.TYPE)) - model.setType(asString(value, "ANY")); - else if (key.last().equals(Model.FILTER)) - model.setFilter(asString(value, "")); - else if (key.last().equals(Model.DEFAULT_INDEX)) - model.setDefaultIndex(asString(value, "")); - else if (key.last().equals(Model.LANGUAGE)) - model.setLanguage(asString(value, "")); - else if (key.last().equals(Model.LOCALE)) - model.setLocale(asString(value, "")); - else if (key.last().equals(Model.ENCODING)) - model.setEncoding(asString(value,"")); - else if (key.last().equals(Model.SEARCH_PATH)) - model.setSearchPath(asString(value,"")); - else if (key.last().equals(Model.SOURCES)) - model.setSources(asString(value,"")); - else if (key.last().equals(Model.RESTRICT)) - model.setRestrict(asString(value,"")); - else - throwIllegalParameter(key.last(), Model.MODEL); - } - else if (key.first().equals(Ranking.RANKING)) { - Ranking ranking = query.getRanking(); - if (key.size() == 2) { - if (key.last().equals(Ranking.LOCATION)) - ranking.setLocation(asString(value,"")); - else if (key.last().equals(Ranking.PROFILE)) - ranking.setProfile(asString(value,"")); - else if (key.last().equals(Ranking.SORTING)) - ranking.setSorting(asString(value,"")); - else if (key.last().equals(Ranking.FRESHNESS)) - ranking.setFreshness(asString(value, "")); - else if (key.last().equals(Ranking.QUERYCACHE)) - ranking.setQueryCache(asBoolean(value, false)); - else if (key.last().equals(Ranking.RERANKCOUNT)) - ranking.setRerankCount(asInteger(value, null)); - else if (key.last().equals(Ranking.KEEPRANKCOUNT)) - ranking.setKeepRankCount(asInteger(value, null)); - else if (key.last().equals(Ranking.RANKSCOREDROPLIMIT)) - ranking.setRankScoreDropLimit(asDouble(value, null)); - else if (key.last().equals(Ranking.LIST_FEATURES)) - ranking.setListFeatures(asBoolean(value,false)); - else - throwIllegalParameter(key.last(), Ranking.RANKING); - } - else if (key.size() >= 3 && key.get(1).equals(Ranking.MATCH_PHASE)) { - if (key.size() == 3) { - MatchPhase matchPhase = ranking.getMatchPhase(); - if (key.last().equals(MatchPhase.ATTRIBUTE)) - matchPhase.setAttribute(asString(value, null)); - else if (key.last().equals(MatchPhase.ASCENDING)) - matchPhase.setAscending(asBoolean(value, false)); - else if (key.last().equals(MatchPhase.MAX_HITS)) - matchPhase.setMaxHits(asLong(value, null)); - else if (key.last().equals(MatchPhase.MAX_FILTER_COVERAGE)) - matchPhase.setMaxFilterCoverage(asDouble(value, 0.2)); - else - throwIllegalParameter(key.rest().toString(), Ranking.MATCH_PHASE); - } - else if (key.size() > 3 && key.get(2).equals(Ranking.DIVERSITY)) { - Diversity diversity = ranking.getMatchPhase().getDiversity(); - if (key.last().equals(Diversity.ATTRIBUTE)) { - diversity.setAttribute(asString(value, null)); - } - else if (key.last().equals(Diversity.MINGROUPS)) { - diversity.setMinGroups(asLong(value, null)); - } - else if ((key.size() > 4) && key.get(3).equals(Diversity.CUTOFF)) { - if (key.last().equals(Diversity.FACTOR)) - diversity.setCutoffFactor(asDouble(value, 10.0)); - else if (key.last().equals(Diversity.STRATEGY)) - diversity.setCutoffStrategy(asString(value, "loose")); - else - throwIllegalParameter(key.rest().toString(), Diversity.CUTOFF); - } - else { - throwIllegalParameter(key.rest().toString(), Ranking.DIVERSITY); - } - } - } - else if (key.size() == 3 && key.get(1).equals(Ranking.SOFTTIMEOUT)) { - SoftTimeout soft = ranking.getSoftTimeout(); - if (key.last().equals(SoftTimeout.ENABLE)) - soft.setEnable(asBoolean(value, true)); - else if (key.last().equals(SoftTimeout.FACTOR)) - soft.setFactor(asDouble(value, null)); - else if (key.last().equals(SoftTimeout.TAILCOST)) - soft.setTailcost(asDouble(value, null)); - else - throwIllegalParameter(key.rest().toString(), Ranking.SOFTTIMEOUT); - } - else if (key.size() == 3 && key.get(1).equals(Ranking.MATCHING)) { - Matching matching = ranking.getMatching(); - if (equalsWithLowerCaseAlias(key.last(), Matching.TERMWISELIMIT)) - matching.setTermwiselimit(asDouble(value, 1.0)); - else if (equalsWithLowerCaseAlias(key.last(), Matching.NUMTHREADSPERSEARCH)) - matching.setNumThreadsPerSearch(asInteger(value, 1)); - else if (equalsWithLowerCaseAlias(key.last(), Matching.NUMSEARCHPARTITIIONS)) - matching.setNumSearchPartitions(asInteger(value, 1)); - else if (equalsWithLowerCaseAlias(key.last(), Matching.MINHITSPERTHREAD)) - matching.setMinHitsPerThread(asInteger(value, 0)); - else if (key.last().equals(Matching.POST_FILTER_THRESHOLD)) - matching.setPostFilterThreshold(asDouble(value, 1.0)); - else if (key.last().equals(Matching.APPROXIMATE_THRESHOLD)) - matching.setApproximateThreshold(asDouble(value, 0.05)); - else - throwIllegalParameter(key.rest().toString(), Ranking.MATCHING); - } - else if (key.size() > 2) { - String restKey = key.rest().rest().toString(); - chained().requireSettable(key, value, context); - if (key.get(1).equals(Ranking.FEATURES)) - setRankFeature(query, restKey, toSpecifiedType(restKey, - value, - profileRegistry.getTypeRegistry().getComponent("features"), - context)); - else if (key.get(1).equals(Ranking.PROPERTIES)) - ranking.getProperties().put(restKey, toSpecifiedType(restKey, - value, - profileRegistry.getTypeRegistry().getComponent("properties"), - context)); - else - throwIllegalParameter(key.rest().toString(), Ranking.RANKING); - } - } - else if (key.first().equals(Presentation.PRESENTATION)) { - if (key.size() == 2) { - if (key.last().equals(Presentation.BOLDING)) - query.getPresentation().setBolding(asBoolean(value, true)); - else if (key.last().equals(Presentation.SUMMARY)) - query.getPresentation().setSummary(asString(value, "")); - else if (key.last().equals(Presentation.FORMAT)) - query.getPresentation().setFormat(asString(value, "")); - else if (key.last().equals(Presentation.TIMING)) - query.getPresentation().setTiming(asBoolean(value, true)); - else if (key.last().equals(Presentation.SUMMARY_FIELDS)) - query.getPresentation().setSummaryFields(asString(value, "")); - else - throwIllegalParameter(key.last(), Presentation.PRESENTATION); - } - else if (key.size() == 3 && key.get(1).equals(Presentation.FORMAT)) { - if (key.last().equals(Presentation.TENSORS)) - query.getPresentation().setTensorFormat(asString(value, "short")); // TODO: Switch default to short-value on Vespa 9 - else - throwIllegalParameter(key.last(), Presentation.FORMAT); - } - else - throwIllegalParameter(key.last(), Presentation.PRESENTATION); - } - else if (key.size() == 2 && key.first().equals(Trace.TRACE)) { - if (key.last().equals(Trace.LEVEL)) - query.getTrace().setLevel(asInteger(value, 0)); - if (key.last().equals(Trace.EXPLAIN_LEVEL)) - query.getTrace().setExplainLevel(asInteger(value, 0)); - if (key.last().equals(Trace.PROFILE_DEPTH)) - query.getTrace().setProfileDepth(asInteger(value, 0)); - if (key.last().equals(Trace.TIMESTAMPS)) - query.getTrace().setTimestamps(asBoolean(value, false)); - if (key.last().equals(Trace.QUERY)) - query.getTrace().setQuery(asBoolean(value, true)); - } - else if ((key.size() == 4) && - key.get(0).equals(Trace.TRACE) && - key.get(1).equals(Trace.PROFILING) && - key.get(3).equals(ProfilingParams.DEPTH)) { - var params = getProfilingParams(query.getTrace().getProfiling(), key.get(2)); - if (params != null) { - params.setDepth(asInteger(value, 0)); - } - } - else if (key.first().equals(Select.SELECT)) { - if (key.size() == 1) { - query.getSelect().setGroupingExpressionString(asString(value, "")); - } - else if (key.size() == 2) { - if (key.last().equals(Select.WHERE)) - query.getSelect().setWhereString(asString(value, "")); - else if (key.last().equals(Select.GROUPING)) - query.getSelect().setGroupingString(asString(value, "")); - else - throwIllegalParameter(key.rest().toString(), Select.SELECT); - } - else { - throwIllegalParameter(key.last(), Select.SELECT); - } - } - else if (key.size() == 1) { - if (key.equals(Query.HITS)) - query.setHits(asInteger(value,10)); - else if (key.equals(Query.OFFSET)) - query.setOffset(asInteger(value,0)); - else if (key.equals(Query.TIMEOUT)) - query.setTimeout(value.toString()); - else if (key.equals(Query.NO_CACHE)) - query.setNoCache(asBoolean(value,false)); - else if (key.equals(Query.GROUPING_SESSION_CACHE)) - query.setGroupingSessionCache(asBoolean(value, true)); - else - super.set(key,value,context); - } - else { - super.set(key, value, context); - } + setInternal(key, value, context); } catch (Exception e) { // Make sure error messages are informative. This should be moved out of this properties implementation if (e.getMessage() != null && e.getMessage().startsWith("Could not set")) @@ -378,20 +209,8 @@ public class QueryProperties extends Properties { } } - private static ProfilingParams getProfilingParams(Profiling prof, String name) { - if (name.equals(Profiling.MATCHING)) { - return prof.getMatching(); - } else if (name.equals(Profiling.FIRST_PHASE_RANKING)) { - return prof.getFirstPhaseRanking(); - } else if (name.equals(Profiling.SECOND_PHASE_RANKING)) { - return prof.getSecondPhaseRanking(); - } - return null; - } - @Override - public Map<String, Object> listProperties(CompoundName prefix, - Map<String,String> context, + public Map<String, Object> listProperties(CompoundName prefix, Map<String, String> context, com.yahoo.processing.request.Properties substitution) { Map<String, Object> properties = super.listProperties(prefix, context, substitution); for (CompoundName queryProperty : Query.nativeProperties) { @@ -430,16 +249,11 @@ public class QueryProperties extends Properties { return field.getType().convertFrom(value, new ConversionContext(key, profileRegistry, embedders, context)); } - private void throwIllegalParameter(String key,String namespace) { + private void throwIllegalParameter(String key, String namespace) { throw new IllegalInputException("'" + key + "' is not a valid property in '" + namespace + "'. See the query api for valid keys starting by '" + namespace + "'."); } - private boolean equalsWithLowerCaseAlias(String key, String property) { - // The lowercase alias is used to provide backwards compatibility of a query property that was wrongly named in the first place. - return key.equals(property) || key.equals(property.toLowerCase()); - } - @Override public final Query getParentQuery() { return query; diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/RankProfileInputProperties.java b/container-search/src/main/java/com/yahoo/search/query/properties/RankProfileInputProperties.java index 6c65a5e898a..6203eadffad 100644 --- a/container-search/src/main/java/com/yahoo/search/query/properties/RankProfileInputProperties.java +++ b/container-search/src/main/java/com/yahoo/search/query/properties/RankProfileInputProperties.java @@ -3,6 +3,7 @@ package com.yahoo.search.query.properties; import com.yahoo.api.annotations.Beta; import com.yahoo.language.process.Embedder; +import com.yahoo.processing.IllegalInputException; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.Query; import com.yahoo.search.schema.SchemaInfo; @@ -46,7 +47,7 @@ public class RankProfileInputProperties extends Properties { query.getModel().getLanguage()); } catch (IllegalArgumentException e) { - throw new IllegalArgumentException("Could not set '" + name + "' to '" + value + "'", e); + throw new IllegalInputException("Could not set '" + name + "' to '" + value + "'", e); } } } @@ -84,7 +85,7 @@ public class RankProfileInputProperties extends Properties { } private void throwIllegalInput(CompoundName name, Object value, TensorType expectedType) { - throw new IllegalArgumentException("Could not set '" + name + "' to '" + value + "': " + + throw new IllegalInputException("Could not set '" + name + "' to '" + value + "': " + "This input is declared in rank profile '" + query.getRanking().getProfile() + "' as " + expectedType); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/FlagValidationException.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/FlagValidationException.java new file mode 100644 index 00000000000..00c88102819 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/FlagValidationException.java @@ -0,0 +1,11 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.systemflags.v1; + +/** + * @author hakonhall + */ +public class FlagValidationException extends RuntimeException { + public FlagValidationException(String message) { + super(message); + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/FlagsTarget.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/FlagsTarget.java index bad53620c81..fbf3a5d9a03 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/FlagsTarget.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/FlagsTarget.java @@ -80,6 +80,54 @@ public interface FlagsTarget { static String zoneFile(SystemName system, ZoneId zone) { return jsonFile(system.value() + "." + zone.environment().value() + "." + zone.region().value()); } static String controllerFile(SystemName system) { return jsonFile(system.value() + ".controller"); } + /** Return true if the filename applies to the system. Throws on invalid filename format. */ + static boolean filenameForSystem(String filename, SystemName system) throws FlagValidationException { + if (filename.equals(defaultFile())) return true; + + String[] parts = filename.split("\\.", -1); + if (parts.length < 2) throw new FlagValidationException("Invalid flag filename: " + filename); + + if (!parts[parts.length - 1].equals("json")) throw new FlagValidationException("Invalid flag filename: " + filename); + + SystemName systemFromFile; + try { + systemFromFile = SystemName.from(parts[0]); + } catch (IllegalArgumentException e) { + throw new FlagValidationException("First part of flag filename is neither 'default' nor a valid system: " + filename); + } + if (!SystemName.hostedVespa().contains(systemFromFile)) + throw new FlagValidationException("Unknown system in flag filename: " + filename); + if (!systemFromFile.equals(system)) return false; + + if (parts.length == 2) return true; // systemFile + + if (parts.length == 3) { + if (parts[1].equals("controller")) return true; // controllerFile + try { + Environment.from(parts[1]); + } catch (IllegalArgumentException e) { + throw new FlagValidationException("Invalid environment in flag filename: " + filename); + } + return true; // environmentFile + } + + if (parts.length == 4) { + try { + Environment.from(parts[1]); + } catch (IllegalArgumentException e) { + throw new FlagValidationException("Invalid environment in flag filename: " + filename); + } + try { + RegionName.from(parts[2]); + } catch (IllegalArgumentException e) { + throw new FlagValidationException("Invalid region in flag filename: " + filename); + } + return true; // zoneFile + } + + throw new FlagValidationException("Invalid flag filename: " + filename); + } + /** Partially resolve inter-zone dimensions, except those dimensions defined by the flag for a controller zone. */ static FlagData partialResolve(FlagData data, SystemName system, CloudName cloud, ZoneId virtualZoneId) { Set<FetchVector.Dimension> flagDimensions = @@ -94,7 +142,7 @@ public interface FlagsTarget { var fetchVector = new FetchVector(); if (!flagDimensions.contains(CLOUD)) fetchVector = fetchVector.with(CLOUD, cloud.value()); if (!flagDimensions.contains(ENVIRONMENT)) fetchVector = fetchVector.with(ENVIRONMENT, virtualZoneId.environment().value()); - if (!flagDimensions.contains(SYSTEM)) fetchVector = fetchVector.with(SYSTEM, system.value()); + fetchVector = fetchVector.with(SYSTEM, system.value()); if (!flagDimensions.contains(ZONE_ID)) fetchVector = fetchVector.with(ZONE_ID, virtualZoneId.value()); return fetchVector.isEmpty() ? data : data.partialResolve(fetchVector); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchive.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchive.java index 1c547fea8ba..c6f1d96ed43 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchive.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchive.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.systemflags.v1; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; @@ -18,11 +19,14 @@ import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.text.JSON; import com.yahoo.vespa.flags.FetchVector; import com.yahoo.vespa.flags.FlagId; +import com.yahoo.vespa.flags.json.Condition; import com.yahoo.vespa.flags.json.DimensionHelper; import com.yahoo.vespa.flags.json.FlagData; +import com.yahoo.vespa.flags.json.RelationalCondition; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import java.io.BufferedInputStream; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -32,7 +36,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Objects; @@ -49,6 +52,7 @@ import java.util.zip.ZipOutputStream; import static com.yahoo.config.provision.CloudName.AWS; import static com.yahoo.config.provision.CloudName.GCP; import static com.yahoo.config.provision.CloudName.YAHOO; +import static com.yahoo.vespa.flags.FetchVector.Dimension.SYSTEM; import static com.yahoo.yolean.Exceptions.uncheck; /** @@ -74,7 +78,7 @@ public class SystemFlagsDataArchive { this.files = files; } - public static SystemFlagsDataArchive fromZip(InputStream rawIn) { + public static SystemFlagsDataArchive fromZip(InputStream rawIn, ZoneRegistry zoneRegistry) { Builder builder = new Builder(); try (ZipInputStream zipIn = new ZipInputStream(new BufferedInputStream(rawIn))) { ZipEntry entry; @@ -82,8 +86,8 @@ public class SystemFlagsDataArchive { String name = entry.getName(); if (!entry.isDirectory() && name.startsWith("flags/")) { Path filePath = Paths.get(name); - String rawData = new String(zipIn.readAllBytes(), StandardCharsets.UTF_8); - addFile(builder, rawData, filePath, Set.of(), null); + String fileContent = new String(zipIn.readAllBytes(), StandardCharsets.UTF_8); + builder.maybeAddFile(filePath, fileContent, zoneRegistry, true); } } return builder.build(); @@ -92,27 +96,19 @@ public class SystemFlagsDataArchive { } } - public static SystemFlagsDataArchive fromDirectoryAndSystem(Path directory, ZoneRegistry systemDefinition) { - return fromDirectory(directory, systemDefinition); - } - - public static SystemFlagsDataArchive fromDirectory(Path directory) { return fromDirectory(directory, null); } - - private static SystemFlagsDataArchive fromDirectory(Path directory, ZoneRegistry systemDefinition) { - Set<String> filenamesForSystem = getFilenamesForSystem(systemDefinition); + public static SystemFlagsDataArchive fromDirectory(Path directory, ZoneRegistry zoneRegistry, boolean simulateInController) { Path root = directory.toAbsolutePath(); Path flagsDirectory = directory.resolve("flags"); if (!Files.isDirectory(flagsDirectory)) { - throw new IllegalArgumentException("Sub-directory 'flags' does not exist: " + flagsDirectory); + throw new FlagValidationException("Sub-directory 'flags' does not exist: " + flagsDirectory); } - try (Stream<Path> directoryStream = Files.walk(root)) { + try (Stream<Path> directoryStream = Files.walk(flagsDirectory)) { Builder builder = new Builder(); - directoryStream.forEach(absolutePath -> { - Path relativePath = root.relativize(absolutePath); - if (!Files.isDirectory(absolutePath) && - relativePath.startsWith("flags")) { - String rawData = uncheck(() -> Files.readString(absolutePath, StandardCharsets.UTF_8)); - addFile(builder, rawData, relativePath, filenamesForSystem, systemDefinition); + directoryStream.forEach(path -> { + Path relativePath = root.relativize(path.toAbsolutePath()); + if (Files.isRegularFile(path)) { + String fileContent = uncheck(() -> Files.readString(path, StandardCharsets.UTF_8)); + builder.maybeAddFile(relativePath, fileContent, zoneRegistry, simulateInController); } }); return builder.build(); @@ -121,6 +117,14 @@ public class SystemFlagsDataArchive { } } + public byte[] toZipBytes() { + try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { + toZip(out); + return out.toByteArray(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } public void toZip(OutputStream out) { ZipOutputStream zipOut = new ZipOutputStream(out); @@ -153,139 +157,134 @@ public class SystemFlagsDataArchive { return targetData; } - public void validateAllFilesAreForTargets(SystemName currentSystem, Set<FlagsTarget> targets) throws IllegalArgumentException { + public void validateAllFilesAreForTargets(Set<FlagsTarget> targets) throws FlagValidationException { Set<String> validFiles = targets.stream() - .flatMap(target -> target.flagDataFilesPrioritized().stream()) - .collect(Collectors.toSet()); - Set<SystemName> otherSystems = Arrays.stream(SystemName.values()) - .filter(systemName -> systemName != currentSystem) - .collect(Collectors.toSet()); - files.forEach((flagId, fileMap) -> { - for (String filename : fileMap.keySet()) { - boolean isFileForOtherSystem = otherSystems.stream() - .anyMatch(system -> filename.startsWith(system.value() + ".")); - boolean isFileForCurrentSystem = validFiles.contains(filename); - if (!isFileForOtherSystem && !isFileForCurrentSystem) { - throw new IllegalArgumentException("Unknown flag file: " + toFilePath(flagId, filename)); - } + .flatMap(target -> target.flagDataFilesPrioritized().stream()) + .collect(Collectors.toSet()); + files.forEach((flagId, fileMap) -> fileMap.keySet().forEach(filename -> { + if (!validFiles.contains(filename)) { + throw new FlagValidationException("Unknown flag file: " + toFilePath(flagId, filename)); } - }); + })); } - private static Set<String> getFilenamesForSystem(ZoneRegistry systemDefinition) { - if (systemDefinition == null) return Set.of(); - return FlagsTarget.getAllTargetsInSystem(systemDefinition, false).stream() - .flatMap(target -> target.flagDataFilesPrioritized().stream()) - .collect(Collectors.toSet()); + boolean hasFlagData(FlagId flagId, String filename) { + return files.getOrDefault(flagId, Map.of()).containsKey(filename); } - private static void addFile(Builder builder, String rawData, Path filePath, Set<String> filenamesForSystem, - ZoneRegistry systemDefinition) { - String filename = filePath.getFileName().toString(); - if (filename.startsWith(".")) { - return; // Ignore files starting with '.' - } - if (!filenamesForSystem.isEmpty() && !filenamesForSystem.contains(filename)) { - if (systemDefinition != null && filename.startsWith(systemDefinition.system().value() + '.')) { - throw new IllegalArgumentException(String.format( - "Environment or zone in filename '%s' does not exist", filename)); - } - return; // Ignore files irrelevant for system - } - if (!filename.endsWith(".json")) { - throw new IllegalArgumentException(String.format("Only JSON files are allowed in 'flags/' directory (found '%s')", filePath.toString())); - } - FlagId directoryDeducedFlagId = new FlagId(filePath.getName(filePath.getNameCount()-2).toString()); - FlagData flagData; - if (rawData.isBlank()) { - flagData = new FlagData(directoryDeducedFlagId); - } else { - Set<ZoneId> zones = systemDefinition == null ? - Set.of() : - systemDefinition.zones().all().zones().stream().map(ZoneApi::getVirtualId).collect(Collectors.toSet()); - String normalizedRawData = normalizeJson(rawData, zones); - flagData = FlagData.deserialize(normalizedRawData); - if (!directoryDeducedFlagId.equals(flagData.id())) { - throw new IllegalArgumentException( - String.format("Flag data file with flag id '%s' in directory for '%s'", - flagData.id(), directoryDeducedFlagId.toString())); - } - - String serializedData = flagData.serializeToJson(); - if (!JSON.equals(serializedData, normalizedRawData)) { - throw new IllegalArgumentException(filePath + " contains unknown non-comment fields: " + - "after removing any comment fields the JSON is:\n " + - normalizedRawData + - "\nbut deserializing this ended up with a JSON that are missing some of the fields:\n " + - serializedData + - "\nSee https://git.ouroath.com/vespa/hosted-feature-flags for more info on the JSON syntax"); + private static void validateSystems(FlagData flagData) throws FlagValidationException { + flagData.rules().forEach(rule -> rule.conditions().forEach(condition -> { + if (condition.dimension() == SYSTEM) { + validateConditionValues(condition, system -> { + if (!SystemName.hostedVespa().contains(SystemName.from(system))) + throw new FlagValidationException("Unknown system: " + system); + }); } - } - - if (builder.hasFile(filename, flagData)) { - throw new IllegalArgumentException( - String.format("Flag data file in '%s' contains redundant flag data for id '%s' already set in another directory!", - filePath, flagData.id())); - } - - builder.addFile(filename, flagData); + })); } - static String normalizeJson(String json, Set<ZoneId> zones) { - JsonNode root = uncheck(() -> mapper.readTree(json)); - removeCommentsRecursively(root); - verifyValues(root, zones); - return root.toString(); + private static void validateForSystem(FlagData flagData, ZoneRegistry zoneRegistry, boolean inController) throws FlagValidationException { + Set<ZoneId> zones = inController ? + zoneRegistry.zonesIncludingSystem().all().zones().stream().map(ZoneApi::getVirtualId).collect(Collectors.toSet()) : + null; + + flagData.rules().forEach(rule -> rule.conditions().forEach(condition -> { + int force_switch_expression_dummy = switch (condition.type()) { + case RELATIONAL -> switch (condition.dimension()) { + case APPLICATION_ID, CLOUD, CLUSTER_ID, CLUSTER_TYPE, CONSOLE_USER_EMAIL, ENVIRONMENT, + HOSTNAME, NODE_TYPE, SYSTEM, TENANT_ID, ZONE_ID -> + throw new FlagValidationException(condition.type().toWire() + " " + + DimensionHelper.toWire(condition.dimension()) + + " condition is not supported"); + case VESPA_VERSION -> { + RelationalCondition rCond = RelationalCondition.create(condition.toCreateParams()); + Version version = Version.fromString(rCond.relationalPredicate().rightOperand()); + if (version.getMajor() < 8) + throw new FlagValidationException("Major Vespa version must be at least 8: " + version); + yield 0; + } + }; + + case WHITELIST, BLACKLIST -> switch (condition.dimension()) { + case APPLICATION_ID -> validateConditionValues(condition, ApplicationId::fromSerializedForm); + case CONSOLE_USER_EMAIL -> validateConditionValues(condition, email -> { + if (!email.contains("@")) + throw new FlagValidationException("Invalid email address: " + email); + }); + case CLOUD -> validateConditionValues(condition, cloud -> { + if (!Set.of(YAHOO, AWS, GCP).contains(CloudName.from(cloud))) + throw new FlagValidationException("Unknown cloud: " + cloud); + }); + case CLUSTER_ID -> validateConditionValues(condition, ClusterSpec.Id::from); + case CLUSTER_TYPE -> validateConditionValues(condition, ClusterSpec.Type::from); + case ENVIRONMENT -> validateConditionValues(condition, Environment::from); + case HOSTNAME -> validateConditionValues(condition, HostName::of); + case NODE_TYPE -> validateConditionValues(condition, NodeType::valueOf); + case SYSTEM -> throw new IllegalStateException("Flag data contains system dimension"); + case TENANT_ID -> validateConditionValues(condition, TenantName::from); + case VESPA_VERSION -> throw new FlagValidationException(condition.type().toWire() + " " + + DimensionHelper.toWire(condition.dimension()) + + " condition is not supported"); + case ZONE_ID -> validateConditionValues(condition, zoneIdString -> { + ZoneId zoneId = ZoneId.from(zoneIdString); + if (inController && !zones.contains(zoneId)) + throw new FlagValidationException("Unknown zone: " + zoneIdString); + }); + }; + }; + })); } - private static void verifyValues(JsonNode root, Set<ZoneId> zones) { - var cursor = new JsonAccessor(root); - cursor.get("rules").forEachArrayElement(rule -> rule.get("conditions").forEachArrayElement(condition -> { - FetchVector.Dimension dimension = DimensionHelper - .fromWire(condition.get("dimension") - .asString() - .orElseThrow(() -> new IllegalArgumentException("Invalid dimension in condition: " + condition))); - switch (dimension) { - case APPLICATION_ID -> validateStringValues(condition, ApplicationId::fromSerializedForm); - case CONSOLE_USER_EMAIL -> validateStringValues(condition, email -> {}); - case CLOUD -> validateStringValues(condition, cloud -> { - if (!Set.of(YAHOO, AWS, GCP).contains(CloudName.from(cloud))) - throw new IllegalArgumentException("Unknown cloud: " + cloud); - }); - case CLUSTER_ID -> validateStringValues(condition, ClusterSpec.Id::from); - case CLUSTER_TYPE -> validateStringValues(condition, ClusterSpec.Type::from); - case ENVIRONMENT -> validateStringValues(condition, Environment::from); - case HOSTNAME -> validateStringValues(condition, HostName::of); - case NODE_TYPE -> validateStringValues(condition, NodeType::valueOf); - case SYSTEM -> validateStringValues(condition, system -> { - if (!Set.of(SystemName.cd, SystemName.main, SystemName.PublicCd, SystemName.Public).contains(SystemName.from(system))) - throw new IllegalArgumentException("Unknown system: " + system); - }); - case TENANT_ID -> validateStringValues(condition, TenantName::from); - case VESPA_VERSION -> validateStringValues(condition, versionString -> { - Version vespaVersion = Version.fromString(versionString); - if (vespaVersion.getMajor() < 8) - throw new IllegalArgumentException("Major Vespa version must be at least 8: " + versionString); - }); - case ZONE_ID -> validateStringValues(condition, zoneId -> { - if (!zones.contains(ZoneId.from(zoneId))) - throw new IllegalArgumentException("Unknown zone: " + zoneId); - }); + private static int validateConditionValues(Condition condition, Consumer<String> valueValidator) { + condition.toCreateParams().values().forEach(value -> { + try { + valueValidator.accept(value); + } catch (IllegalArgumentException e) { + String dimension = DimensionHelper.toWire(condition.dimension()); + String type = condition.type().toWire(); + throw new FlagValidationException("Invalid %s '%s' in %s condition: %s".formatted(dimension, value, type, e.getMessage())); } - })); + }); + + return 0; // dummy to force switch expression } - private static void validateStringValues(JsonAccessor condition, Consumer<String> valueValidator) { - condition.get("values").forEachArrayElement(conditionValue -> { - String value = conditionValue.asString() - .orElseThrow(() -> { - String dimension = condition.get("dimension").asString().orElseThrow(); - String type = condition.get("type").asString().orElseThrow(); - return new IllegalArgumentException("Non-string value in %s %s condition: %s".formatted( - dimension, type, conditionValue)); - }); - valueValidator.accept(value); - }); + private static FlagData parseFlagData(FlagId flagId, String fileContent, ZoneRegistry zoneRegistry, boolean inController) { + if (fileContent.isBlank()) return new FlagData(flagId); + + final JsonNode root; + try { + root = mapper.readTree(fileContent); + } catch (JsonProcessingException e) { + throw new FlagValidationException("Invalid JSON: " + e.getMessage()); + } + + removeCommentsRecursively(root); + removeNullRuleValues(root); + String normalizedRawData = root.toString(); + FlagData flagData = FlagData.deserialize(normalizedRawData); + + if (!flagId.equals(flagData.id())) + throw new FlagValidationException("Flag ID specified in file (%s) doesn't match the directory name (%s)" + .formatted(flagData.id(), flagId.toString())); + + String serializedData = flagData.serializeToJson(); + if (!JSON.equals(serializedData, normalizedRawData)) + throw new FlagValidationException(""" + Unknown non-comment fields or rules with null values: after removing any comment fields the JSON is: + %s + but deserializing this ended up with: + %s + These fields may be spelled wrong, or remove them? + See https://git.ouroath.com/vespa/hosted-feature-flags for more info on the JSON syntax + """.formatted(normalizedRawData, serializedData)); + + validateSystems(flagData); + flagData = flagData.partialResolve(new FetchVector().with(SYSTEM, zoneRegistry.system().value())); + + validateForSystem(flagData, zoneRegistry, inController); + + return flagData; } private static void removeCommentsRecursively(JsonNode node) { @@ -297,6 +296,22 @@ public class SystemFlagsDataArchive { node.forEach(SystemFlagsDataArchive::removeCommentsRecursively); } + private static void removeNullRuleValues(JsonNode root) { + if (root instanceof ObjectNode objectNode) { + JsonNode rules = objectNode.get("rules"); + if (rules != null) { + rules.forEach(ruleNode -> { + if (ruleNode instanceof ObjectNode rule) { + JsonNode value = rule.get("value"); + if (value != null && value.isNull()) { + rule.remove("value"); + } + } + }); + } + } + } + private static String toFilePath(FlagId flagId, String filename) { return "flags/" + flagId.toString() + "/" + filename; } @@ -306,56 +321,46 @@ public class SystemFlagsDataArchive { public Builder() {} - public Builder addFile(String filename, FlagData data) { - files.computeIfAbsent(data.id(), k -> new TreeMap<>()).put(filename, data); - return this; - } + boolean maybeAddFile(Path filePath, String fileContent, ZoneRegistry zoneRegistry, boolean inController) { + String filename = filePath.getFileName().toString(); - public boolean hasFile(String filename, FlagData data) { - return files.containsKey(data.id()) && files.get(data.id()).containsKey(filename); - } + if (filename.startsWith(".")) + return false; // Ignore files starting with '.' - public SystemFlagsDataArchive build() { - Map<FlagId, Map<String, FlagData>> copy = new TreeMap<>(); - files.forEach((flagId, map) -> copy.put(flagId, new TreeMap<>(map))); - return new SystemFlagsDataArchive(copy); - } - - } + if (!inController && !FlagsTarget.filenameForSystem(filename, zoneRegistry.system())) + return false; // Ignore files for other systems - private static class JsonAccessor { - private final JsonNode jsonNode; + FlagId directoryDeducedFlagId = new FlagId(filePath.getName(filePath.getNameCount()-2).toString()); - public JsonAccessor(JsonNode jsonNode) { - this.jsonNode = jsonNode; - } + if (hasFile(filename, directoryDeducedFlagId)) + throw new FlagValidationException("Flag data file in '%s' contains redundant flag data for id '%s' already set in another directory!" + .formatted(filePath, directoryDeducedFlagId)); - public JsonAccessor get(String fieldName) { - if (jsonNode == null) { - return this; - } else { - return new JsonAccessor(jsonNode.get(fieldName)); + final FlagData flagData; + try { + flagData = parseFlagData(directoryDeducedFlagId, fileContent, zoneRegistry, inController); + } catch (FlagValidationException e) { + throw new FlagValidationException("In file " + filePath + ": " + e.getMessage()); } - } - public Optional<String> asString() { - return jsonNode != null && jsonNode.isTextual() ? Optional.of(jsonNode.textValue()) : Optional.empty(); + addFile(filename, flagData); + return true; } - public void forEachArrayElement(Consumer<JsonAccessor> consumer) { - if (jsonNode != null && jsonNode.isArray()) { - jsonNode.forEach(jsonNodeElement -> consumer.accept(new JsonAccessor(jsonNodeElement))); - } + public Builder addFile(String filename, FlagData data) { + files.computeIfAbsent(data.id(), k -> new TreeMap<>()).put(filename, data); + return this; } - /** Returns true if this (JsonNode) is a string and equal to value. */ - public boolean isEqualTo(String value) { - return jsonNode != null && jsonNode.isTextual() && Objects.equals(jsonNode.textValue(), value); + public boolean hasFile(String filename, FlagId id) { + return files.containsKey(id) && files.get(id).containsKey(filename); } - @Override - public String toString() { - return jsonNode == null ? "undefined" : jsonNode.toString(); + public SystemFlagsDataArchive build() { + Map<FlagId, Map<String, FlagData>> copy = new TreeMap<>(); + files.forEach((flagId, map) -> copy.put(flagId, new TreeMap<>(map))); + return new SystemFlagsDataArchive(copy); } + } } diff --git a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/FlagsTargetTest.java b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/FlagsTargetTest.java new file mode 100644 index 00000000000..9177813e38f --- /dev/null +++ b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/FlagsTargetTest.java @@ -0,0 +1,41 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.systemflags.v1; + +import com.yahoo.config.provision.SystemName; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * @author hakonhall + */ +class FlagsTargetTest { + @Test + void sanityCheckFilename() { + assertTrue(FlagsTarget.filenameForSystem("default.json", SystemName.main)); + assertTrue(FlagsTarget.filenameForSystem("main.json", SystemName.main)); + assertTrue(FlagsTarget.filenameForSystem("main.controller.json", SystemName.main)); + assertTrue(FlagsTarget.filenameForSystem("main.prod.json", SystemName.main)); + assertTrue(FlagsTarget.filenameForSystem("main.prod.us-west-1.json", SystemName.main)); + assertTrue(FlagsTarget.filenameForSystem("main.prod.abc-foo-3.json", SystemName.main)); + + assertFalse(FlagsTarget.filenameForSystem("public.json", SystemName.main)); + assertFalse(FlagsTarget.filenameForSystem("public.controller.json", SystemName.main)); + assertFalse(FlagsTarget.filenameForSystem("public.prod.json", SystemName.main)); + assertFalse(FlagsTarget.filenameForSystem("public.prod.us-west-1.json", SystemName.main)); + assertFalse(FlagsTarget.filenameForSystem("public.prod.abc-foo-3.json", SystemName.main)); + + assertFlagValidationException("First part of flag filename is neither 'default' nor a valid system: defaults.json", "defaults.json"); + assertFlagValidationException("Invalid flag filename: default", "default"); + assertFlagValidationException("Invalid flag filename: README", "README"); + assertFlagValidationException("First part of flag filename is neither 'default' nor a valid system: nosystem.json", "nosystem.json"); + assertFlagValidationException("Invalid environment in flag filename: main.noenv.json", "main.noenv.json"); + assertFlagValidationException("Invalid region in flag filename: main.prod.%badregion.json", "main.prod.%badregion.json"); + } + + private void assertFlagValidationException(String expectedMessage, String filename) { + FlagValidationException e = assertThrows(FlagValidationException.class, () -> FlagsTarget.filenameForSystem(filename, SystemName.main)); + assertEquals(expectedMessage, e.getMessage()); + } + +}
\ No newline at end of file diff --git a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchiveTest.java b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchiveTest.java index a24bed54a8a..759f21579d4 100644 --- a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchiveTest.java +++ b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchiveTest.java @@ -14,6 +14,7 @@ import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.flags.FetchVector; import com.yahoo.vespa.flags.FlagId; import com.yahoo.vespa.flags.RawFlag; +import com.yahoo.vespa.flags.json.Condition; import com.yahoo.vespa.flags.json.FlagData; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import org.junit.jupiter.api.Test; @@ -31,8 +32,9 @@ import java.net.URI; import java.nio.file.Path; import java.nio.file.Paths; import java.util.List; -import java.util.Map; +import java.util.Optional; import java.util.Set; +import java.util.stream.Stream; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -75,40 +77,73 @@ public class SystemFlagsDataArchiveTest { @Test void can_serialize_and_deserialize_archive() throws IOException { + can_serialize_and_deserialize_archive(false); + can_serialize_and_deserialize_archive(true); + } + + private void can_serialize_and_deserialize_archive(boolean simulateInController) throws IOException { File tempFile = File.createTempFile("serialized-flags-archive", null, temporaryFolder); try (OutputStream out = new BufferedOutputStream(new FileOutputStream(tempFile))) { - var archive = SystemFlagsDataArchive.fromDirectory(Paths.get("src/test/resources/system-flags/")); + var archive = fromDirectory("system-flags", simulateInController); + if (simulateInController) + archive.validateAllFilesAreForTargets(Set.of(mainControllerTarget, prodUsWestCfgTarget)); archive.toZip(out); } try (InputStream in = new BufferedInputStream(new FileInputStream(tempFile))) { - SystemFlagsDataArchive archive = SystemFlagsDataArchive.fromZip(in); + SystemFlagsDataArchive archive = SystemFlagsDataArchive.fromZip(in, createZoneRegistryMock()); assertArchiveReturnsCorrectTestFlagDataForTarget(archive); } } @Test void retrieves_correct_flag_data_for_target() { - var archive = SystemFlagsDataArchive.fromDirectory(Paths.get("src/test/resources/system-flags/")); + retrieves_correct_flag_data_for_target(false); + retrieves_correct_flag_data_for_target(true); + } + + private void retrieves_correct_flag_data_for_target(boolean simulateInController) { + var archive = fromDirectory("system-flags", simulateInController); + if (simulateInController) + archive.validateAllFilesAreForTargets(Set.of(mainControllerTarget, prodUsWestCfgTarget)); assertArchiveReturnsCorrectTestFlagDataForTarget(archive); } @Test void supports_multi_level_flags_directory() { - var archive = SystemFlagsDataArchive.fromDirectory(Paths.get("src/test/resources/system-flags-multi-level/")); + supports_multi_level_flags_directory(false); + supports_multi_level_flags_directory(true); + } + + private void supports_multi_level_flags_directory(boolean simulateInController) { + var archive = fromDirectory("system-flags-multi-level", simulateInController); + if (simulateInController) + archive.validateAllFilesAreForTargets(Set.of(mainControllerTarget, prodUsWestCfgTarget)); assertFlagDataHasValue(archive, MY_TEST_FLAG, mainControllerTarget, "default"); } @Test void duplicated_flagdata_is_detected() { - Throwable exception = assertThrows(IllegalArgumentException.class, () -> { - var archive = SystemFlagsDataArchive.fromDirectory(Paths.get("src/test/resources/system-flags-multi-level-with-duplicated-flagdata/")); - }); + duplicated_flagdata_is_detected(false); + duplicated_flagdata_is_detected(true); + } + + private void duplicated_flagdata_is_detected(boolean simulateInController) { + Throwable exception = assertThrows(FlagValidationException.class, () -> { + fromDirectory("system-flags-multi-level-with-duplicated-flagdata", simulateInController); + }); assertTrue(exception.getMessage().contains("contains redundant flag data for id 'my-test-flag' already set in another directory!")); } @Test void empty_files_are_handled_as_no_flag_data_for_target() { - var archive = SystemFlagsDataArchive.fromDirectory(Paths.get("src/test/resources/system-flags/")); + empty_files_are_handled_as_no_flag_data_for_target(false); + empty_files_are_handled_as_no_flag_data_for_target(true); + } + + private void empty_files_are_handled_as_no_flag_data_for_target(boolean simulateInController) { + var archive = fromDirectory("system-flags", simulateInController); + if (simulateInController) + archive.validateAllFilesAreForTargets(Set.of(mainControllerTarget, prodUsWestCfgTarget)); assertNoFlagData(archive, FLAG_WITH_EMPTY_DATA, mainControllerTarget); assertFlagDataHasValue(archive, FLAG_WITH_EMPTY_DATA, prodUsWestCfgTarget, "main.prod.us-west-1"); assertNoFlagData(archive, FLAG_WITH_EMPTY_DATA, prodUsEast3CfgTarget); @@ -116,148 +151,234 @@ public class SystemFlagsDataArchiveTest { } @Test - void throws_exception_on_non_json_file() { - Throwable exception = assertThrows(IllegalArgumentException.class, () -> { - SystemFlagsDataArchive.fromDirectory(Paths.get("src/test/resources/system-flags-with-invalid-file-name/")); + void hv_throws_exception_on_non_json_file() { + Throwable exception = assertThrows(FlagValidationException.class, () -> { + fromDirectory("system-flags-with-invalid-file-name", false); }); - assertTrue(exception.getMessage().contains("Only JSON files are allowed in 'flags/' directory (found 'flags/my-test-flag/file-name-without-dot-json')")); + assertEquals("Invalid flag filename: file-name-without-dot-json", + exception.getMessage()); } @Test void throws_exception_on_unknown_file() { - Throwable exception = assertThrows(IllegalArgumentException.class, () -> { - SystemFlagsDataArchive archive = SystemFlagsDataArchive.fromDirectory(Paths.get("src/test/resources/system-flags-with-unknown-file-name/")); - archive.validateAllFilesAreForTargets(SystemName.main, Set.of(mainControllerTarget, prodUsWestCfgTarget)); + Throwable exception = assertThrows(FlagValidationException.class, () -> { + SystemFlagsDataArchive archive = fromDirectory("system-flags-with-unknown-file-name", true); + archive.validateAllFilesAreForTargets(Set.of(mainControllerTarget, prodUsWestCfgTarget)); }); - assertTrue(exception.getMessage().contains("Unknown flag file: flags/my-test-flag/main.prod.unknown-region.json")); + assertEquals("Unknown flag file: flags/my-test-flag/main.prod.unknown-region.json", exception.getMessage()); + } + + @Test + void unknown_region_is_still_zipped() { + // This is useful when the program zipping the files is on a different version than the controller + var archive = fromDirectory("system-flags-with-unknown-file-name", false); + assertTrue(archive.hasFlagData(MY_TEST_FLAG, "main.prod.unknown-region.json")); } @Test void throws_exception_on_unknown_region() { - Throwable exception = assertThrows(IllegalArgumentException.class, () -> { - Path directory = Paths.get("src/test/resources/system-flags-with-unknown-file-name/"); - SystemFlagsDataArchive.fromDirectoryAndSystem(directory, createZoneRegistryMock()); + Throwable exception = assertThrows(FlagValidationException.class, () -> { + var archive = fromDirectory("system-flags-with-unknown-file-name", true); + archive.validateAllFilesAreForTargets(Set.of(mainControllerTarget, prodUsWestCfgTarget)); }); - assertTrue(exception.getMessage().contains("Environment or zone in filename 'main.prod.unknown-region.json' does not exist")); + assertEquals("Unknown flag file: flags/my-test-flag/main.prod.unknown-region.json", exception.getMessage()); } @Test void throws_on_unknown_field() { - Throwable exception = assertThrows(IllegalArgumentException.class, () -> { - SystemFlagsDataArchive.fromDirectory(Paths.get("src/test/resources/system-flags-with-unknown-field-name/")); + Throwable exception = assertThrows(FlagValidationException.class, () -> { + fromDirectory("system-flags-with-unknown-field-name", true); }); - assertTrue(exception.getMessage().contains("flags/my-test-flag/main.prod.us-west-1.json contains unknown non-comment fields: after removing any comment fields the JSON is:\n" + - " {\"id\":\"my-test-flag\",\"rules\":[{\"condition\":[{\"type\":\"whitelist\",\"dimension\":\"hostname\",\"values\":[\"foo.com\"]}],\"value\":\"default\"}]}\n" + - "but deserializing this ended up with a JSON that are missing some of the fields:\n" + - " {\"id\":\"my-test-flag\",\"rules\":[{\"value\":\"default\"}]}\n" + - "See https://git.ouroath.com/vespa/hosted-feature-flags for more info on the JSON syntax")); + assertEquals(""" + In file flags/my-test-flag/main.prod.us-west-1.json: Unknown non-comment fields or rules with null values: after removing any comment fields the JSON is: + {"id":"my-test-flag","rules":[{"condition":[{"type":"whitelist","dimension":"hostname","values":["foo.com"]}],"value":"default"}]} + but deserializing this ended up with: + {"id":"my-test-flag","rules":[{"value":"default"}]} + These fields may be spelled wrong, or remove them? + See https://git.ouroath.com/vespa/hosted-feature-flags for more info on the JSON syntax + """, + exception.getMessage()); + } + + @Test + void handles_absent_rule_value() { + SystemFlagsDataArchive archive = fromDirectory("system-flags-with-null-value", true); + + // west has null value on first rule + List<FlagData> westFlagData = archive.flagData(prodUsWestCfgTarget); + assertEquals(1, westFlagData.size()); + assertEquals(2, westFlagData.get(0).rules().size()); + assertEquals(Optional.empty(), westFlagData.get(0).rules().get(0).getValueToApply()); + + // east has no value on first rule + List<FlagData> eastFlagData = archive.flagData(prodUsEast3CfgTarget); + assertEquals(1, eastFlagData.size()); + assertEquals(2, eastFlagData.get(0).rules().size()); + assertEquals(Optional.empty(), eastFlagData.get(0).rules().get(0).getValueToApply()); } @Test - void remove_comments() { - assertTrue(JSON.equals("{\n" + - " \"a\": {\n" + - " \"b\": 1\n" + - " },\n" + - " \"list\": [\n" + - " {\n" + - " \"c\": 2\n" + - " },\n" + - " {\n" + - " }\n" + - " ]\n" + - "}", - SystemFlagsDataArchive.normalizeJson("{\n" + - " \"comment\": \"comment a\",\n" + - " \"a\": {\n" + - " \"comment\": \"comment b\",\n" + - " \"b\": 1\n" + - " },\n" + - " \"list\": [\n" + - " {\n" + - " \"comment\": \"comment c\",\n" + - " \"c\": 2\n" + - " },\n" + - " {\n" + - " \"comment\": \"comment d\"\n" + - " }\n" + - " ]\n" + - "}", Set.of()))); + void remove_comments_and_null_value_in_rules() { + assertTrue(JSON.equals(""" + { + "id": "foo", + "rules": [ + { + "conditions": [ + { + "type": "whitelist", + "dimension": "hostname", + "values": [ "foo.com" ] + } + ] + }, + { + "conditions": [ + { + "type": "whitelist", + "dimension": "zone", + "values": [ "prod.us-west-1" ] + } + ] + }, + { + "conditions": [ + { + "type": "whitelist", + "dimension": "application", + "values": [ "f:o:o" ] + } + ], + "value": true + } + ] + }""", + normalizeJson(""" + { + "id": "foo", + "comment": "bar", + "rules": [ + { + "comment": "bar", + "conditions": [ + { + "comment": "bar", + "type": "whitelist", + "dimension": "hostname", + "values": [ "foo.com" ] + } + ], + "value": null + }, + { + "comment": "bar", + "conditions": [ + { + "comment": "bar", + "type": "whitelist", + "dimension": "zone", + "values": [ "prod.us-west-1" ] + } + ] + }, + { + "comment": "bar", + "conditions": [ + { + "comment": "bar", + "type": "whitelist", + "dimension": "application", + "values": [ "f:o:o" ] + } + ], + "value": true + } + ] + }"""))); + } + + private static String normalizeJson(String json) { + SystemFlagsDataArchive.Builder builder = new SystemFlagsDataArchive.Builder(); + assertTrue(builder.maybeAddFile(Path.of("flags/temporary/foo/default.json"), json, createZoneRegistryMock(), true)); + List<FlagData> flagData = builder.build().flagData(prodUsWestCfgTarget); + assertEquals(1, flagData.size()); + return JSON.canonical(flagData.get(0).serializeToJson()); } @Test void normalize_json_succeed_on_valid_values() { - normalizeJson("application", "\"a:b:c\""); - normalizeJson("cloud", "\"yahoo\""); - normalizeJson("cloud", "\"aws\""); - normalizeJson("cloud", "\"gcp\""); - normalizeJson("cluster-id", "\"some-id\""); - normalizeJson("cluster-type", "\"admin\""); - normalizeJson("cluster-type", "\"container\""); - normalizeJson("cluster-type", "\"content\""); - normalizeJson("console-user-email", "\"name@domain.com\""); - normalizeJson("environment", "\"prod\""); - normalizeJson("environment", "\"staging\""); - normalizeJson("environment", "\"test\""); - normalizeJson("hostname", "\"2080046-v6-11.ostk.bm2.prod.gq1.yahoo.com\""); - normalizeJson("node-type", "\"tenant\""); - normalizeJson("node-type", "\"host\""); - normalizeJson("node-type", "\"config\""); - normalizeJson("node-type", "\"host\""); - normalizeJson("system", "\"main\""); - normalizeJson("system", "\"public\""); - normalizeJson("tenant", "\"vespa\""); - normalizeJson("vespa-version", "\"8.201.13\""); - normalizeJson("zone", "\"prod.us-west-1\"", Set.of(ZoneId.from("prod.us-west-1"))); - } - - private void normalizeJson(String dimension, String jsonValue) { - normalizeJson(dimension, jsonValue, Set.of()); - } - - private void normalizeJson(String dimension, String jsonValue, Set<ZoneId> zones) { - SystemFlagsDataArchive.normalizeJson(""" + addFile(Condition.Type.WHITELIST, "application", "a:b:c"); + addFile(Condition.Type.WHITELIST, "cloud", "yahoo"); + addFile(Condition.Type.WHITELIST, "cloud", "aws"); + addFile(Condition.Type.WHITELIST, "cloud", "gcp"); + addFile(Condition.Type.WHITELIST, "cluster-id", "some-id"); + addFile(Condition.Type.WHITELIST, "cluster-type", "admin"); + addFile(Condition.Type.WHITELIST, "cluster-type", "container"); + addFile(Condition.Type.WHITELIST, "cluster-type", "content"); + addFile(Condition.Type.WHITELIST, "console-user-email", "name@domain.com"); + addFile(Condition.Type.WHITELIST, "environment", "prod"); + addFile(Condition.Type.WHITELIST, "environment", "staging"); + addFile(Condition.Type.WHITELIST, "environment", "test"); + addFile(Condition.Type.WHITELIST, "hostname", "2080046-v6-11.ostk.bm2.prod.gq1.yahoo.com"); + addFile(Condition.Type.WHITELIST, "node-type", "tenant"); + addFile(Condition.Type.WHITELIST, "node-type", "host"); + addFile(Condition.Type.WHITELIST, "node-type", "config"); + addFile(Condition.Type.WHITELIST, "node-type", "host"); + addFile(Condition.Type.WHITELIST, "system", "main"); + addFile(Condition.Type.WHITELIST, "system", "public"); + addFile(Condition.Type.WHITELIST, "tenant", "vespa"); + addFile(Condition.Type.RELATIONAL, "vespa-version", ">=8.201.13"); + addFile(Condition.Type.WHITELIST, "zone", "prod.us-west-1"); + } + + private void addFile(Condition.Type type, String dimension, String jsonValue) { + SystemFlagsDataArchive.Builder builder = new SystemFlagsDataArchive.Builder(); + + String valuesField = type == Condition.Type.RELATIONAL ? + "\"predicate\": \"%s\"".formatted(jsonValue) : + "\"values\": [ \"%s\" ]".formatted(jsonValue); + + assertTrue(builder.maybeAddFile(Path.of("flags/temporary/foo/default.json"), """ { "id": "foo", "rules": [ { "conditions": [ { - "type": "whitelist", + "type": "%s", "dimension": "%s", - "values": [ %s ] + %s } ], "value": true } ] } - """.formatted(dimension, jsonValue), zones); + """.formatted(type.toWire(), dimension, valuesField), + createZoneRegistryMock(), + true)); } @Test void normalize_json_fail_on_invalid_values() { - failNormalizeJson("application", "\"a.b.c\"", "Application ids must be on the form tenant:application:instance, but was a.b.c"); - failNormalizeJson("cloud", "\"foo\"", "Unknown cloud: foo"); - // failNormalizeJson("cluster-id", ... any String is valid - failNormalizeJson("cluster-type", "\"foo\"", "Illegal cluster type 'foo'"); - failNormalizeJson("console-user-email", "123", "Non-string value in console-user-email whitelist condition: 123"); - failNormalizeJson("environment", "\"foo\"", "'foo' is not a valid environment identifier"); - failNormalizeJson("hostname", "\"not:a:hostname\"", "hostname must match '(([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9-]{0,61}[A-Za-z0-9])\\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9-]{0,61}[A-Za-z0-9])\\.?', but got: 'not:a:hostname'"); - failNormalizeJson("node-type", "\"footype\"", "No enum constant com.yahoo.config.provision.NodeType.footype"); - failNormalizeJson("system", "\"bar\"", "'bar' is not a valid system"); - failNormalizeJson("tenant", "123", "Non-string value in tenant whitelist condition: 123"); - failNormalizeJson("vespa-version", "\"not-a-version\"", "Invalid version component in 'not-a-version'"); - failNormalizeJson("zone", "\"dev.non-existing-zone\"", Set.of(ZoneId.from("prod.example-region")), "Unknown zone: dev.non-existing-zone"); - } - - private void failNormalizeJson(String dimension, String jsonValue, String expectedExceptionMessage) { - failNormalizeJson(dimension, jsonValue, Set.of(), expectedExceptionMessage); + failAddFile(Condition.Type.WHITELIST, "application", "a.b.c", "In file flags/temporary/foo/default.json: Invalid application 'a.b.c' in whitelist condition: Application ids must be on the form tenant:application:instance, but was a.b.c"); + failAddFile(Condition.Type.WHITELIST, "cloud", "foo", "In file flags/temporary/foo/default.json: Unknown cloud: foo"); + // cluster-id: any String is valid + failAddFile(Condition.Type.WHITELIST, "cluster-type", "foo", "In file flags/temporary/foo/default.json: Invalid cluster-type 'foo' in whitelist condition: Illegal cluster type 'foo'"); + failAddFile(Condition.Type.WHITELIST, "console-user-email", "not-valid-email-address", "In file flags/temporary/foo/default.json: Invalid email address: not-valid-email-address"); + failAddFile(Condition.Type.WHITELIST, "environment", "foo", "In file flags/temporary/foo/default.json: Invalid environment 'foo' in whitelist condition: 'foo' is not a valid environment identifier"); + failAddFile(Condition.Type.WHITELIST, "hostname", "not:a:hostname", "In file flags/temporary/foo/default.json: Invalid hostname 'not:a:hostname' in whitelist condition: hostname must match '(([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9-]{0,61}[A-Za-z0-9])\\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9-]{0,61}[A-Za-z0-9])\\.?', but got: 'not:a:hostname'"); + failAddFile(Condition.Type.WHITELIST, "node-type", "footype", "In file flags/temporary/foo/default.json: Invalid node-type 'footype' in whitelist condition: No enum constant com.yahoo.config.provision.NodeType.footype"); + failAddFile(Condition.Type.WHITELIST, "system", "bar", "In file flags/temporary/foo/default.json: Invalid system 'bar' in whitelist condition: 'bar' is not a valid system"); + failAddFile(Condition.Type.WHITELIST, "tenant", "a tenant", "In file flags/temporary/foo/default.json: Invalid tenant 'a tenant' in whitelist condition: tenant name must match '[a-zA-Z0-9_-]{1,256}', but got: 'a tenant'"); + failAddFile(Condition.Type.WHITELIST, "vespa-version", "not-a-version", "In file flags/temporary/foo/default.json: whitelist vespa-version condition is not supported"); + failAddFile(Condition.Type.RELATIONAL, "vespa-version", ">7.1.2", "In file flags/temporary/foo/default.json: Major Vespa version must be at least 8: 7.1.2"); + failAddFile(Condition.Type.WHITELIST, "zone", "dev.%illegal", "In file flags/temporary/foo/default.json: Invalid zone 'dev.%illegal' in whitelist condition: region name must match '[a-z]([a-z0-9-]*[a-z0-9])*', but got: '%illegal'"); } - private void failNormalizeJson(String dimension, String jsonValue, Set<ZoneId> zones, String expectedExceptionMessage) { + private void failAddFile(Condition.Type type, String dimension, String jsonValue, String expectedExceptionMessage) { try { - normalizeJson(dimension, jsonValue, zones); + addFile(type, dimension, jsonValue); fail(); } catch (RuntimeException e) { assertEquals(expectedExceptionMessage, e.getMessage()); @@ -266,14 +387,16 @@ public class SystemFlagsDataArchiveTest { @Test void ignores_files_not_related_to_specified_system_definition() { - ZoneRegistry registry = createZoneRegistryMock(); - Path testDirectory = Paths.get("src/test/resources/system-flags-for-multiple-systems/"); - var archive = SystemFlagsDataArchive.fromDirectoryAndSystem(testDirectory, registry); + var archive = fromDirectory("system-flags-for-multiple-systems", false); assertFlagDataHasValue(archive, MY_TEST_FLAG, cdControllerTarget, "default"); // Would be 'cd.controller' if files for CD system were included assertFlagDataHasValue(archive, MY_TEST_FLAG, mainControllerTarget, "default"); assertFlagDataHasValue(archive, MY_TEST_FLAG, prodUsWestCfgTarget, "main.prod.us-west-1"); } + private SystemFlagsDataArchive fromDirectory(String testDirectory, boolean simulateInController) { + return SystemFlagsDataArchive.fromDirectory(Paths.get("src/test/resources/" + testDirectory), createZoneRegistryMock(), simulateInController); + } + @SuppressWarnings("unchecked") // workaround for mocking a method for generic return type private static ZoneRegistry createZoneRegistryMock() { // Cannot use the standard registry mock as it's located in controller-server module @@ -286,12 +409,21 @@ public class SystemFlagsDataArchiveTest { when(registryMock.systemZone()).thenReturn(zoneApi); when(registryMock.getConfigServerVipUri(any())).thenReturn(URI.create("http://localhost:8080/")); when(registryMock.getConfigServerHttpsIdentity(any())).thenReturn(new AthenzService("domain", "servicename")); + ZoneList zones = mockZoneList("prod.us-west-1", "prod.us-east-3"); + when(registryMock.zones()).thenReturn(zones); + ZoneList zonesIncludingSystem = mockZoneList("prod.us-west-1", "prod.us-east-3", "prod.controller"); + when(registryMock.zonesIncludingSystem()).thenReturn(zonesIncludingSystem); + return registryMock; + } + + @SuppressWarnings("unchecked") // workaround for mocking a method for generic return type + private static ZoneList mockZoneList(String... zones) { ZoneList zoneListMock = mock(ZoneList.class); when(zoneListMock.reachable()).thenReturn(zoneListMock); when(zoneListMock.all()).thenReturn(zoneListMock); - when(zoneListMock.zones()).thenReturn((List)List.of(new SimpleZone("prod.us-west-1"), new SimpleZone("prod.us-east-3"))); - when(registryMock.zones()).thenReturn(zoneListMock); - return registryMock; + List<? extends ZoneApi> zoneList = Stream.of(zones).map(SimpleZone::new).toList(); + when(zoneListMock.zones()).thenReturn((List) zoneList); + return zoneListMock; } private static void assertArchiveReturnsCorrectTestFlagDataForTarget(SystemFlagsDataArchive archive) { @@ -305,7 +437,7 @@ public class SystemFlagsDataArchiveTest { List<FlagData> data = getData(archive, flagId, target); assertEquals(1, data.size()); FlagData flagData = data.get(0); - RawFlag rawFlag = flagData.resolve(FetchVector.fromMap(Map.of())).get(); + RawFlag rawFlag = flagData.resolve(new FetchVector()).get(); assertEquals(String.format("\"%s\"", value), rawFlag.asJson()); } diff --git a/controller-api/src/test/resources/system-flags-with-null-value/flags/my-test-flag/main.prod.us-east-3.json b/controller-api/src/test/resources/system-flags-with-null-value/flags/my-test-flag/main.prod.us-east-3.json new file mode 100644 index 00000000000..b79e0913c22 --- /dev/null +++ b/controller-api/src/test/resources/system-flags-with-null-value/flags/my-test-flag/main.prod.us-east-3.json @@ -0,0 +1,24 @@ +{ + "id" : "my-test-flag", + "rules" : [ + { + "conditions": [ + { + "type": "whitelist", + "dimension": "application", + "values": ["a:b:c"] + } + ] + }, + { + "conditions": [ + { + "type": "whitelist", + "dimension": "hostname", + "values": ["foo.com"] + } + ], + "value" : true + } + ] +} diff --git a/controller-api/src/test/resources/system-flags-with-null-value/flags/my-test-flag/main.prod.us-west-1.json b/controller-api/src/test/resources/system-flags-with-null-value/flags/my-test-flag/main.prod.us-west-1.json new file mode 100644 index 00000000000..75cffdea009 --- /dev/null +++ b/controller-api/src/test/resources/system-flags-with-null-value/flags/my-test-flag/main.prod.us-west-1.json @@ -0,0 +1,25 @@ +{ + "id" : "my-test-flag", + "rules" : [ + { + "conditions": [ + { + "type": "whitelist", + "dimension": "application", + "values": ["a:b:c"] + } + ], + "value" : null + }, + { + "conditions": [ + { + "type": "whitelist", + "dimension": "hostname", + "values": ["foo.com"] + } + ], + "value" : true + } + ] +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/OsController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/OsController.java index c426c27418d..58c3b4da5e4 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/OsController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/OsController.java @@ -11,6 +11,7 @@ import com.yahoo.vespa.hosted.controller.versions.OsVersionStatus; import com.yahoo.vespa.hosted.controller.versions.OsVersionTarget; import java.time.Instant; +import java.util.Comparator; import java.util.HashSet; import java.util.Map; import java.util.Objects; @@ -161,9 +162,12 @@ public record OsController(Controller controller) { /** Remove certifications for non-existent OS versions */ public void removeStaleCertifications(OsVersionStatus currentStatus) { try (Mutex lock = curator().lockCertifiedOsVersions()) { - Set<OsVersion> knownVersions = currentStatus.versions().keySet(); + Optional<OsVersion> minKnownVersion = currentStatus.versions().keySet().stream() + .filter(v -> !v.version().isEmpty()) + .min(Comparator.naturalOrder()); + if (minKnownVersion.isEmpty()) return; Set<CertifiedOsVersion> certifiedVersions = new HashSet<>(readCertified()); - if (certifiedVersions.removeIf(cv -> !knownVersions.contains(cv.osVersion()))) { + if (certifiedVersions.removeIf(cv -> cv.osVersion().version().isBefore(minKnownVersion.get().version()))) { curator().writeCertifiedOsVersions(certifiedVersions); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/TestPackage.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/TestPackage.java index b4c9b2ebd57..790121b35dc 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/TestPackage.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/TestPackage.java @@ -97,7 +97,7 @@ public class TestPackage { keyPair = null; this.certificate = null; } - this.applicationPackageStream = new ApplicationPackageStream(inZip, () -> __ -> false, () -> new Replacer() { + this.applicationPackageStream = new ApplicationPackageStream(inZip, () -> name -> name.endsWith(".xml"), () -> new Replacer() { // Initially skips all declared entries, ensuring they're generated and appended after all input entries. final Map<String, UnaryOperator<InputStream>> entries = new HashMap<>(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java index 52206d41c00..998b72665d7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; -import com.yahoo.component.Version; +import ai.vespa.metrics.ControllerMetrics; import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterResources; @@ -71,8 +71,8 @@ public class ResourceMeterMaintainer extends ControllerMaintainer { private final Metric metric; private final Clock clock; - private static final String METERING_LAST_REPORTED = "metering_last_reported"; - private static final String METERING_TOTAL_REPORTED = "metering_total_reported"; + private static final String METERING_LAST_REPORTED = ControllerMetrics.METERING_LAST_REPORTED.baseName(); + private static final String METERING_TOTAL_REPORTED = ControllerMetrics.METERING_TOTAL_REPORTED.baseName(); private static final int METERING_REFRESH_INTERVAL_SECONDS = 1800; @SuppressWarnings("WeakerAccess") diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 6294fc59b5e..94c7829a851 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -979,7 +979,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { fingerprintObject.setString("fingerprint", tokenVersion.fingerPrint().value()); fingerprintObject.setString("created", tokenVersion.creationTime().toString()); fingerprintObject.setString("author", tokenVersion.author()); - fingerprintObject.setString("expiration", tokenVersion.expiration().map(Instant::toString).orElse("<none>")); + fingerprintObject.setString("expiration", tokenVersion.expiration().map(Instant::toString).orElse("none")); } } return new SlimeJsonResponse(slime); @@ -995,7 +995,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { tokenObject.setString("id", token.tokenId().value()); tokenObject.setString("token", token.tokenValue()); tokenObject.setString("fingerprint", token.fingerPrint().value()); - tokenObject.setString("expiration", token.expiration().map(Instant::toString).orElse("<none>")); + tokenObject.setString("expiration", token.expiration().map(Instant::toString).orElse("none")); return new SlimeJsonResponse(slime); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java index 43c8e7c9469..c526b335c90 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java @@ -23,7 +23,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.application.Change; -import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.deployment.ConvergenceSummary; import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus; @@ -53,7 +52,6 @@ import java.util.Locale; import java.util.Map; import java.util.NavigableMap; import java.util.Optional; -import java.util.SortedMap; import java.util.stream.Stream; import static com.yahoo.config.application.api.DeploymentSpec.UpgradePolicy.canary; @@ -109,10 +107,10 @@ class JobControllerApiHandlerHelper { int limit = limitStr.map(Integer::parseInt).orElse(Integer.MAX_VALUE); toSlime(cursor.setArray("runs"), runs.values(), application, limit, baseUriForJobType); - controller.applications().decideCloudAccountOf(new DeploymentId(id.application(), - runs.lastEntry().getValue().id().job().type().zone()), // Urgh, must use a job with actual zone. - application.deploymentSpec()) - .ifPresent(cloudAccount -> cursor.setObject("enclave").setString("cloudAccount", cloudAccount.value())); + Optional.ofNullable(runs.lastEntry()) + .map(entry -> new DeploymentId(id.application(), entry.getValue().id().job().type().zone())) // Urgh, must use a job with actual zone. + .flatMap(deployment -> controller.applications().decideCloudAccountOf(deployment, application.deploymentSpec())) + .ifPresent(cloudAccount -> cursor.setObject("enclave").setString("cloudAccount", cloudAccount.value())); return new SlimeJsonResponse(slime); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployer.java index 355f06fc753..2c38066eddd 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployer.java @@ -8,6 +8,7 @@ import com.yahoo.vespa.flags.FlagId; import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.json.FlagData; import com.yahoo.vespa.hosted.controller.api.integration.ControllerIdentityProvider; +import com.yahoo.vespa.hosted.controller.api.systemflags.v1.FlagValidationException; import com.yahoo.vespa.hosted.controller.api.systemflags.v1.FlagsTarget; import com.yahoo.vespa.hosted.controller.api.systemflags.v1.SystemFlagsDataArchive; import com.yahoo.vespa.hosted.controller.restapi.systemflags.SystemFlagsDeployResult.OperationError; @@ -57,6 +58,12 @@ class SystemFlagsDeployer { } SystemFlagsDeployResult deployFlags(SystemFlagsDataArchive archive, boolean dryRun) { + try { + archive.validateAllFilesAreForTargets(targets); + } catch (FlagValidationException e) { + return new SystemFlagsDeployResult(List.of(OperationError.archiveValidationFailed(e.getMessage()))); + } + Map<FlagsTarget, Future<SystemFlagsDeployResult>> futures = new HashMap<>(); for (FlagsTarget target : targets) { futures.put(target, executor.submit(() -> deployFlags(target, archive.flagData(target), dryRun))); @@ -70,11 +77,6 @@ class SystemFlagsDeployer { throw new RuntimeException(e); } }); - try { - archive.validateAllFilesAreForTargets(system, targets); - } catch (IllegalArgumentException e) { - results.add(new SystemFlagsDeployResult(List.of(OperationError.archiveValidationFailed(e.getMessage())))); - } return SystemFlagsDeployResult.merge(results); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsHandler.java index e9b087690ff..bb285b8b742 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsHandler.java @@ -10,10 +10,13 @@ import com.yahoo.restapi.JacksonJsonResponse; import com.yahoo.restapi.Path; import com.yahoo.vespa.hosted.controller.api.integration.ControllerIdentityProvider; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; +import com.yahoo.vespa.hosted.controller.api.systemflags.v1.FlagValidationException; import com.yahoo.vespa.hosted.controller.api.systemflags.v1.FlagsTarget; import com.yahoo.vespa.hosted.controller.api.systemflags.v1.SystemFlagsDataArchive; import com.yahoo.vespa.hosted.controller.restapi.ErrorResponses; +import java.io.InputStream; +import java.util.List; import java.util.concurrent.Executor; /** @@ -27,12 +30,14 @@ public class SystemFlagsHandler extends ThreadedHttpRequestHandler { private static final String API_PREFIX = "/system-flags/v1"; private final SystemFlagsDeployer deployer; + private final ZoneRegistry zoneRegistry; @Inject public SystemFlagsHandler(ZoneRegistry zoneRegistry, ControllerIdentityProvider identityProvider, Executor executor) { super(executor); + this.zoneRegistry = zoneRegistry; this.deployer = new SystemFlagsDeployer(identityProvider, zoneRegistry.system(), FlagsTarget.getAllTargetsInSystem(zoneRegistry, true)); } @@ -57,12 +62,22 @@ public class SystemFlagsHandler extends ThreadedHttpRequestHandler { if (!contentType.equalsIgnoreCase("application/zip")) { return ErrorResponse.badRequest("Invalid content type: " + contentType); } - SystemFlagsDataArchive archive = SystemFlagsDataArchive.fromZip(request.getData()); - SystemFlagsDeployResult result = deployer.deployFlags(archive, dryRun); + SystemFlagsDeployResult result = deploy(request.getData(), dryRun); return new JacksonJsonResponse<>(200, result.toWire()); } catch (Exception e) { return ErrorResponses.logThrowing(request, log, e); } } + private SystemFlagsDeployResult deploy(InputStream zipStream, boolean dryRun) { + SystemFlagsDataArchive archive; + try { + archive = SystemFlagsDataArchive.fromZip(zipStream, zoneRegistry); + } catch (FlagValidationException e) { + return new SystemFlagsDeployResult(List.of(SystemFlagsDeployResult.OperationError.archiveValidationFailed(e.getMessage()))); + } + + return deployer.deployFlags(archive, dryRun); + } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/TestPackageTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/TestPackageTest.java index f529d81bf32..c948da6936c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/TestPackageTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/TestPackageTest.java @@ -148,8 +148,8 @@ public class TestPackageTest { "components/foo-tests.jar", "artifacts/key"), bundlePackage.keySet()); - assertEquals(Map.of(), - unzip(bundleTests.asApplicationPackage().truncatedPackage().zippedContent())); + assertEquals(Set.of("deployment.xml", "services.xml"), + unzip(bundleTests.asApplicationPackage().truncatedPackage().zippedContent()).keySet()); } @Test diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java index 63d479d4c6c..dbb7f80df0e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java @@ -61,8 +61,8 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry public ZoneRegistryMock(SystemName system) { this.system = system; if (system.isPublic()) { - this.zones = List.of(ZoneApiMock.fromId("test.us-east-1"), - ZoneApiMock.fromId("staging.us-east-3"), + this.zones = List.of(ZoneApiMock.newBuilder().withId("test.us-east-1").withCloud("aws").withCloudNativeAvailabilityZone("use1-az4").build(), + ZoneApiMock.newBuilder().withId("staging.us-east-3").withCloud("aws").withCloudNativeAvailabilityZone("use3-az1").build(), ZoneApiMock.newBuilder().withId("prod.aws-us-east-1c").withCloud("aws").withCloudNativeAvailabilityZone("use1-az2").build(), ZoneApiMock.newBuilder().withId("prod.aws-eu-west-1a").withCloud("aws").withCloudNativeAvailabilityZone("euw1-az3").build(), ZoneApiMock.newBuilder().withId("dev.aws-us-east-1c").withCloud("aws").withCloudNativeAvailabilityZone("use1-az2").build()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdaterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdaterTest.java index af535abce26..6f4052bf0ef 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdaterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdaterTest.java @@ -68,12 +68,14 @@ public class OsVersionStatusUpdaterTest { .filter(osVersion -> !osVersion.version().isEmpty()) .collect(Collectors.toSet()); List<OsVersion> versionsToCertify = new ArrayList<>(knownVersions); - versionsToCertify.addAll(List.of(new OsVersion(Version.fromString("95.0.1"), cloud), - new OsVersion(Version.fromString("98.0.2"), cloud))); + OsVersion futureVersion = new OsVersion(Version.fromString("98.0.2"), cloud); // Keep future version + versionsToCertify.addAll(List.of(new OsVersion(Version.fromString("3.11"), cloud), + futureVersion)); for (OsVersion version : versionsToCertify) { tester.controller().os().certify(version.version(), version.cloud(), Version.fromString("1.2.3")); } - assertEquals(knownVersions.size() + 2, certifiedOsVersions(tester).size()); + knownVersions.add(futureVersion); + assertEquals(knownVersions.size() + 1, certifiedOsVersions(tester).size()); statusUpdater.maintain(); assertEquals(knownVersions, certifiedOsVersions(tester)); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java index 93937bdc4af..905330c6daf 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java @@ -14,13 +14,10 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TestReport; -import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; -import com.yahoo.vespa.hosted.controller.notification.Notification.Type; -import com.yahoo.vespa.hosted.controller.notification.NotificationSource; import org.junit.jupiter.api.Test; import java.io.ByteArrayOutputStream; @@ -34,6 +31,7 @@ import java.util.List; import java.util.Optional; import static com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException.ErrorCode.INVALID_APPLICATION_PACKAGE; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.applicationPackage; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.devAwsUsEast2a; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.devUsEast1; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.productionUsCentral1; @@ -42,8 +40,6 @@ import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.pro import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.stagingTest; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.systemTest; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.testUsCentral1; -import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.applicationPackage; -import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.deploymentFailed; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.installationFailed; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.invalidApplication; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.running; @@ -208,16 +204,18 @@ public class JobControllerApiHandlerHelperTest { void testEnclave() { var cloudAccount = CloudAccount.from("aws:123456789012"); var applicationPackage = new ApplicationPackageBuilder() + .cloudAccount(cloudAccount.value()) .stagingTest() .systemTest() - .region("aws-us-east-1c", cloudAccount.value()) + .region("aws-us-east-1c") .build(); var tester = new DeploymentTester(new ControllerTester(SystemName.Public)); tester.controllerTester().flagSource().withListFlag(PermanentFlags.CLOUD_ACCOUNTS.id(), List.of(cloudAccount.value()), String.class); - tester.controllerTester().zoneRegistry().configureCloudAccount(cloudAccount, ZoneId.from("prod.aws-us-east-1c")); + tester.controllerTester().zoneRegistry().configureCloudAccount(cloudAccount, systemTest.zone(), stagingTest.zone(), ZoneId.from("prod.aws-us-east-1c")); var app = tester.newDeploymentContext(); app.submit(applicationPackage).deploy(); + assertEquals(Optional.of(cloudAccount), tester.controllerTester().configServer().cloudAccount(app.deploymentIdIn(systemTest.zone()))); assertResponse(JobControllerApiHandlerHelper.overviewResponse(tester.controller(), app.application().id(), URI.create("https://some.url:43/root/")), "overview-enclave.json"); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview-enclave.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview-enclave.json index 3673c1bdf07..9d82ed97849 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview-enclave.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview-enclave.json @@ -5,11 +5,11 @@ "steps": [ { "type": "instance", - "dependencies": [ ], + "dependencies": [], "declared": true, "instance": "default", "readyAt": 0, - "deploying": { }, + "deploying": {}, "latestVersions": { "platform": { "platform": "6.1.0", @@ -21,7 +21,7 @@ "upgrade": false } ], - "blockers": [ ] + "blockers": [] }, "application": { "application": { @@ -42,21 +42,24 @@ } } ], - "blockers": [ ] + "blockers": [] } }, "delayCause": null }, { "type": "test", - "dependencies": [ ], + "dependencies": [], "declared": true, "instance": "default", "readyAt": 0, "jobName": "staging-test", "url": "https://some.url:43/instance/default/job/staging-test", "environment": "staging", - "toRun": [ ], + "toRun": [], + "enclave": { + "cloudAccount": "aws:123456789012" + }, "runs": [ { "id": 1, @@ -137,14 +140,17 @@ }, { "type": "test", - "dependencies": [ ], + "dependencies": [], "declared": true, "instance": "default", "readyAt": 0, "jobName": "system-test", "url": "https://some.url:43/instance/default/job/system-test", "environment": "test", - "toRun": [ ], + "toRun": [], + "enclave": { + "cloudAccount": "aws:123456789012" + }, "runs": [ { "id": 1, @@ -209,11 +215,7 @@ }, { "type": "deployment", - "dependencies": [ - 0, - 1, - 2 - ], + "dependencies": [0, 1, 2], "declared": true, "instance": "default", "readyAt": 1600000000000, @@ -228,7 +230,7 @@ "sourceUrl": "repository1/tree/commit1", "commit": "commit1" }, - "toRun": [ ], + "toRun": [], "enclave": { "cloudAccount": "aws:123456789012" }, diff --git a/eval/src/vespa/eval/eval/value_codec.cpp b/eval/src/vespa/eval/eval/value_codec.cpp index d5fc2eb7c16..19cf2012bcb 100644 --- a/eval/src/vespa/eval/eval/value_codec.cpp +++ b/eval/src/vespa/eval/eval/value_codec.cpp @@ -335,7 +335,9 @@ std::unique_ptr<Value> decode_value(nbostream &input, const ValueBuilderFactory std::unique_ptr<Value> value_from_spec(const TensorSpec &spec, const ValueBuilderFactory &factory) { ValueType type = ValueType::from_spec(spec.type()); - assert(!type.is_error()); + if (type.is_error()) { + throw IllegalArgumentException(fmt("Failed decoding value type from tensorspec(%s)", spec.type().c_str()), VESPA_STRLOC); + } return typify_invoke<1,TypifyCellType,CreateValueFromTensorSpec>(type.cell_type(), type, spec, factory); } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 73283958cc7..c6d141764fb 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -14,8 +14,6 @@ import java.util.TreeMap; import java.util.function.Predicate; import static com.yahoo.vespa.flags.FetchVector.Dimension.APPLICATION_ID; -import static com.yahoo.vespa.flags.FetchVector.Dimension.CLUSTER_ID; -import static com.yahoo.vespa.flags.FetchVector.Dimension.CLUSTER_TYPE; import static com.yahoo.vespa.flags.FetchVector.Dimension.CONSOLE_USER_EMAIL; import static com.yahoo.vespa.flags.FetchVector.Dimension.HOSTNAME; import static com.yahoo.vespa.flags.FetchVector.Dimension.NODE_TYPE; @@ -47,15 +45,6 @@ public class Flags { private static volatile TreeMap<FlagId, FlagDefinition> flags = new TreeMap<>(); - public static final UnboundBooleanFlag DROP_CACHES = defineFeatureFlag( - "drop-caches", false, - List.of("hakonhall", "baldersheim"), "2023-03-06", "2023-08-05", - "Drop caches on tenant hosts", - "Takes effect on next tick", - // The application ID is the exclusive application ID associated with the host, - // if any, or otherwise hosted-vespa:tenant-host:default. - APPLICATION_ID, TENANT_ID, CLUSTER_ID, CLUSTER_TYPE); - public static final UnboundDoubleFlag DEFAULT_TERM_WISE_LIMIT = defineDoubleFlag( "default-term-wise-limit", 1.0, List.of("baldersheim"), "2020-12-02", "2023-12-31", @@ -105,27 +94,6 @@ public class Flags { "Takes effect at redeployment", APPLICATION_ID); - public static final UnboundBooleanFlag SKIP_COMMUNICATIONMANAGER_THREAD = defineFeatureFlag( - "skip-communicationmanager-thread", false, - List.of("baldersheim"), "2020-12-02", "2023-12-31", - "Should we skip the communicationmanager thread", - "Takes effect at redeployment", - APPLICATION_ID); - - public static final UnboundBooleanFlag SKIP_MBUS_REQUEST_THREAD = defineFeatureFlag( - "skip-mbus-request-thread", false, - List.of("baldersheim"), "2020-12-02", "2023-12-31", - "Should we skip the mbus request thread", - "Takes effect at redeployment", - APPLICATION_ID); - - public static final UnboundBooleanFlag SKIP_MBUS_REPLY_THREAD = defineFeatureFlag( - "skip-mbus-reply-thread", false, - List.of("baldersheim"), "2020-12-02", "2023-12-31", - "Should we skip the mbus reply thread", - "Takes effect at redeployment", - APPLICATION_ID); - public static final UnboundBooleanFlag USE_ASYNC_MESSAGE_HANDLING_ON_SCHEDULE = defineFeatureFlag( "async-message-handling-on-schedule", false, List.of("baldersheim"), "2020-12-02", "2023-12-31", @@ -222,7 +190,7 @@ public class Flags { // TODO: Move to a permanent flag public static final UnboundListFlag<String> ALLOWED_ATHENZ_PROXY_IDENTITIES = defineListFlag( "allowed-athenz-proxy-identities", List.of(), String.class, - List.of("bjorncs", "tokle"), "2021-02-10", "2023-08-01", + List.of("bjorncs", "tokle"), "2021-02-10", "2023-09-01", "Allowed Athenz proxy identities", "takes effect at redeployment"); @@ -263,13 +231,6 @@ public class Flags { TENANT_ID, CONSOLE_USER_EMAIL ); - public static final UnboundBooleanFlag IGNORE_THREAD_STACK_SIZES = defineFeatureFlag( - "ignore-thread-stack-sizes", false, - List.of("arnej"), "2021-11-12", "2023-12-31", - "Whether C++ thread creation should ignore any requested stack size", - "Triggers restart, takes effect immediately", - APPLICATION_ID); - public static final UnboundBooleanFlag USE_V8_GEO_POSITIONS = defineFeatureFlag( "use-v8-geo-positions", true, List.of("arnej"), "2021-11-15", "2023-12-31", @@ -284,23 +245,9 @@ public class Flags { "Takes effect at redeployment", APPLICATION_ID); - public static final UnboundBooleanFlag USE_QRSERVER_SERVICE_NAME = defineFeatureFlag( - "use-qrserver-service-name", false, - List.of("arnej"), "2022-01-18", "2023-12-31", - "Use backwards-compatible 'qrserver' service name for containers with only 'search' API", - "Takes effect at redeployment", - APPLICATION_ID); - - public static final UnboundBooleanFlag AVOID_RENAMING_SUMMARY_FEATURES = defineFeatureFlag( - "avoid-renaming-summary-features", true, - List.of("arnej"), "2022-01-15", "2023-12-31", - "Tell backend about the original name of summary-features that were wrapped in a rankingExpression feature", - "Takes effect at redeployment", - APPLICATION_ID); - public static final UnboundBooleanFlag ENABLE_PROXY_PROTOCOL_MIXED_MODE = defineFeatureFlag( "enable-proxy-protocol-mixed-mode", true, - List.of("tokle"), "2022-05-09", "2023-08-01", + List.of("tokle"), "2022-05-09", "2023-09-01", "Enable or disable proxy protocol mixed mode", "Takes effect on redeployment", APPLICATION_ID); @@ -329,7 +276,7 @@ public class Flags { public static final UnboundBooleanFlag RESTRICT_DATA_PLANE_BINDINGS = defineFeatureFlag( "restrict-data-plane-bindings", false, - List.of("mortent"), "2022-09-08", "2023-08-01", + List.of("mortent"), "2022-09-08", "2023-09-01", "Use restricted data plane bindings", "Takes effect at redeployment", APPLICATION_ID); @@ -355,14 +302,6 @@ public class Flags { "Takes effect at redeployment", APPLICATION_ID); - public static final UnboundBooleanFlag NODE_ADMIN_TENANT_SERVICE_REGISTRY = defineFeatureFlag( - "node-admin-tenant-service-registry", true, - List.of("olaa"), "2023-04-12", "2023-08-07", - "Whether AthenzCredentialsMaintainer in node-admin should create tenant service identity certificate", - "Takes effect on next tick", - HOSTNAME, VESPA_VERSION, APPLICATION_ID - ); - public static final UnboundBooleanFlag ENABLE_CROWDSTRIKE = defineFeatureFlag( "enable-crowdstrike", true, List.of("andreer"), "2023-04-13", "2023-08-31", "Whether to enable CrowdStrike.", "Takes effect on next host admin tick", @@ -386,6 +325,11 @@ public class Flags { "Takes effect on next run of CertPoolMaintainer" ); + public static final UnboundStringFlag CONTAINER_IMAGE_PULL_IO_MAX = defineStringFlag( + "container-image-pull-io-max", "", List.of("freva"), "2023-08-04", "2023-09-15", + "The value (excluding the device name) of io.max cgroup used by container image pull, e.g. 'wiops=100', or 'wbps=10000 riops=20', or empty for unlimited", + "Takes effect at next host-admin tick"); + public static final UnboundBooleanFlag ENABLE_THE_ONE_THAT_SHOULD_NOT_BE_NAMED = defineFeatureFlag( "enable-the-one-that-should-not-be-named", false, List.of("hmusum"), "2023-05-08", "2023-08-15", "Whether to enable the one program that should not be named", @@ -397,18 +341,9 @@ public class Flags { "Where specified, CNAME records are used instead of the default ALIAS records, which have a default 60s TTL.", "Takes effect at redeployment from controller"); - public static final UnboundBooleanFlag ENABLE_CONDITIONAL_PUT_REMOVE_WRITE_REPAIR = defineFeatureFlag( - "enable-conditional-put-remove-write-repair", true, - List.of("vekterli", "havardpe"), "2023-05-10", "2023-09-01", - "If set, a conditional Put or Remove operation for a document in an inconsistent bucket " + - "will initiate a write-repair that evaluates the condition across all mutually inconsistent " + - "replicas, with the newest document version (if any) being authoritative", - "Takes effect at redeployment", - APPLICATION_ID); - public static final UnboundBooleanFlag ENABLE_DATAPLANE_PROXY = defineFeatureFlag( "enable-dataplane-proxy", false, - List.of("mortent", "olaa"), "2023-05-15", "2023-08-01", + List.of("mortent", "olaa"), "2023-05-15", "2023-09-01", "Whether to enable dataplane proxy", "Takes effect at redeployment", APPLICATION_ID diff --git a/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java b/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java index 18f5f5f860d..5a791522977 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java @@ -21,6 +21,7 @@ import static com.yahoo.vespa.flags.FetchVector.Dimension.HOSTNAME; import static com.yahoo.vespa.flags.FetchVector.Dimension.NODE_TYPE; import static com.yahoo.vespa.flags.FetchVector.Dimension.TENANT_ID; import static com.yahoo.vespa.flags.FetchVector.Dimension.VESPA_VERSION; +import static com.yahoo.vespa.flags.FetchVector.Dimension.ZONE_ID; /** * Definition for permanent feature flags @@ -146,6 +147,11 @@ public class PermanentFlags { "Takes effect on next deployment from controller", APPLICATION_ID); + public static final UnboundBooleanFlag SEND_LIMITED_METRIC_SET = defineFeatureFlag( + "send-limited-metric-set", true, + "Whether a limited metric set should be fetched from metrics-proxy (CD systems only)", + "Takes effect on next host admin tick"); + private static final String VERSION_QUALIFIER_REGEX = "[a-zA-Z0-9_-]+"; private static final Pattern QUALIFIER_PATTERN = Pattern.compile("^" + VERSION_QUALIFIER_REGEX + "$"); private static final Pattern VERSION_PATTERN = Pattern.compile("^\\d\\.\\d\\.\\d(\\." + VERSION_QUALIFIER_REGEX + ")?$"); @@ -346,6 +352,7 @@ public class PermanentFlags { "Takes effect immediately", TENANT_ID); + // TODO: Remove when not in use anymore, replaced by KEEP_FILE_REFERENCES_DAYS public static final UnboundIntFlag KEEP_FILE_REFERENCES_ON_TENANT_NODES = defineIntFlag( "keep-file-references-on-tenant-nodes", 30, "How many days to keep file references on tenant nodes (based on last modification time)", @@ -353,6 +360,20 @@ public class PermanentFlags { APPLICATION_ID ); + public static final UnboundIntFlag KEEP_FILE_REFERENCES_DAYS = defineIntFlag( + "keep-file-references-days", 30, + "How many days to keep file references on tenant nodes (based on last modification time)", + "Takes effect on restart of Docker container", + APPLICATION_ID + ); + + public static final UnboundIntFlag KEEP_FILE_REFERENCES_COUNT = defineIntFlag( + "keep-file-references-count", 20, + "How many file references to keep on tenant nodes (no matter what last modification time is)", + "Takes effect on restart of Docker container", + ZONE_ID, APPLICATION_ID + ); + public static final UnboundIntFlag ENDPOINT_CONNECTION_TTL = defineIntFlag( "endpoint-connection-ttl", 45, "Time to live for connections to endpoints in seconds", @@ -371,6 +392,14 @@ public class PermanentFlags { "triggered", "Takes effect immediately"); + public static final UnboundBooleanFlag DROP_CACHES = defineFeatureFlag( + "drop-caches", false, + "Drop caches on tenant hosts", + "Takes effect on next tick", + // The application ID is the exclusive application ID associated with the host, + // if any, or otherwise hosted-vespa:tenant-host:default. + APPLICATION_ID, TENANT_ID, CLUSTER_ID, CLUSTER_TYPE); + private PermanentFlags() {} private static UnboundBooleanFlag defineFeatureFlag( diff --git a/flags/src/main/java/com/yahoo/vespa/flags/json/FlagData.java b/flags/src/main/java/com/yahoo/vespa/flags/json/FlagData.java index acda3b9db42..bc103c2d68a 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/json/FlagData.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/json/FlagData.java @@ -69,6 +69,7 @@ public class FlagData { } } } + newRules = optimizeRules(newRules); FetchVector newDefaultFetchVector = defaultFetchVector.without(fetchVector.dimensions()); @@ -188,7 +189,26 @@ public class FlagData { private static List<Rule> rulesFromWire(List<WireRule> wireRules) { if (wireRules == null) return List.of(); - return wireRules.stream().map(Rule::fromWire).toList(); + return optimizeRules(wireRules.stream().map(Rule::fromWire).toList()); + } + + /** Take a raw list of rules from e.g. deserialization or partial resolution and normalize/simplify it. */ + private static List<Rule> optimizeRules(List<Rule> rules) { + // Remove trailing rules without value, as absent value implies the code default. + // Removing trailing rules may further simplify when e.g. this results in no rules, + // which is equivalent to no flag data at all, and flag data may be deleted from a zone. + if (rules.isEmpty()) return rules; + if (rules.get(rules.size() - 1).getValueToApply().isPresent()) return rules; + var newRules = new ArrayList<>(rules); + while (newRules.size() > 0) { + Rule lastRule = newRules.get(newRules.size() - 1); + if (lastRule.getValueToApply().isEmpty()) { + newRules.remove(newRules.size() - 1); + } else { + break; + } + } + return newRules; } } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/json/RelationalCondition.java b/flags/src/main/java/com/yahoo/vespa/flags/json/RelationalCondition.java index 749f6830870..031b61c8e7e 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/json/RelationalCondition.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/json/RelationalCondition.java @@ -68,6 +68,10 @@ public class RelationalCondition implements Condition { return fetchVector.getValue(dimension).map(predicate::test).orElse(false); } + public RelationalPredicate relationalPredicate() { + return relationalPredicate; + } + @Override public WireCondition toWire() { var condition = new WireCondition(); diff --git a/flags/src/main/java/com/yahoo/vespa/flags/json/Rule.java b/flags/src/main/java/com/yahoo/vespa/flags/json/Rule.java index 127c2b4f9da..b1b15faa938 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/json/Rule.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/json/Rule.java @@ -83,9 +83,11 @@ public class Rule { public static Rule fromWire(WireRule wireRule) { List<Condition> conditions = wireRule.andConditions == null ? - Collections.emptyList() : + List.of() : wireRule.andConditions.stream().map(Condition::fromWire).toList(); - Optional<RawFlag> value = wireRule.value == null ? Optional.empty() : Optional.of(JsonNodeRawFlag.fromJsonNode(wireRule.value)); + Optional<RawFlag> value = wireRule.value == null || wireRule.value.isNull() ? + Optional.empty() : + Optional.of(JsonNodeRawFlag.fromJsonNode(wireRule.value)); return new Rule(value, conditions); } diff --git a/flags/src/test/java/com/yahoo/vespa/flags/json/FlagDataTest.java b/flags/src/test/java/com/yahoo/vespa/flags/json/FlagDataTest.java index c7da1abe7e2..3ca7f59c759 100644 --- a/flags/src/test/java/com/yahoo/vespa/flags/json/FlagDataTest.java +++ b/flags/src/test/java/com/yahoo/vespa/flags/json/FlagDataTest.java @@ -1,10 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.flags.json; +import com.yahoo.text.JSON; import com.yahoo.vespa.flags.FetchVector; import com.yahoo.vespa.flags.RawFlag; import org.junit.jupiter.api.Test; +import java.util.List; import java.util.Optional; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -212,6 +214,62 @@ public class FlagDataTest { assertTrue(fullyResolved.isEmpty()); } + @Test + void testRemovalOfSentinelRuleWithNullValue() { + FlagData data = FlagData.deserialize(""" + { + "id": "id1", + "rules": [ + { + "conditions": [ + { + "type": "whitelist", + "dimension": "zone", + "values": [ "zone1", "zone2" ] + } + ], + "value": null + } + ] + }"""); + assertEquals(data, new FlagData(data.id(), new FetchVector(), List.of())); + assertTrue(data.isEmpty()); + } + + @Test + void testRemovalOfSentinelRuleWithoutValue() { + String json = """ + { + "id": "id1", + "rules": [ + { + "conditions": [ + { + "type": "whitelist", + "dimension": "zone", + "values": [ "zone1", "zone2" ] + } + ] + }, + { + "conditions": [ + { + "type": "whitelist", + "dimension": "cloud", + "values": [ "aws" ] + } + ], + "value": true + } + ] + }"""; + FlagData data = FlagData.deserialize(json); + assertTrue(JSON.equals(data.serializeToJson(), json)); + FlagData flagData = data.partialResolve(vector.with(FetchVector.Dimension.CLOUD, "gcp")); + assertEquals(flagData, new FlagData(data.id(), new FetchVector(), List.of())); + assertTrue(flagData.isEmpty()); + } + private void verify(Optional<String> expectedValue, FetchVector vector) { FlagData data = FlagData.deserialize(json); assertEquals("id1", data.id().toString()); diff --git a/flags/src/test/java/com/yahoo/vespa/flags/json/SerializationTest.java b/flags/src/test/java/com/yahoo/vespa/flags/json/SerializationTest.java index 2cc19917793..2192739c96c 100644 --- a/flags/src/test/java/com/yahoo/vespa/flags/json/SerializationTest.java +++ b/flags/src/test/java/com/yahoo/vespa/flags/json/SerializationTest.java @@ -108,24 +108,25 @@ public class SerializationTest { @Test void jsonWithStrayFields() { - String json = "{\n" + - " \"id\": \"id3\",\n" + - " \"foo\": true,\n" + - " \"rules\": [\n" + - " {\n" + - " \"conditions\": [\n" + - " {\n" + - " \"type\": \"whitelist\",\n" + - " \"dimension\": \"zone\",\n" + - " \"bar\": \"zoo\"\n" + - " }\n" + - " ],\n" + - " \"other\": true\n" + - " }\n" + - " ],\n" + - " \"attributes\": {\n" + - " }\n" + - "}"; + String json = """ + { + "id": "id3", + "foo": true, + "rules": [ + { + "conditions": [ + { + "type": "whitelist", + "dimension": "zone", + "bar": "zoo" + } + ], + "other": true + } + ], + "attributes": { + } + }"""; WireFlagData wireData = WireFlagData.deserialize(json); @@ -140,6 +141,6 @@ public class SerializationTest { assertThat(wireData.serializeToJson(), equalTo("{\"id\":\"id3\",\"rules\":[{\"conditions\":[{\"type\":\"whitelist\",\"dimension\":\"zone\"}]}],\"attributes\":{}}")); - assertThat(FlagData.deserialize(json).serializeToJson(), equalTo("{\"id\":\"id3\",\"rules\":[{\"conditions\":[{\"type\":\"whitelist\",\"dimension\":\"zone\"}]}]}")); + assertThat(FlagData.deserialize(json).serializeToJson(), equalTo("{\"id\":\"id3\"}")); } } diff --git a/fnet/src/tests/sync_execute/sync_execute.cpp b/fnet/src/tests/sync_execute/sync_execute.cpp index 5d2f4097ab4..0dd65b08874 100644 --- a/fnet/src/tests/sync_execute/sync_execute.cpp +++ b/fnet/src/tests/sync_execute/sync_execute.cpp @@ -17,6 +17,8 @@ TEST("sync execute") { DoIt exe2; DoIt exe3; DoIt exe4; + DoIt exe5; + DoIt exe6; FNET_Transport transport; ASSERT_TRUE(transport.execute(&exe1)); ASSERT_TRUE(transport.Start()); @@ -26,14 +28,16 @@ TEST("sync execute") { ASSERT_TRUE(exe2.gate.getCount() == 0u); ASSERT_TRUE(transport.execute(&exe3)); transport.ShutDown(false); - ASSERT_TRUE(!transport.execute(&exe4)); + uint32_t expect_cnt_4 = transport.execute(&exe4) ? 0 : 1; transport.sync(); transport.WaitFinished(); + ASSERT_TRUE(!transport.execute(&exe5)); transport.sync(); ASSERT_TRUE(exe1.gate.getCount() == 0u); ASSERT_TRUE(exe2.gate.getCount() == 0u); ASSERT_TRUE(exe3.gate.getCount() == 0u); - ASSERT_TRUE(exe4.gate.getCount() == 1u); + ASSERT_TRUE(exe4.gate.getCount() == expect_cnt_4); + ASSERT_TRUE(exe5.gate.getCount() == 1u); } TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/fnet/src/vespa/fnet/transport_thread.cpp b/fnet/src/vespa/fnet/transport_thread.cpp index 0b0df02c04c..217738b7364 100644 --- a/fnet/src/vespa/fnet/transport_thread.cpp +++ b/fnet/src/vespa/fnet/transport_thread.cpp @@ -119,7 +119,7 @@ FNET_TransportThread::PostEvent(FNET_ControlPacket *cpacket, size_t qLen; { std::unique_lock<std::mutex> guard(_lock); - if (IsShutDown()) { + if (_reject_events) { guard.unlock(); DiscardEvent(cpacket, context); return false; @@ -243,7 +243,8 @@ FNET_TransportThread::FNET_TransportThread(FNET_Transport &owner_in) _started(false), _shutdown(false), _finished(false), - _detaching() + _detaching(), + _reject_events(false) { trapsigpipe(); } @@ -384,9 +385,9 @@ FNET_TransportThread::ShutDown(bool waitFinished) bool wasEmpty = false; { std::lock_guard<std::mutex> guard(_lock); - if (!IsShutDown()) { + if (!should_shut_down()) { _shutdown.store(true, std::memory_order_relaxed); - wasEmpty = _queue.IsEmpty_NoLock(); + wasEmpty = _queue.IsEmpty_NoLock(); } } if (wasEmpty) { @@ -503,7 +504,7 @@ FNET_TransportThread::handle_event(FNET_IOComponent &ctx, bool read, bool write) bool FNET_TransportThread::EventLoopIteration() { - if (!IsShutDown()) { + if (!should_shut_down()) { int msTimeout = vespalib::count_ms(time_tools().event_timeout()); // obtain I/O events _selector.poll(msTimeout); @@ -530,7 +531,7 @@ FNET_TransportThread::EventLoopIteration() { FlushDeleteList(); } // -- END OF MAIN EVENT LOOP -- - if (!IsShutDown()) + if (!should_shut_down()) return true; if (is_finished()) return false; @@ -552,10 +553,22 @@ FNET_TransportThread::checkTimedoutComponents(vespalib::duration timeout) { void FNET_TransportThread::endEventLoop() { + // close and remove all I/O Components + FNET_IOComponent *component = _componentsHead; + while (component != nullptr) { + assert(component == _componentsHead); + FNET_IOComponent *tmp = component; + component = component->_ioc_next; + RemoveComponent(tmp); + tmp->Close(); + tmp->internal_subref(); + } + // flush event queue { std::lock_guard<std::mutex> guard(_lock); _queue.FlushPackets_NoLock(&_myQueue); + _reject_events = true; } // discard remaining events @@ -569,16 +582,6 @@ FNET_TransportThread::endEventLoop() { } } - // close and remove all I/O Components - FNET_IOComponent *component = _componentsHead; - while (component != nullptr) { - assert(component == _componentsHead); - FNET_IOComponent *tmp = component; - component = component->_ioc_next; - RemoveComponent(tmp); - tmp->Close(); - tmp->internal_subref(); - } assert(_componentsHead == nullptr && _componentsTail == nullptr && _timeOutHead == nullptr && @@ -588,7 +591,7 @@ FNET_TransportThread::endEventLoop() { { std::lock_guard<std::mutex> guard(_shutdownLock); - _finished.store(true, std::memory_order_relaxed); + _finished.store(true, std::memory_order_release); _shutdownCond.notify_all(); } diff --git a/fnet/src/vespa/fnet/transport_thread.h b/fnet/src/vespa/fnet/transport_thread.h index 6047d4e3482..c7ada472501 100644 --- a/fnet/src/vespa/fnet/transport_thread.h +++ b/fnet/src/vespa/fnet/transport_thread.h @@ -52,6 +52,7 @@ private: std::atomic<bool> _shutdown; // should stop event loop ? std::atomic<bool> _finished; // event loop stopped ? std::set<FNET_IServerAdapter*> _detaching; // server adapters being detached + bool _reject_events; // the transport thread does not want any more events /** * Add an IOComponent to the list of components. This operation is @@ -169,12 +170,12 @@ private: **/ bool EventLoopIteration(); - bool IsShutDown() const noexcept { + [[nodiscard]] bool should_shut_down() const noexcept { return _shutdown.load(std::memory_order_relaxed); } [[nodiscard]] bool is_finished() const noexcept { - return _finished.load(std::memory_order_relaxed); + return _finished.load(std::memory_order_acquire); } public: diff --git a/lucene-linguistics/src/main/java/com/yahoo/language/lucene/AnalyzerFactory.java b/lucene-linguistics/src/main/java/com/yahoo/language/lucene/AnalyzerFactory.java index b7d3a618954..71e31de34b3 100644 --- a/lucene-linguistics/src/main/java/com/yahoo/language/lucene/AnalyzerFactory.java +++ b/lucene-linguistics/src/main/java/com/yahoo/language/lucene/AnalyzerFactory.java @@ -14,9 +14,16 @@ import java.io.IOException; import java.nio.file.Path; import java.util.HashMap; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import java.util.logging.Logger; -public class AnalyzerFactory { +/** + * Analyzers for various languages. + * + * @author dainiusjocas + */ +class AnalyzerFactory { + private static final Logger log = Logger.getLogger(AnalyzerFactory.class.getName()); private final LuceneAnalysisConfig config; @@ -27,7 +34,7 @@ public class AnalyzerFactory { // Registry of analyzers per language // The idea is to create analyzers ONLY WHEN they are needed // Analyzers are thread safe so no need to recreate them for every document - private final Map<String, Analyzer> languageAnalyzers = new HashMap<>(); + private final Map<AnalyzerKey, Analyzer> languageAnalyzers = new ConcurrentHashMap<>(); private final Analyzer defaultAnalyzer = new StandardAnalyzer(); @@ -40,59 +47,42 @@ public class AnalyzerFactory { this.config = config; this.configDir = config.configDir(); this.analyzerComponents = analyzers; - this.defaultAnalyzers = DefaultAnalyzers.getInstance(); - log.info("Available in classpath char filters: " + CharFilterFactory.availableCharFilters()); - log.info("Available in classpath tokenizers: " + TokenizerFactory.availableTokenizers()); - log.info("Available in classpath token filters: " + TokenFilterFactory.availableTokenFilters()); + this.defaultAnalyzers = new DefaultAnalyzers(); + log.config("Available in classpath char filters: " + CharFilterFactory.availableCharFilters()); + log.config("Available in classpath tokenizers: " + TokenizerFactory.availableTokenizers()); + log.config("Available in classpath token filters: " + TokenFilterFactory.availableTokenFilters()); } /** * Retrieves an analyzer with a given params. * Sets up the analyzer if config is provided. * Default analyzer is the `StandardAnalyzer`. - * @param language - * @param stemMode - * @param removeAccents - * @return */ public Analyzer getAnalyzer(Language language, StemMode stemMode, boolean removeAccents) { - String analyzerKey = generateKey(language, stemMode, removeAccents); + return languageAnalyzers.computeIfAbsent(new AnalyzerKey(language, stemMode, removeAccents), + this::createAnalyzer); + } - // If analyzer for language is already known - if (null != languageAnalyzers.get(analyzerKey)) { - return languageAnalyzers.get(analyzerKey); - } - if (null != config.analysis(analyzerKey)) { - return setAndReturn(analyzerKey, setUpAnalyzer(analyzerKey)); + private Analyzer createAnalyzer(AnalyzerKey analyzerKey) { + if (null != config.analysis(analyzerKey.languageCode())) { + log.config("Creating analyzer for " + analyzerKey + " from config"); + return createAnalyzer(analyzerKey, config.analysis(analyzerKey.languageCode())); } - if (null != analyzerComponents.getComponent(analyzerKey)) { - log.info("Analyzer for language=" + analyzerKey + " is from components."); - return setAndReturn(analyzerKey, analyzerComponents.getComponent(analyzerKey)); + if (null != analyzerComponents.getComponent(analyzerKey.languageCode())) { + log.config("Using analyzer for " + analyzerKey + " from components"); + return analyzerComponents.getComponent(analyzerKey.languageCode()); } - if (null != defaultAnalyzers.get(language)) { - log.info("Analyzer for language=" + analyzerKey + " is from a list of default language analyzers."); - return setAndReturn(analyzerKey, defaultAnalyzers.get(language)); + if (null != defaultAnalyzers.get(analyzerKey.language())) { + log.config("Using Analyzer for " + analyzerKey + " from a list of default language analyzers"); + return defaultAnalyzers.get(analyzerKey.language()); } // set the default analyzer for the language - log.info("StandardAnalyzer is used for language=" + analyzerKey); - return setAndReturn(analyzerKey, defaultAnalyzer); - } - - private Analyzer setAndReturn(String analyzerKey, Analyzer analyzer) { - languageAnalyzers.put(analyzerKey, analyzer); - return analyzer; + log.config("StandardAnalyzer is used for " + analyzerKey); + return defaultAnalyzer; } - // TODO: Would it make sense to combine language + stemMode + removeAccents to make - // a composite key so we can have more variations possible? - private String generateKey(Language language, StemMode stemMode, boolean removeAccents) { - return language.languageCode(); - } - - private Analyzer setUpAnalyzer(String analyzerKey) { + private Analyzer createAnalyzer(AnalyzerKey analyzerKey, LuceneAnalysisConfig.Analysis analysis) { try { - LuceneAnalysisConfig.Analysis analysis = config.analysis(analyzerKey); - log.info("Creating analyzer for: '" + analyzerKey + "' with config: " + analysis); CustomAnalyzer.Builder builder = CustomAnalyzer.builder(configDir); builder = withTokenizer(builder, analysis); builder = addCharFilters(builder, analysis); @@ -102,13 +92,8 @@ public class AnalyzerFactory { // Failing to set up the Analyzer, should blow up during testing and VAP should not be deployed. // Most likely cause for problems is that a specified resource is not available in VAP. // Unit tests should catch such problems and prevent the VAP being deployed. - log.severe("Failed to build analyzer: '" - + analyzerKey - + "', with configuration: '" - + config.analysis(analyzerKey) - + "' with exception: '" - + e.getMessage() + "'" ); - throw new RuntimeException(e); + throw new RuntimeException("Failed to build analyzer " + analyzerKey + + ", with configuration " + analysis, e); } } @@ -120,7 +105,7 @@ public class AnalyzerFactory { } String tokenizerName = analysis.tokenizer().name(); Map<String, String> conf = analysis.tokenizer().conf(); - return builder.withTokenizer(tokenizerName, toModifiable(conf)); + return builder.withTokenizer(tokenizerName, asModifiable(conf)); } private CustomAnalyzer.Builder addCharFilters(CustomAnalyzer.Builder builder, @@ -130,7 +115,7 @@ public class AnalyzerFactory { return builder; } for (LuceneAnalysisConfig.Analysis.CharFilters charFilter : analysis.charFilters()) { - builder.addCharFilter(charFilter.name(), toModifiable(charFilter.conf())); + builder.addCharFilter(charFilter.name(), asModifiable(charFilter.conf())); } return builder; } @@ -142,7 +127,7 @@ public class AnalyzerFactory { return builder; } for (LuceneAnalysisConfig.Analysis.TokenFilters tokenFilter : analysis.tokenFilters()) { - builder.addTokenFilter(tokenFilter.name(), toModifiable(tokenFilter.conf())); + builder.addTokenFilter(tokenFilter.name(), asModifiable(tokenFilter.conf())); } return builder; } @@ -151,10 +136,33 @@ public class AnalyzerFactory { * A config map coming from the Vespa ConfigInstance is immutable while CustomAnalyzer builders * mutates the map to mark that a param was consumed. Immutable maps can't be mutated! * To overcome this conflict we can wrap the ConfigInstance map in a new HashMap. - * @param map - * @return Mutable Map */ - private Map<String, String> toModifiable(Map<String, String> map) { + private Map<String, String> asModifiable(Map<String, String> map) { return new HashMap<>(map); } + + private record AnalyzerKey(Language language, StemMode stemMode, boolean removeAccents) { + + // TODO: Identity here is determined by language only. + // Would it make sense to combine language + stemMode + removeAccents to make + // a composite key so we can have more variations possible? + + public String languageCode() { + return language.languageCode(); + } + + @Override + public boolean equals(Object o) { + if (o == this) return true; + if ( ! (o instanceof AnalyzerKey other)) return false; + return other.language == this.language; + } + + @Override + public int hashCode() { + return language.hashCode(); + } + + } + } diff --git a/lucene-linguistics/src/main/java/com/yahoo/language/lucene/DefaultAnalyzers.java b/lucene-linguistics/src/main/java/com/yahoo/language/lucene/DefaultAnalyzers.java index 955e18474f7..95b11301d47 100644 --- a/lucene-linguistics/src/main/java/com/yahoo/language/lucene/DefaultAnalyzers.java +++ b/lucene-linguistics/src/main/java/com/yahoo/language/lucene/DefaultAnalyzers.java @@ -44,12 +44,14 @@ import java.util.Map; import static java.util.Map.entry; -public class DefaultAnalyzers { +/** + * @author dainiusjocas + */ +class DefaultAnalyzers { - private static DefaultAnalyzers INSTANCE; private final Map<Language, Analyzer> analyzerClasses; - private DefaultAnalyzers() { + public DefaultAnalyzers() { analyzerClasses = Map.ofEntries( entry(Language.ARABIC, new ArabicAnalyzer()), entry(Language.BULGARIAN, new BulgarianAnalyzer()), @@ -93,18 +95,8 @@ public class DefaultAnalyzers { ); } - public static DefaultAnalyzers getInstance() { - if (INSTANCE == null) { - INSTANCE = new DefaultAnalyzers(); - } - return INSTANCE; - } - public Analyzer get(Language language) { return analyzerClasses.get(language); } - public Analyzer get(String languageCode) { - return analyzerClasses.get(Language.fromLanguageTag(languageCode)); - } } diff --git a/lucene-linguistics/src/main/java/com/yahoo/language/lucene/LuceneLinguistics.java b/lucene-linguistics/src/main/java/com/yahoo/language/lucene/LuceneLinguistics.java index b5c5ba47ab6..8b193c103d6 100644 --- a/lucene-linguistics/src/main/java/com/yahoo/language/lucene/LuceneLinguistics.java +++ b/lucene-linguistics/src/main/java/com/yahoo/language/lucene/LuceneLinguistics.java @@ -3,80 +3,41 @@ package com.yahoo.language.lucene; import com.google.inject.Inject; import com.yahoo.component.provider.ComponentRegistry; import com.yahoo.language.Linguistics; -import com.yahoo.language.process.*; +import com.yahoo.language.process.Tokenizer; import com.yahoo.language.simple.SimpleLinguistics; import org.apache.lucene.analysis.Analyzer; -import java.util.ArrayList; import java.util.logging.Logger; /** - * Factory of Lucene based linguistics processor. + * Factory of Lucene based linguistics processors. * As described in the Linguistics docstring * > the tokenizer should typically stem, transform and normalize - * The Stemmer, Transformer, Normalizer, and Segmenter implementations are mostly NOOP. * * TODO: docs for all available analysis components. - * TODO: some registry for available language Analyzers. + * + * @author dainiusjocas */ public class LuceneLinguistics extends SimpleLinguistics { private static final Logger log = Logger.getLogger(LuceneLinguistics.class.getName()); - private final Normalizer normalizer; - private final Transformer transformer; - private final Tokenizer tokenizer; - private final Stemmer stemmer; - private final Segmenter segmenter; + private final LuceneTokenizer tokenizer; private final LuceneAnalysisConfig config; @Inject public LuceneLinguistics(LuceneAnalysisConfig config, ComponentRegistry<Analyzer> analyzers) { - log.info("Creating LuceneLinguistics with: " + config); + log.config("Creating LuceneLinguistics with: " + config); this.config = config; this.tokenizer = new LuceneTokenizer(config, analyzers); - // NOOP stemmer - this.stemmer = (word, stemMode, language) -> { - ArrayList<StemList> stemLists = new ArrayList<>(); - StemList stems = new StemList(); - stems.add(word); - stemLists.add(stems); - return stemLists; - }; - // Segmenter that just wraps a tokenizer - this.segmenter = (string, language) -> { - ArrayList<String> segments = new ArrayList<>(); - Iterable<Token> tokens = tokenizer.tokenize(string, language, StemMode.NONE, false); - tokens.forEach(token -> segments.add(token.getTokenString())); - return segments; - }; - // NOOP normalizer - this.normalizer = (string) -> string; - // NOOP transformer - this.transformer = (string, language) -> string; } @Override - public Stemmer getStemmer() { return stemmer; } - - @Override public Tokenizer getTokenizer() { return tokenizer; } @Override - public Normalizer getNormalizer() { return normalizer; } - - @Override - public Transformer getTransformer() { return transformer; } - - @Override - public Segmenter getSegmenter() { return segmenter; } - - public LuceneAnalysisConfig getConfig() { - return config; + public boolean equals(Linguistics other) { + // Config actually determines if Linguistics are equal + return (other instanceof LuceneLinguistics) && config.equals(((LuceneLinguistics) other).config); } - @Override - public boolean equals(Linguistics other) { - return (other instanceof LuceneLinguistics) - // Config actually determines if Linguistics are equal - && config.equals(((LuceneLinguistics) other).getConfig()); } } diff --git a/lucene-linguistics/src/main/java/com/yahoo/language/lucene/LuceneTokenizer.java b/lucene-linguistics/src/main/java/com/yahoo/language/lucene/LuceneTokenizer.java index 0cde849fd6e..c1fa4da4989 100644 --- a/lucene-linguistics/src/main/java/com/yahoo/language/lucene/LuceneTokenizer.java +++ b/lucene-linguistics/src/main/java/com/yahoo/language/lucene/LuceneTokenizer.java @@ -2,7 +2,11 @@ package com.yahoo.language.lucene; import com.yahoo.component.provider.ComponentRegistry; import com.yahoo.language.Language; -import com.yahoo.language.process.*; +import com.yahoo.language.process.StemMode; +import com.yahoo.language.process.Token; +import com.yahoo.language.process.TokenScript; +import com.yahoo.language.process.TokenType; +import com.yahoo.language.process.Tokenizer; import com.yahoo.language.simple.SimpleToken; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.TokenStream; @@ -15,7 +19,10 @@ import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; -public class LuceneTokenizer implements Tokenizer { +/** + * @author dainiusjocas + */ +class LuceneTokenizer implements Tokenizer { private static final Logger log = Logger.getLogger(LuceneTokenizer.class.getName()); @@ -36,7 +43,7 @@ public class LuceneTokenizer implements Tokenizer { if (input.isEmpty()) return List.of(); List<Token> tokens = textToTokens(input, analyzerFactory.getAnalyzer(language, stemMode, removeAccents)); - log.log(Level.FINEST, "Tokenized '" + language + "' text='" + input + "' into: n=" + tokens.size() + ", tokens=" + tokens); + log.log(Level.FINEST, () -> "Tokenized '" + language + "' text='" + input + "' into: n=" + tokens.size() + ", tokens=" + tokens); return tokens; } @@ -49,7 +56,6 @@ public class LuceneTokenizer implements Tokenizer { try { tokenStream.reset(); while (tokenStream.incrementToken()) { - // TODO: is SimpleToken good enough? Maybe a custom implementation. // TODO: what to do with cases when multiple tokens are inserted into the position? String originalString = text.substring(offsetAttribute.startOffset(), offsetAttribute.endOffset()); String tokenString = charTermAttribute.toString(); @@ -65,4 +71,5 @@ public class LuceneTokenizer implements Tokenizer { } return tokens; } + } diff --git a/lucene-linguistics/src/test/java/com/yahoo/language/lucene/LuceneTokenizerTest.java b/lucene-linguistics/src/test/java/com/yahoo/language/lucene/LuceneTokenizerTest.java index 568f295b39d..21d3a7bd33d 100644 --- a/lucene-linguistics/src/test/java/com/yahoo/language/lucene/LuceneTokenizerTest.java +++ b/lucene-linguistics/src/test/java/com/yahoo/language/lucene/LuceneTokenizerTest.java @@ -3,6 +3,8 @@ package com.yahoo.language.lucene; import com.yahoo.component.provider.ComponentRegistry; import com.yahoo.config.FileReference; import com.yahoo.language.Language; +import com.yahoo.language.Linguistics; +import com.yahoo.language.process.StemList; import com.yahoo.language.process.StemMode; import com.yahoo.language.process.Token; import org.junit.Test; @@ -15,16 +17,15 @@ import java.util.Map; import static org.junit.Assert.assertEquals; +/** + * @author dainiusjocas + */ public class LuceneTokenizerTest { @Test public void testTokenizer() { String text = "This is my Text"; - var tokenizer = new LuceneTokenizer(new LuceneAnalysisConfig - .Builder() - .configDir(FileReference.mockFileReferenceForUnitTesting(new File("."))) - .build()); - Iterable<Token> tokens = tokenizer + Iterable<Token> tokens = luceneLinguistics().getTokenizer() .tokenize(text, Language.ENGLISH, StemMode.ALL, true); assertEquals(List.of("my", "text"), tokenStrings(tokens)); } @@ -32,15 +33,26 @@ public class LuceneTokenizerTest { @Test public void testLithuanianTokenizer() { String text = "Žalgirio mūšio data yra 1410 metai"; - var tokenizer = new LuceneTokenizer(new LuceneAnalysisConfig - .Builder() - .configDir(FileReference.mockFileReferenceForUnitTesting(new File("."))) - .build()); - Iterable<Token> tokens = tokenizer + Iterable<Token> tokens = luceneLinguistics().getTokenizer() .tokenize(text, Language.LITHUANIAN, StemMode.ALL, true); assertEquals(List.of("žalgir", "mūš", "dat", "1410", "met"), tokenStrings(tokens)); } + @Test + public void testStemming() { + String text = "mūšio"; + List<StemList> tokens = luceneLinguistics().getStemmer().stem(text, StemMode.ALL, Language.LITHUANIAN); + assertEquals(1, tokens.size()); + assertEquals("mūš", tokens.get(0).get(0)); + } + + private Linguistics luceneLinguistics() { + return new LuceneLinguistics(new LuceneAnalysisConfig.Builder() + .configDir(FileReference.mockFileReferenceForUnitTesting(new File("."))) + .build(), + new ComponentRegistry<>()); + } + private void assertToken(String tokenString, Iterator<Token> tokens) { Token t = tokens.next(); assertEquals(tokenString, t.getTokenString()); @@ -136,4 +148,5 @@ public class LuceneTokenizerTest { .tokenize("Dogs and Cats", Language.ENGLISH, StemMode.ALL, false); assertEquals(List.of("Dog", "Cat"), tokenStrings(tokens)); } + } diff --git a/messagebus/src/vespa/messagebus/network/rpctarget.cpp b/messagebus/src/vespa/messagebus/network/rpctarget.cpp index 9c6ca9dff69..d7f3e77c6fd 100644 --- a/messagebus/src/vespa/messagebus/network/rpctarget.cpp +++ b/messagebus/src/vespa/messagebus/network/rpctarget.cpp @@ -5,8 +5,8 @@ namespace mbus { -RPCTarget::RPCTarget(const string &spec, FRT_Supervisor &orb) : - _lock(), +RPCTarget::RPCTarget(const string &spec, FRT_Supervisor &orb, ctor_tag) + : _lock(), _orb(orb), _name(spec), _target(*_orb.GetTarget(spec.c_str())), @@ -48,6 +48,7 @@ RPCTarget::resolveVersion(duration timeout, RPCTarget::IVersionHandler &handler) handler.handleVersion(_version.get()); } else if (shouldInvoke) { FRT_RPCRequest *req = _orb.AllocRPCRequest(); + req->getStash().create<SP>(shared_from_this()); req->SetMethodName("mbus.getVersion"); _target.InvokeAsync(req, vespalib::to_s(timeout), this); } @@ -67,8 +68,9 @@ RPCTarget::isValid() const } void -RPCTarget::RequestDone(FRT_RPCRequest *req) +RPCTarget::RequestDone(FRT_RPCRequest *raw_req) { + auto req = vespalib::ref_counted<FRT_RPCRequest>::internal_attach(raw_req); HandlerList handlers; { std::lock_guard guard(_lock); @@ -94,7 +96,6 @@ RPCTarget::RequestDone(FRT_RPCRequest *req) _state = (_version.get() ? VERSION_RESOLVED : VERSION_NOT_RESOLVED); } _cond.notify_all(); - req->internal_subref(); } } // namespace mbus diff --git a/messagebus/src/vespa/messagebus/network/rpctarget.h b/messagebus/src/vespa/messagebus/network/rpctarget.h index fffffae64f7..77fcef5f48f 100644 --- a/messagebus/src/vespa/messagebus/network/rpctarget.h +++ b/messagebus/src/vespa/messagebus/network/rpctarget.h @@ -13,7 +13,7 @@ namespace mbus { * target. Instances of this class are returned by {@link RPCService}, and * cached by {@link RPCTargetPool}. */ -class RPCTarget : public FRT_IRequestWait { +class RPCTarget : public FRT_IRequestWait, public std::enable_shared_from_this<RPCTarget> { public: /** * Declares a version handler used when resolving the version of a target. @@ -58,6 +58,7 @@ private: Version_UP _version; HandlerList _versionHandlers; + struct ctor_tag {}; public: /** * Convenience typedefs. @@ -72,7 +73,10 @@ public: * @param spec The connection spec of this target. * @param orb The FRT supervisor to use when connecting to target. */ - RPCTarget(const string &name, FRT_Supervisor &orb); + RPCTarget(const string &name, FRT_Supervisor &orb, ctor_tag); + static SP create(const string &name, FRT_Supervisor &orb) { + return std::make_shared<RPCTarget>(name, orb, ctor_tag{}); + } /** * Destructor. Subrefs the contained FRT target. diff --git a/messagebus/src/vespa/messagebus/network/rpctargetpool.cpp b/messagebus/src/vespa/messagebus/network/rpctargetpool.cpp index b403c65f863..db09b127114 100644 --- a/messagebus/src/vespa/messagebus/network/rpctargetpool.cpp +++ b/messagebus/src/vespa/messagebus/network/rpctargetpool.cpp @@ -97,7 +97,7 @@ RPCTargetPool::getTarget(FRT_Supervisor &orb, const RPCServiceAddress &address) std::vector<RPCTarget::SP> targets; targets.reserve(_numTargetsPerSpec); for (size_t i(0); i < _numTargetsPerSpec; i++) { - targets.push_back(std::make_shared<RPCTarget>(spec, orb)); + targets.push_back(RPCTarget::create(spec, orb)); } _targets.insert(TargetMap::value_type(spec, Entry(std::move(targets), currentTime))); return _targets.find(spec)->second.getTarget(guard, currentTime); diff --git a/metrics/src/main/java/ai/vespa/metrics/ConfigServerMetrics.java b/metrics/src/main/java/ai/vespa/metrics/ConfigServerMetrics.java index ca028547171..2a6a2986d9b 100644 --- a/metrics/src/main/java/ai/vespa/metrics/ConfigServerMetrics.java +++ b/metrics/src/main/java/ai/vespa/metrics/ConfigServerMetrics.java @@ -135,7 +135,6 @@ public enum ConfigServerMetrics implements VespaMetrics { HOSTED_VESPA_READY_NODES("hostedVespa.readyNodes", Unit.HOST, "The number of managed nodes that are in state \"ready\""), HOSTED_VESPA_RESERVED_NODES("hostedVespa.reservedNodes", Unit.HOST, "The number of managed nodes that are in state \"reserved\""), - OVERCOMMITTED_HOSTS("overcommittedHosts", Unit.HOST, "The number of hosts with over-committed resources"), SPARE_HOST_CAPACITY("spareHostCapacity", Unit.HOST, "The number of spare hosts"), THROTTLED_HOST_FAILURES("throttledHostFailures", Unit.HOST, "Number of host failures stopped due to throttling"), @@ -143,8 +142,9 @@ public enum ConfigServerMetrics implements VespaMetrics { NODE_FAIL_THROTTLING("nodeFailThrottling", Unit.BINARY, "Metric indicating when node failure throttling is active. The value 1 means active, 0 means inactive"), DEPLOYMENT_PREPARE_MILLIS("deployment.prepareMillis", Unit.MILLISECOND, "Duration of deployment preparations"), - DEPLOYMENT_ACTIVATE_MILLIS("deployment.activateMillis", Unit.MILLISECOND, "Duration of deployment activations"); + DEPLOYMENT_ACTIVATE_MILLIS("deployment.activateMillis", Unit.MILLISECOND, "Duration of deployment activations"), + THROTTLED_HOST_PROVISIONING("throttledHostProvisioning", Unit.BINARY, "Value 1 if host provisioning is throttled, 0 if not"); private final String name; private final Unit unit; diff --git a/metrics/src/main/java/ai/vespa/metrics/ContainerMetrics.java b/metrics/src/main/java/ai/vespa/metrics/ContainerMetrics.java index 98bd6230762..aed6098e109 100644 --- a/metrics/src/main/java/ai/vespa/metrics/ContainerMetrics.java +++ b/metrics/src/main/java/ai/vespa/metrics/ContainerMetrics.java @@ -28,7 +28,7 @@ public enum ContainerMetrics implements VespaMetrics { JDISC_THREAD_POOL_MAX_ALLOWED_SIZE("jdisc.thread_pool.max_allowed_size", Unit.THREAD, "The maximum allowed number of threads in the pool"), JDISC_THREAD_POOL_ACTIVE_THREADS("jdisc.thread_pool.active_threads", Unit.THREAD, "Number of threads that are active"), - JDISC_DEACTIVATED_CONTAINERS("jdisc.deactivated_containers.total", Unit.ITEM, "JDISC Deactivated container instances"), + JDISC_DEACTIVATED_CONTAINERS_TOTAL("jdisc.deactivated_containers.total", Unit.ITEM, "JDISC Deactivated container instances"), JDISC_DEACTIVATED_CONTAINERS_WITH_RETAINED_REFS("jdisc.deactivated_containers.with_retained_refs.last", Unit.ITEM, "JDISC Deactivated container nodes with retained refs"), JDISC_APPLICATION_FAILED_COMPONENT_GRAPHS("jdisc.application.failed_component_graphs", Unit.ITEM, "JDISC Application failed component graphs"), JDISC_APPLICATION_COMPONENT_GRAPH_CREATION_TIME_MILLIS("jdisc.application.component_graph.creation_time_millis", Unit.MILLISECOND, "JDISC Application component graph creation time"), @@ -162,9 +162,9 @@ public enum ContainerMetrics implements VespaMetrics { ERROR_UNHANDLED_EXCEPTION("error.unhandled_exception", Unit.OPERATION, "Requests that failed due to an unhandled exception"), // Deprecated metrics. TODO: Remove in Vespa 9. - SERVER_REJECTED_REQUESTS("serverRejectedRequests", Unit.OPERATION, "Deprecated. Use jdisc.thread_pool.rejected_tasks instead."), - SERVER_THREAD_POOL_SIZE("serverThreadPoolSize", Unit.THREAD, "Deprecated. Use jdisc.thread_pool.size instead."), - SERVER_ACTIVE_THREADS("serverActiveThreads", Unit.THREAD, "Deprecated. Use jdisc.thread_pool.active_threads instead."), + SERVER_REJECTED_REQUESTS("serverRejectedRequests", Unit.OPERATION, "Deprecated. Use jdisc.thread_pool.rejected_tasks instead."), // TODO: Remove in Vespa 9. + SERVER_THREAD_POOL_SIZE("serverThreadPoolSize", Unit.THREAD, "Deprecated. Use jdisc.thread_pool.size instead."), // TODO: Remove in Vespa 9. + SERVER_ACTIVE_THREADS("serverActiveThreads", Unit.THREAD, "Deprecated. Use jdisc.thread_pool.active_threads instead."), // TODO: Remove in Vespa 9. // Java (JRT) TLS metrics JRT_TRANSPORT_TLS_CERTIFICATE_VERIFICATION_FAILURES("jrt.transport.tls-certificate-verification-failures", Unit.FAILURE, "TLS certificate verification failures"), diff --git a/metrics/src/main/java/ai/vespa/metrics/ControllerMetrics.java b/metrics/src/main/java/ai/vespa/metrics/ControllerMetrics.java index 4770fe51830..0f200308862 100644 --- a/metrics/src/main/java/ai/vespa/metrics/ControllerMetrics.java +++ b/metrics/src/main/java/ai/vespa/metrics/ControllerMetrics.java @@ -37,6 +37,7 @@ public enum ControllerMetrics implements VespaMetrics { DNS_QUEUED_REQUESTS("dns.queuedRequests", Unit.REQUEST, "Queued DNS requests"), ZMS_QUOTA_USAGE("zms.quota.usage", Unit.FRACTION, "ZMS Quota usage per resource type"), COREDUMP_PROCESSED("coredump.processed", Unit.FAILURE,"Controller: Core dumps processed"), + AUTH0_EXCEPTIONS("auth0.exceptions", Unit.FAILURE, "Controller: Auth0 exceptions"), // Metrics per API, metrics names generated in ControllerMaintainer/MetricsReporter OPERATION_APPLICATION("operation.application", Unit.REQUEST, "Controller: Requests for /application API"), diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/AutoscalingMetrics.java b/metrics/src/main/java/ai/vespa/metrics/set/AutoscalingMetrics.java index 685ca377e70..01793f292cf 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/AutoscalingMetrics.java +++ b/metrics/src/main/java/ai/vespa/metrics/set/AutoscalingMetrics.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.admin.monitoring; +package ai.vespa.metrics.set; import ai.vespa.metrics.ContainerMetrics; import ai.vespa.metrics.HostedNodeAdminMetrics; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultMetrics.java b/metrics/src/main/java/ai/vespa/metrics/set/DefaultMetrics.java index 8652acbd546..cb3a13acb85 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultMetrics.java +++ b/metrics/src/main/java/ai/vespa/metrics/set/DefaultMetrics.java @@ -1,9 +1,14 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.admin.monitoring; +package ai.vespa.metrics.set; import ai.vespa.metrics.ContainerMetrics; import ai.vespa.metrics.SearchNodeMetrics; +import ai.vespa.metrics.StorageMetrics; +import ai.vespa.metrics.DistributorMetrics; +import ai.vespa.metrics.ClusterControllerMetrics; +import ai.vespa.metrics.SentinelMetrics; +import ai.vespa.metrics.NodeAdminMetrics; import ai.vespa.metrics.Suffix; import ai.vespa.metrics.VespaMetrics; @@ -14,11 +19,13 @@ import java.util.Set; import static ai.vespa.metrics.Suffix.average; import static ai.vespa.metrics.Suffix.count; +import static ai.vespa.metrics.Suffix.last; import static ai.vespa.metrics.Suffix.max; +import static ai.vespa.metrics.Suffix.min; import static ai.vespa.metrics.Suffix.ninety_five_percentile; import static ai.vespa.metrics.Suffix.ninety_nine_percentile; import static ai.vespa.metrics.Suffix.sum; -import static com.yahoo.vespa.model.admin.monitoring.DefaultVespaMetrics.defaultVespaMetricSet; +import static ai.vespa.metrics.set.DefaultVespaMetrics.defaultVespaMetricSet; /** * Metrics for the 'default' consumer, which is used by default for the generic metrics api and @@ -41,9 +48,15 @@ public class DefaultMetrics { private static Set<Metric> getAllMetrics() { Set<Metric> metrics = new LinkedHashSet<>(); - addContentMetrics(metrics); addContainerMetrics(metrics); addSearchChainMetrics(metrics); + addDocprocMetrics(metrics); + addSearchNodeMetrics(metrics); + addContentMetrics(metrics); + addStorageMetrics(metrics); + addDistributorMetrics(metrics); + addClusterControllerMetrics(metrics); + addOtherMetrics(metrics); return Collections.unmodifiableSet(metrics); } @@ -53,8 +66,24 @@ public class DefaultMetrics { addMetric(metrics, ContainerMetrics.HTTP_STATUS_3XX.rate()); addMetric(metrics, ContainerMetrics.HTTP_STATUS_4XX.rate()); addMetric(metrics, ContainerMetrics.HTTP_STATUS_5XX.rate()); - addMetric(metrics, ContainerMetrics.JDISC_GC_MS.average()); + addMetric(metrics, ContainerMetrics.JDISC_GC_MS, EnumSet.of(max, average)); addMetric(metrics, ContainerMetrics.MEM_HEAP_FREE.average()); + addMetric(metrics, ContainerMetrics.FEED_LATENCY, EnumSet.of(sum, count)); + // addMetric(metrics, ContainerMetrics.CPU.baseName()); // TODO: Add to container metrics + addMetric(metrics, ContainerMetrics.JDISC_THREAD_POOL_SIZE.max()); + addMetric(metrics, ContainerMetrics.JDISC_THREAD_POOL_ACTIVE_THREADS, EnumSet.of(sum, count, min, max)); + addMetric(metrics, ContainerMetrics.JDISC_THREAD_POOL_WORK_QUEUE_CAPACITY.max()); + addMetric(metrics, ContainerMetrics.JDISC_THREAD_POOL_WORK_QUEUE_SIZE, EnumSet.of(sum, count, min, max)); + addMetric(metrics, ContainerMetrics.SERVER_ACTIVE_THREADS.average()); + + // Metrics needed for alerting + addMetric(metrics, ContainerMetrics.JDISC_SINGLETON_IS_ACTIVE, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last + addMetric(metrics, ContainerMetrics.JDISC_HTTP_SSL_HANDSHAKE_FAILURE_MISSING_CLIENT_CERT.rate()); + addMetric(metrics, ContainerMetrics.JDISC_HTTP_SSL_HANDSHAKE_FAILURE_INCOMPATIBLE_PROTOCOLS.rate()); + addMetric(metrics, ContainerMetrics.JDISC_HTTP_SSL_HANDSHAKE_FAILURE_INCOMPATIBLE_CHIFERS.rate()); + addMetric(metrics, ContainerMetrics.JDISC_HTTP_SSL_HANDSHAKE_FAILURE_UNKNOWN.rate()); + addMetric(metrics, ContainerMetrics.JDISC_APPLICATION_FAILED_COMPONENT_GRAPHS.rate()); + addMetric(metrics, ContainerMetrics.ATHENZ_TENANT_CERT_EXPIRY_SECONDS, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last } private static void addSearchChainMetrics(Set<Metric> metrics) { @@ -64,7 +93,17 @@ public class DefaultMetrics { addMetric(metrics, ContainerMetrics.TOTAL_HITS_PER_QUERY, EnumSet.of(sum, count, max, average)); // TODO: Remove average with Vespa 9 addMetric(metrics, ContainerMetrics.DEGRADED_QUERIES.rate()); addMetric(metrics, ContainerMetrics.FAILED_QUERIES.rate()); - addMetric(metrics, ContainerMetrics.SERVER_ACTIVE_THREADS.average()); // TODO: Remove on Vespa 9. Use jdisc.thread_pool.active_threads. + } + + private static void addDocprocMetrics(Set<Metric> metrics) { + addMetric(metrics, ContainerMetrics.DOCPROC_DOCUMENTS.sum()); + } + + private static void addSearchNodeMetrics(Set<Metric> metrics) { + // Metrics needed for alerting + addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_RESOURCE_USAGE_DISK.average()); + addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_RESOURCE_USAGE_MEMORY.average()); + addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_RESOURCE_USAGE_FEEDING_BLOCKED, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last } private static void addContentMetrics(Set<Metric> metrics) { @@ -72,9 +111,9 @@ public class DefaultMetrics { addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_SEARCH_PROTOCOL_DOCSUM_LATENCY, EnumSet.of(sum, count, max, average)); // TODO: Remove average with Vespa 9 addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_SEARCH_PROTOCOL_QUERY_LATENCY, EnumSet.of(sum, count, max, average)); // TODO: Remove average with Vespa 9 - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_DOCUMENTS_TOTAL.last()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_DOCUMENTS_READY.last()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_DOCUMENTS_ACTIVE.last()); + addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_DOCUMENTS_TOTAL, EnumSet.of(max,last)); // TODO: Vespa 9: Remove last + addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_DOCUMENTS_READY, EnumSet.of(max,last)); // TODO: Vespa 9: Remove last + addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_DOCUMENTS_ACTIVE, EnumSet.of(max,last)); // TODO: Vespa 9: Remove last addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_DISK_USAGE.last()); addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_MEMORY_USAGE_ALLOCATED_BYTES.last()); @@ -89,6 +128,41 @@ public class DefaultMetrics { addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_TRANSACTIONLOG_DISK_USAGE.last()); } + private static void addStorageMetrics(Set<Metric> metrics) { + addMetric(metrics, StorageMetrics.VDS_FILESTOR_ALLTHREADS_PUT_COUNT.rate()); + addMetric(metrics, StorageMetrics.VDS_FILESTOR_ALLTHREADS_UPDATE_COUNT.rate()); + addMetric(metrics, StorageMetrics.VDS_FILESTOR_ALLTHREADS_REMOVE_COUNT.rate()); + } + + private static void addDistributorMetrics(Set<Metric> metrics) { + addMetric(metrics, DistributorMetrics.VDS_DISTRIBUTOR_DOCSSTORED.average()); + + // Metrics needed for alerting + addMetric(metrics, DistributorMetrics.VDS_BOUNCER_CLOCK_SKEW_ABORTS.count()); + } + + private static void addClusterControllerMetrics(Set<Metric> metrics) { + // Metrics needed for alerting + addMetric(metrics, ClusterControllerMetrics.DOWN_COUNT, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last + addMetric(metrics, ClusterControllerMetrics.MAINTENANCE_COUNT, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last + addMetric(metrics, ClusterControllerMetrics.UP_COUNT.last()); + addMetric(metrics, ClusterControllerMetrics.IS_MASTER, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last + addMetric(metrics, ClusterControllerMetrics.RESOURCE_USAGE_NODES_ABOVE_LIMIT, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last + addMetric(metrics, ClusterControllerMetrics.RESOURCE_USAGE_MAX_MEMORY_UTILIZATION, EnumSet.of(last, max)); // TODO: Vespa 9: Remove last + addMetric(metrics, ClusterControllerMetrics.RESOURCE_USAGE_MAX_DISK_UTILIZATION, EnumSet.of(last, max)); // TODO: Vespa 9: Remove last + } + + private static void addSentinelMetrics(Set<Metric> metrics) { + // Metrics needed for alerting + addMetric(metrics, SentinelMetrics.SENTINEL_TOTAL_RESTARTS, EnumSet.of(sum, last)); // TODO: Vespa 9: Remove last + } + + private static void addOtherMetrics(Set<Metric> metrics) { + // Metrics needed for alerting + addMetric(metrics, NodeAdminMetrics.ENDPOINT_CERTIFICATE_EXPIRY_SECONDS.baseName()); + addMetric(metrics, NodeAdminMetrics.NODE_CERTIFICATE_EXPIRY_SECONDS.baseName()); + } + private static void addMetric(Set<Metric> metrics, String nameWithSuffix) { metrics.add(new Metric(nameWithSuffix)); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultVespaMetrics.java b/metrics/src/main/java/ai/vespa/metrics/set/DefaultVespaMetrics.java index ac03fcf56cb..93b6bfab002 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultVespaMetrics.java +++ b/metrics/src/main/java/ai/vespa/metrics/set/DefaultVespaMetrics.java @@ -1,10 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.admin.monitoring; +package ai.vespa.metrics.set; import ai.vespa.metrics.ContainerMetrics; import ai.vespa.metrics.SearchNodeMetrics; -import com.google.common.collect.ImmutableSet; +import java.util.LinkedHashSet; import java.util.Set; /** @@ -19,20 +19,11 @@ public class DefaultVespaMetrics { public static final MetricSet defaultVespaMetricSet = createDefaultVespaMetricSet(); private static MetricSet createDefaultVespaMetricSet() { + Set<Metric> metrics = new LinkedHashSet<>(); - Set<Metric> defaultContainerMetrics = - ImmutableSet.of(new Metric(ContainerMetrics.FEED_OPERATIONS.rate()) - ); + metrics.add(new Metric(ContainerMetrics.FEED_OPERATIONS.rate())); + metrics.add(new Metric(SearchNodeMetrics.CONTENT_PROTON_RESOURCE_USAGE_FEEDING_BLOCKED.last())); - Set<Metric> defaultContentMetrics = - ImmutableSet.of(new Metric(SearchNodeMetrics.CONTENT_PROTON_RESOURCE_USAGE_FEEDING_BLOCKED.last()) - ); - - Set<Metric> defaultMetrics = ImmutableSet.<Metric>builder() - .addAll(defaultContainerMetrics) - .addAll(defaultContentMetrics) - .build(); - - return new MetricSet("default-vespa", defaultMetrics); + return new MetricSet("default-vespa", metrics); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/InfrastructureMetricSet.java b/metrics/src/main/java/ai/vespa/metrics/set/InfrastructureMetricSet.java index 92156c959a0..38533f40950 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/InfrastructureMetricSet.java +++ b/metrics/src/main/java/ai/vespa/metrics/set/InfrastructureMetricSet.java @@ -1,7 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.admin.monitoring; +package ai.vespa.metrics.set; -import ai.vespa.metrics.*; +import ai.vespa.metrics.ConfigServerMetrics; +import ai.vespa.metrics.ContainerMetrics; +import ai.vespa.metrics.ControllerMetrics; +import ai.vespa.metrics.LogdMetrics; +import ai.vespa.metrics.Suffix; +import ai.vespa.metrics.VespaMetrics; import java.util.Collections; import java.util.EnumSet; @@ -12,6 +17,7 @@ import static ai.vespa.metrics.Suffix.average; import static ai.vespa.metrics.Suffix.count; import static ai.vespa.metrics.Suffix.last; import static ai.vespa.metrics.Suffix.max; +import static ai.vespa.metrics.Suffix.min; import static ai.vespa.metrics.Suffix.sum; /** @@ -56,7 +62,8 @@ public class InfrastructureMetricSet { addMetric(metrics, ConfigServerMetrics.ZK_OUTSTANDING_REQUESTS, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last. // Node repository metrics - addMetric(metrics, ConfigServerMetrics.NODES_NON_ACTIVE_FRACTION.last()); + addMetric(metrics, ConfigServerMetrics.NODES_ACTIVE, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last + addMetric(metrics, ConfigServerMetrics.NODES_NON_ACTIVE_FRACTION, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last addMetric(metrics, ConfigServerMetrics.CLUSTER_COST.last()); addMetric(metrics, ConfigServerMetrics.CLUSTER_LOAD_IDEAL_CPU.last()); addMetric(metrics, ConfigServerMetrics.CLUSTER_LOAD_IDEAL_MEMORY.last()); @@ -66,44 +73,54 @@ public class InfrastructureMetricSet { addMetric(metrics, ConfigServerMetrics.WANT_TO_RETIRE.max()); addMetric(metrics, ConfigServerMetrics.RETIRED.max()); addMetric(metrics, ConfigServerMetrics.WANT_TO_CHANGE_VESPA_VERSION.max()); - addMetric(metrics, ConfigServerMetrics.HAS_WIRE_GUARD_KEY.last()); + addMetric(metrics, ConfigServerMetrics.HAS_WIRE_GUARD_KEY, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last addMetric(metrics, ConfigServerMetrics.WANT_TO_DEPROVISION.max()); - addMetric(metrics, ConfigServerMetrics.SUSPENDED.max()); + addMetric(metrics, ConfigServerMetrics.SUSPENDED, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last addMetric(metrics, ConfigServerMetrics.SOME_SERVICES_DOWN.max()); - addMetric(metrics, ConfigServerMetrics.NODE_FAILER_BAD_NODE.last()); + addMetric(metrics, ConfigServerMetrics.NODE_FAILER_BAD_NODE, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last. addMetric(metrics, ConfigServerMetrics.LOCK_ATTEMPT_LOCKED_LOAD, EnumSet.of(max,average)); - addMetric(metrics, ConfigServerMetrics.HOSTED_VESPA_DOCKER_ALLOCATED_CAPACITY_CPU.average()); - addMetric(metrics, ConfigServerMetrics.HOSTED_VESPA_DOCKER_ALLOCATED_CAPACITY_MEM.average()); - addMetric(metrics, ConfigServerMetrics.HOSTED_VESPA_DOCKER_ALLOCATED_CAPACITY_DISK.average()); + addMetric(metrics, ConfigServerMetrics.HOSTED_VESPA_DOCKER_ALLOCATED_CAPACITY_CPU, EnumSet.of(max, average, last)); // TODO: Vespa 9: Remove last, average? + addMetric(metrics, ConfigServerMetrics.HOSTED_VESPA_DOCKER_ALLOCATED_CAPACITY_MEM, EnumSet.of(max, average, last)); // TODO: Vespa 9: Remove last, average? + addMetric(metrics, ConfigServerMetrics.HOSTED_VESPA_DOCKER_ALLOCATED_CAPACITY_DISK, EnumSet.of(max, average, last)); // TODO: Vespa 9: Remove last, average? addMetric(metrics, ConfigServerMetrics.HOSTED_VESPA_DOCKER_FREE_CAPACITY_CPU.max()); addMetric(metrics, ConfigServerMetrics.HOSTED_VESPA_DOCKER_FREE_CAPACITY_MEM.max()); addMetric(metrics, ConfigServerMetrics.HOSTED_VESPA_DOCKER_FREE_CAPACITY_DISK.max()); addMetric(metrics, ConfigServerMetrics.HOSTED_VESPA_DOCKER_TOTAL_CAPACITY_CPU, EnumSet.of(max,average)); addMetric(metrics, ConfigServerMetrics.HOSTED_VESPA_DOCKER_TOTAL_CAPACITY_DISK, EnumSet.of(max,average)); addMetric(metrics, ConfigServerMetrics.HOSTED_VESPA_DOCKER_TOTAL_CAPACITY_MEM, EnumSet.of(max,average)); - addMetric(metrics, ConfigServerMetrics.HOSTED_VESPA_DOCKER_SKEW.last()); - addMetric(metrics, ConfigServerMetrics.HOSTED_VESPA_PENDING_REDEPLOYMENTS.last()); + addMetric(metrics, ConfigServerMetrics.HOSTED_VESPA_DOCKER_SKEW, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last + addMetric(metrics, ConfigServerMetrics.HOSTED_VESPA_PENDING_REDEPLOYMENTS, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last addMetric(metrics, ConfigServerMetrics.HOSTED_VESPA_ACTIVE_HOSTS.max()); addMetric(metrics, ConfigServerMetrics.HOSTED_VESPA_DIRTY_HOSTS.max()); addMetric(metrics, ConfigServerMetrics.HOSTED_VESPA_FAILED_HOSTS.max()); addMetric(metrics, ConfigServerMetrics.HOSTED_VESPA_INACTIVE_HOSTS.max()); - addMetric(metrics, ConfigServerMetrics.HOSTED_VESPA_PROVISIONED_HOSTS.last()); + addMetric(metrics, ConfigServerMetrics.HOSTED_VESPA_PROVISIONED_HOSTS, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last addMetric(metrics, ConfigServerMetrics.HOSTED_VESPA_READY_HOSTS.max()); addMetric(metrics, ConfigServerMetrics.HOSTED_VESPA_RESERVED_HOSTS.max()); + addMetric(metrics, ConfigServerMetrics.HOSTED_VESPA_PARKED_HOSTS, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last + addMetric(metrics, ConfigServerMetrics.HOSTED_VESPA_ACTIVE_NODES.max()); addMetric(metrics, ConfigServerMetrics.HOSTED_VESPA_FAILED_NODES.max()); + addMetric(metrics, ConfigServerMetrics.HOSTED_VESPA_PARKED_NODES, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last addMetric(metrics, ConfigServerMetrics.RPC_SERVER_WORK_QUEUE_SIZE.average()); - addMetric(metrics, ConfigServerMetrics.DEPLOYMENT_ACTIVATE_MILLIS.last()); - addMetric(metrics, ConfigServerMetrics.DEPLOYMENT_PREPARE_MILLIS.last()); + addMetric(metrics, ConfigServerMetrics.DEPLOYMENT_ACTIVATE_MILLIS, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last + addMetric(metrics, ConfigServerMetrics.DEPLOYMENT_PREPARE_MILLIS, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last addMetric(metrics, ConfigServerMetrics.LOCK_ATTEMPT_LOCKED_LOAD, EnumSet.of(max, average)); - addMetric(metrics, ConfigServerMetrics.MAINTENANCE_SUCCESS_FACTOR_DEVIATION.last()); + addMetric(metrics, ConfigServerMetrics.MAINTENANCE_SUCCESS_FACTOR_DEVIATION, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last addMetric(metrics, ConfigServerMetrics.MAINTENANCE_DEPLOYMENT_FAILURE.count()); addMetric(metrics, ConfigServerMetrics.MAINTENANCE_DEPLOYMENT_TRANSIENT_FAILURE.count()); addMetric(metrics, ConfigServerMetrics.OVERCOMMITTED_HOSTS.max()); - addMetric(metrics, ConfigServerMetrics.SPARE_HOST_CAPACITY.last()); - addMetric(metrics, ConfigServerMetrics.THROTTLED_NODE_FAILURES.max()); + addMetric(metrics, ConfigServerMetrics.SPARE_HOST_CAPACITY, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last. + addMetric(metrics, ConfigServerMetrics.THROTTLED_HOST_FAILURES, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last + addMetric(metrics, ConfigServerMetrics.THROTTLED_NODE_FAILURES, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last + addMetric(metrics, ConfigServerMetrics.NODE_FAIL_THROTTLING, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last + + addMetric(metrics, ConfigServerMetrics.ORCHESTRATOR_LOCK_ACQUIRE_SUCCESS.count()); + addMetric(metrics, ConfigServerMetrics.ORCHESTRATOR_LOCK_ACQUIRE_TIMEOUT.count()); + addMetric(metrics, ConfigServerMetrics.ZONE_WORKING, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last + addMetric(metrics, ConfigServerMetrics.THROTTLED_HOST_PROVISIONING.max()); // Container metrics that should be stored for the config-server addMetric(metrics, ContainerMetrics.HANDLED_LATENCY.max()); @@ -111,11 +128,11 @@ public class InfrastructureMetricSet { addMetric(metrics, ContainerMetrics.HTTP_STATUS_2XX.count()); addMetric(metrics, ContainerMetrics.HTTP_STATUS_4XX.count()); addMetric(metrics, ContainerMetrics.HTTP_STATUS_5XX.count()); - addMetric(metrics, ContainerMetrics.JDISC_GC_MS.last()); + addMetric(metrics, ContainerMetrics.JDISC_GC_MS, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last. addMetric(metrics, ContainerMetrics.MEM_HEAP_USED.average()); addMetric(metrics, ContainerMetrics.SERVER_NUM_REQUESTS.count()); - addMetric(metrics, ContainerMetrics.SERVER_STARTED_MILLIS.last()); - addMetric(metrics, ContainerMetrics.SERVER_TOTAL_SUCCESSFUL_RESPONSE_LATENCY.last()); + addMetric(metrics, ContainerMetrics.SERVER_STARTED_MILLIS, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last. + addMetric(metrics, ContainerMetrics.SERVER_TOTAL_SUCCESSFUL_RESPONSE_LATENCY, EnumSet.of(max, sum, count, last)); // TODO: Vespa 9: Remove last. return metrics; } @@ -124,40 +141,42 @@ public class InfrastructureMetricSet { Set<Metric> metrics = new LinkedHashSet<>(); addMetric(metrics, ControllerMetrics.ATHENZ_REQUEST_ERROR.count()); - addMetric(metrics, ControllerMetrics.ARCHIVE_BUCKET_COUNT.last()); - addMetric(metrics, ControllerMetrics.BILLING_TENANTS.last()); + addMetric(metrics, ControllerMetrics.ARCHIVE_BUCKET_COUNT, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last + addMetric(metrics, ControllerMetrics.BILLING_TENANTS, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last addMetric(metrics, ControllerMetrics.DEPLOYMENT_ABORT.count()); - addMetric(metrics, ControllerMetrics.DEPLOYMENT_AVERAGE_DURATION, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last. + addMetric(metrics, ControllerMetrics.DEPLOYMENT_AVERAGE_DURATION, EnumSet.of(max, min, last)); // TODO: Vespa 9: Remove last. addMetric(metrics, ControllerMetrics.DEPLOYMENT_CONVERGENCE_FAILURE.count()); addMetric(metrics, ControllerMetrics.DEPLOYMENT_DEPLOYMENT_FAILURE.count()); addMetric(metrics, ControllerMetrics.DEPLOYMENT_ERROR.count()); - addMetric(metrics, ControllerMetrics.DEPLOYMENT_FAILING_UPGRADES.last()); - addMetric(metrics, ControllerMetrics.DEPLOYMENT_FAILURE_PERCENTAGE.last()); + addMetric(metrics, ControllerMetrics.DEPLOYMENT_FAILING_UPGRADES, EnumSet.of(min, last)); // TODO: Vespa 9: Remove last. + addMetric(metrics, ControllerMetrics.DEPLOYMENT_FAILURE_PERCENTAGE, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last. addMetric(metrics, ControllerMetrics.DEPLOYMENT_NODE_COUNT_BY_OS_VERSION.max()); addMetric(metrics, ControllerMetrics.DEPLOYMENT_OS_CHANGE_DURATION.max()); addMetric(metrics, ControllerMetrics.DEPLOYMENT_START.count()); addMetric(metrics, ControllerMetrics.DEPLOYMENT_SUCCESS.count()); addMetric(metrics, ControllerMetrics.DEPLOYMENT_TEST_FAILURE.count()); - addMetric(metrics, ControllerMetrics.DEPLOYMENT_WARNINGS.last()); + addMetric(metrics, ControllerMetrics.DEPLOYMENT_WARNINGS, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last. addMetric(metrics, ControllerMetrics.DEPLOYMENT_ENDPOINT_CERTIFICATE_TIMEOUT.count()); - addMetric(metrics, ControllerMetrics.DEPLOYMENT_BROKEN_SYSTEM_VERSION.last()); - - addMetric(metrics, ControllerMetrics.OPERATION_APPLICATION.last()); - addMetric(metrics, ControllerMetrics.OPERATION_CHANGEMANAGEMENT.last()); - addMetric(metrics, ControllerMetrics.OPERATION_CONFIGSERVER.last()); - addMetric(metrics, ControllerMetrics.OPERATION_CONTROLLER.last()); - addMetric(metrics, ControllerMetrics.OPERATION_FLAGS.last()); - addMetric(metrics, ControllerMetrics.OPERATION_OS.last()); - addMetric(metrics, ControllerMetrics.OPERATION_ROUTING.last()); - addMetric(metrics, ControllerMetrics.OPERATION_ZONE.last()); - - addMetric(metrics, ControllerMetrics.REMAINING_ROTATIONS.last()); - addMetric(metrics, ControllerMetrics.DNS_QUEUED_REQUESTS.last()); - addMetric(metrics, ControllerMetrics.ZMS_QUOTA_USAGE.last()); + addMetric(metrics, ControllerMetrics.DEPLOYMENT_BROKEN_SYSTEM_VERSION, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last + + addMetric(metrics, ControllerMetrics.OPERATION_APPLICATION, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last + addMetric(metrics, ControllerMetrics.OPERATION_CHANGEMANAGEMENT, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last + addMetric(metrics, ControllerMetrics.OPERATION_CONFIGSERVER, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last + addMetric(metrics, ControllerMetrics.OPERATION_CONTROLLER, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last + addMetric(metrics, ControllerMetrics.OPERATION_FLAGS, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last + addMetric(metrics, ControllerMetrics.OPERATION_OS, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last + addMetric(metrics, ControllerMetrics.OPERATION_ROUTING, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last + addMetric(metrics, ControllerMetrics.OPERATION_ZONE, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last + + addMetric(metrics, ControllerMetrics.REMAINING_ROTATIONS, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last + addMetric(metrics, ControllerMetrics.DNS_QUEUED_REQUESTS, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last + addMetric(metrics, ControllerMetrics.ZMS_QUOTA_USAGE, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last addMetric(metrics, ControllerMetrics.COREDUMP_PROCESSED.count()); + addMetric(metrics, ControllerMetrics.AUTH0_EXCEPTIONS.count()); - addMetric(metrics, ControllerMetrics.METERING_AGE_SECONDS.last()); + addMetric(metrics, ControllerMetrics.METERING_AGE_SECONDS, EnumSet.of(min, last)); // TODO: Vespa 9: Remove last + addMetric(metrics, ControllerMetrics.METERING_LAST_REPORTED, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last return metrics; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/Metric.java b/metrics/src/main/java/ai/vespa/metrics/set/Metric.java index b2df37c47b2..53ca75b7f30 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/Metric.java +++ b/metrics/src/main/java/ai/vespa/metrics/set/Metric.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.admin.monitoring; +package ai.vespa.metrics.set; import java.util.LinkedHashMap; import java.util.Map; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricSet.java b/metrics/src/main/java/ai/vespa/metrics/set/MetricSet.java index fb67100b435..b8409fb7663 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricSet.java +++ b/metrics/src/main/java/ai/vespa/metrics/set/MetricSet.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.admin.monitoring; +package ai.vespa.metrics.set; import java.util.Collection; import java.util.LinkedHashMap; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/NetworkMetrics.java b/metrics/src/main/java/ai/vespa/metrics/set/NetworkMetrics.java index 21e7d35fc7c..f967e2a2c31 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/NetworkMetrics.java +++ b/metrics/src/main/java/ai/vespa/metrics/set/NetworkMetrics.java @@ -1,8 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.admin.monitoring; +package ai.vespa.metrics.set; import ai.vespa.metrics.HostedNodeAdminMetrics; -import com.google.common.collect.ImmutableSet; import java.util.Set; @@ -17,7 +16,7 @@ public class NetworkMetrics { private static MetricSet createNetworkMetricSet() { Set<Metric> dockerNetworkMetrics = - ImmutableSet.of(new Metric(HostedNodeAdminMetrics.NET_IN_BYTES.baseName()), + Set.of(new Metric(HostedNodeAdminMetrics.NET_IN_BYTES.baseName()), new Metric(HostedNodeAdminMetrics.NET_IN_ERROR.baseName()), new Metric(HostedNodeAdminMetrics.NET_IN_DROPPED.baseName()), new Metric(HostedNodeAdminMetrics.NET_OUT_BYTES.baseName()), diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/SystemMetrics.java b/metrics/src/main/java/ai/vespa/metrics/set/SystemMetrics.java index 64f2a72d1b4..0560daebc43 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/SystemMetrics.java +++ b/metrics/src/main/java/ai/vespa/metrics/set/SystemMetrics.java @@ -1,9 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.admin.monitoring; +package ai.vespa.metrics.set; import ai.vespa.metrics.HostedNodeAdminMetrics; -import com.google.common.collect.ImmutableSet; +import java.util.LinkedHashSet; import java.util.Set; /** @@ -17,7 +17,7 @@ public class SystemMetrics { private static MetricSet createSystemMetricSet() { Set<Metric> dockerNodeMetrics = - ImmutableSet.of(new Metric(HostedNodeAdminMetrics.CPU_UTIL.baseName()), + Set.of(new Metric(HostedNodeAdminMetrics.CPU_UTIL.baseName()), new Metric(HostedNodeAdminMetrics.CPU_SYS_UTIL.baseName()), new Metric(HostedNodeAdminMetrics.CPU_THROTTLED_TIME.baseName()), new Metric(HostedNodeAdminMetrics.CPU_THROTTLED_CPU_TIME.baseName()), @@ -37,19 +37,19 @@ public class SystemMetrics { Set<Metric> nonDockerNodeMetrics = // Disk metrics should be based on /home, or else '/' - or simply add filesystem as dimension - ImmutableSet.of(new Metric("cpu.busy.pct", HostedNodeAdminMetrics.CPU_UTIL.baseName()), + Set.of(new Metric("cpu.busy.pct", HostedNodeAdminMetrics.CPU_UTIL.baseName()), new Metric("mem.used.pct", HostedNodeAdminMetrics.MEM_UTIL.baseName()), new Metric("mem.active.kb", HostedNodeAdminMetrics.MEM_USED.baseName()), new Metric("mem.total.kb", HostedNodeAdminMetrics.MEM_LIMIT.baseName()), new Metric("used.kb", HostedNodeAdminMetrics.DISK_USED.baseName()) ); - Set<Metric> systemMetrics = ImmutableSet.<Metric>builder() - .addAll(dockerNodeMetrics) - .addAll(nonDockerNodeMetrics) - .build(); - return new MetricSet("system", systemMetrics); + Set<Metric> metrics = new LinkedHashSet<>(); + metrics.addAll(dockerNodeMetrics); + metrics.addAll(nonDockerNodeMetrics); + + return new MetricSet("system", metrics); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/metrics/src/main/java/ai/vespa/metrics/set/VespaMetricSet.java index a0d866fb001..f9e37f4a85b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/metrics/src/main/java/ai/vespa/metrics/set/VespaMetricSet.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.admin.monitoring; +package ai.vespa.metrics.set; import ai.vespa.metrics.ClusterControllerMetrics; import ai.vespa.metrics.ContainerMetrics; @@ -28,7 +28,7 @@ import static ai.vespa.metrics.Suffix.ninety_five_percentile; import static ai.vespa.metrics.Suffix.ninety_nine_percentile; import static ai.vespa.metrics.Suffix.rate; import static ai.vespa.metrics.Suffix.sum; -import static com.yahoo.vespa.model.admin.monitoring.DefaultVespaMetrics.defaultVespaMetricSet; +import static ai.vespa.metrics.set.DefaultVespaMetrics.defaultVespaMetricSet; import static java.util.Collections.singleton; /** @@ -62,7 +62,7 @@ public class VespaMetricSet { Set<Metric> metrics = new LinkedHashSet<>(); addMetric(metrics, SentinelMetrics.SENTINEL_RESTARTS.count()); - addMetric(metrics, SentinelMetrics.SENTINEL_TOTAL_RESTARTS.last()); + addMetric(metrics, SentinelMetrics.SENTINEL_TOTAL_RESTARTS, EnumSet.of(sum, last)); // TODO: Vespa 9: Remove last addMetric(metrics, SentinelMetrics.SENTINEL_UPTIME.last()); addMetric(metrics, SentinelMetrics.SENTINEL_RUNNING, EnumSet.of(count, last)); @@ -173,10 +173,10 @@ public class VespaMetricSet { addMetric(metrics, ContainerMetrics.JDISC_GC_COUNT, EnumSet.of(average, max, last)); // TODO: Vespa 9: Remove last addMetric(metrics, ContainerMetrics.JDISC_GC_MS, EnumSet.of(average, max, last)); // TODO: Vespa 9: Remove last - addMetric(metrics, ContainerMetrics.JDISC_DEACTIVATED_CONTAINERS.last()); + addMetric(metrics, ContainerMetrics.JDISC_DEACTIVATED_CONTAINERS_TOTAL, EnumSet.of(sum, last)); // TODO: Vespa 9: Remove last addMetric(metrics, ContainerMetrics.JDISC_DEACTIVATED_CONTAINERS_WITH_RETAINED_REFS.last()); - addMetric(metrics, ContainerMetrics.JDISC_SINGLETON_IS_ACTIVE.last()); + addMetric(metrics, ContainerMetrics.JDISC_SINGLETON_IS_ACTIVE, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last addMetric(metrics, ContainerMetrics.JDISC_SINGLETON_ACTIVATION_COUNT.last()); addMetric(metrics, ContainerMetrics.JDISC_SINGLETON_ACTIVATION_FAILURE_COUNT.last()); addMetric(metrics, ContainerMetrics.JDISC_SINGLETON_ACTIVATION_MILLIS.last()); @@ -184,7 +184,7 @@ public class VespaMetricSet { addMetric(metrics, ContainerMetrics.JDISC_SINGLETON_DEACTIVATION_FAILURE_COUNT.last()); addMetric(metrics, ContainerMetrics.JDISC_SINGLETON_DEACTIVATION_MILLIS.last()); - addMetric(metrics, ContainerMetrics.ATHENZ_TENANT_CERT_EXPIRY_SECONDS.last()); + addMetric(metrics, ContainerMetrics.ATHENZ_TENANT_CERT_EXPIRY_SECONDS, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last addMetric(metrics, ContainerMetrics.CONTAINER_IAM_ROLE_EXPIRY_SECONDS.baseName()); addMetric(metrics, ContainerMetrics.HTTP_STATUS_1XX.rate()); @@ -222,10 +222,13 @@ public class VespaMetricSet { addMetric(metrics, ContainerMetrics.JDISC_JVM.last()); + addMetric(metrics, ContainerMetrics.FEED_LATENCY, EnumSet.of(sum, count, max)); + addMetric(metrics, ContainerMetrics.FEED_HTTP_REQUESTS, EnumSet.of(count, rate)); + // Deprecated metrics. TODO: Remove on Vespa 9. - addMetric(metrics, ContainerMetrics.SERVER_REJECTED_REQUESTS, EnumSet.of(rate, count)); // TODO: Remove on Vespa 9. Use jdisc.thread_pool.rejected_tasks. - addMetric(metrics, ContainerMetrics.SERVER_THREAD_POOL_SIZE, EnumSet.of(max, last)); // TODO: Remove on Vespa 9. Use jdisc.thread_pool.rejected_tasks. - addMetric(metrics, ContainerMetrics.SERVER_ACTIVE_THREADS, EnumSet.of(min, max, sum, count, last)); // TODO: Remove on Vespa 9. Use jdisc.thread_pool.rejected_tasks. + addMetric(metrics, ContainerMetrics.SERVER_REJECTED_REQUESTS, EnumSet.of(rate, count)); + addMetric(metrics, ContainerMetrics.SERVER_THREAD_POOL_SIZE, EnumSet.of(max, last)); + addMetric(metrics, ContainerMetrics.SERVER_ACTIVE_THREADS, EnumSet.of(min, max, sum, count, last)); addMetric(metrics, ContainerMetrics.JDISC_TLS_CAPABILITY_CHECKS_SUCCEEDED.rate()); addMetric(metrics, ContainerMetrics.JDISC_TLS_CAPABILITY_CHECKS_FAILED.rate()); @@ -236,10 +239,10 @@ public class VespaMetricSet { private static Set<Metric> getClusterControllerMetrics() { Set<Metric> metrics = new LinkedHashSet<>(); - addMetric(metrics, ClusterControllerMetrics.DOWN_COUNT.last()); - addMetric(metrics, ClusterControllerMetrics.INITIALIZING_COUNT.last()); - addMetric(metrics, ClusterControllerMetrics.MAINTENANCE_COUNT.last()); - addMetric(metrics, ClusterControllerMetrics.RETIRED_COUNT.last()); + addMetric(metrics, ClusterControllerMetrics.DOWN_COUNT, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last + addMetric(metrics, ClusterControllerMetrics.INITIALIZING_COUNT, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last + addMetric(metrics, ClusterControllerMetrics.MAINTENANCE_COUNT, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last + addMetric(metrics, ClusterControllerMetrics.RETIRED_COUNT, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last addMetric(metrics, ClusterControllerMetrics.STOPPING_COUNT.last()); addMetric(metrics, ClusterControllerMetrics.UP_COUNT.last()); addMetric(metrics, ClusterControllerMetrics.CLUSTER_STATE_CHANGE_COUNT.baseName()); @@ -248,7 +251,7 @@ public class VespaMetricSet { addMetric(metrics, ClusterControllerMetrics.WORK_MS, EnumSet.of(last, sum, count)); // TODO: Vespa 9: Remove last - addMetric(metrics, ClusterControllerMetrics.IS_MASTER.last()); + addMetric(metrics, ClusterControllerMetrics.IS_MASTER, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last addMetric(metrics, ClusterControllerMetrics.REMOTE_TASK_QUEUE_SIZE.last()); // TODO(hakonhall): Update this name once persistent "count" metrics has been implemented. // DO NOT RELY ON THIS METRIC YET. @@ -256,9 +259,9 @@ public class VespaMetricSet { addMetric(metrics, ClusterControllerMetrics.RESOURCE_USAGE_NODES_ABOVE_LIMIT, EnumSet.of(last, max)); // TODO: Vespa 9: Remove last addMetric(metrics, ClusterControllerMetrics.RESOURCE_USAGE_MAX_MEMORY_UTILIZATION, EnumSet.of(last, max)); // TODO: Vespa 9: Remove last addMetric(metrics, ClusterControllerMetrics.RESOURCE_USAGE_MAX_DISK_UTILIZATION, EnumSet.of(last, max)); // TODO: Vespa 9: Remove last - addMetric(metrics, ClusterControllerMetrics.RESOURCE_USAGE_MEMORY_LIMIT.last()); - addMetric(metrics, ClusterControllerMetrics.RESOURCE_USAGE_DISK_LIMIT.last()); - addMetric(metrics, ClusterControllerMetrics.REINDEXING_PROGRESS.last()); + addMetric(metrics, ClusterControllerMetrics.RESOURCE_USAGE_MEMORY_LIMIT, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last + addMetric(metrics, ClusterControllerMetrics.RESOURCE_USAGE_DISK_LIMIT, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last + addMetric(metrics, ClusterControllerMetrics.REINDEXING_PROGRESS, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last return metrics; } @@ -269,6 +272,9 @@ public class VespaMetricSet { // per chain metrics.add(new Metric("documents_processed.rate")); + addMetric(metrics, ContainerMetrics.DOCPROC_PROC_TIME, EnumSet.of(sum, count, max)); + addMetric(metrics, ContainerMetrics.DOCPROC_DOCUMENTS, EnumSet.of(sum, count, max, min)); + return metrics; } @@ -277,8 +283,6 @@ public class VespaMetricSet { addMetric(metrics, ContainerMetrics.PEAK_QPS.max()); addMetric(metrics, ContainerMetrics.SEARCH_CONNECTIONS, EnumSet.of(sum, count, max)); - addMetric(metrics, ContainerMetrics.FEED_LATENCY, EnumSet.of(sum, count, max)); - addMetric(metrics, ContainerMetrics.FEED_HTTP_REQUESTS, EnumSet.of(count, rate)); addMetric(metrics, ContainerMetrics.QUERIES.rate()); addMetric(metrics, ContainerMetrics.QUERY_CONTAINER_LATENCY, EnumSet.of(sum, count, max)); addMetric(metrics, ContainerMetrics.QUERY_LATENCY, EnumSet.of(sum, count, max, ninety_five_percentile, ninety_nine_percentile)); @@ -296,8 +300,6 @@ public class VespaMetricSet { addMetric(metrics, ContainerMetrics.TOTAL_HITS_PER_QUERY, EnumSet.of(sum, count, max, ninety_five_percentile, ninety_nine_percentile)); addMetric(metrics, ContainerMetrics.EMPTY_RESULTS.rate()); addMetric(metrics, ContainerMetrics.REQUESTS_OVER_QUOTA, EnumSet.of(rate, count)); - addMetric(metrics, ContainerMetrics.DOCPROC_PROC_TIME, EnumSet.of(sum, count, max)); - addMetric(metrics, ContainerMetrics.DOCPROC_DOCUMENTS, EnumSet.of(sum, count, max, min)); addMetric(metrics, ContainerMetrics.RELEVANCE_AT_1, EnumSet.of(sum, count)); addMetric(metrics, ContainerMetrics.RELEVANCE_AT_3, EnumSet.of(sum, count)); @@ -325,15 +327,15 @@ public class VespaMetricSet { addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_CONFIG_GENERATION.last()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_DOCUMENTS_TOTAL.last()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_DOCUMENTS_READY.last()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_DOCUMENTS_ACTIVE.last()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_DOCUMENTS_REMOVED.last()); + addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_DOCUMENTS_TOTAL, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last + addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_DOCUMENTS_READY, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last + addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_DOCUMENTS_ACTIVE, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last + addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_DOCUMENTS_REMOVED, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_INDEX_DOCS_IN_MEMORY.last()); + addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_INDEX_DOCS_IN_MEMORY, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_DISK_USAGE.last()); addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_MEMORY_USAGE_ALLOCATED_BYTES.max()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_HEART_BEAT_AGE.last()); + addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_HEART_BEAT_AGE, EnumSet.of(min, last)); // TODO: Vespa 9: Remove last addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCSUM_DOCS.rate()); addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCSUM_LATENCY, EnumSet.of(max, sum, count)); @@ -404,22 +406,22 @@ public class VespaMetricSet { // lid space addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_READY_LID_SPACE_LID_BLOAT_FACTOR.average()); addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_READY_LID_SPACE_LID_FRAGMENTATION_FACTOR.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_READY_LID_SPACE_LID_LIMIT.last()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_READY_LID_SPACE_HIGHEST_USED_LID.last()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_READY_LID_SPACE_USED_LIDS.last()); + addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_READY_LID_SPACE_LID_LIMIT, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last + addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_READY_LID_SPACE_HIGHEST_USED_LID, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last + addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_READY_LID_SPACE_USED_LIDS, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_NOTREADY_LID_SPACE_LID_BLOAT_FACTOR.average()); addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_NOTREADY_LID_SPACE_LID_FRAGMENTATION_FACTOR.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_NOTREADY_LID_SPACE_LID_LIMIT.last()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_NOTREADY_LID_SPACE_HIGHEST_USED_LID.last()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_NOTREADY_LID_SPACE_USED_LIDS.last()); + addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_NOTREADY_LID_SPACE_LID_LIMIT, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last + addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_NOTREADY_LID_SPACE_HIGHEST_USED_LID, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last + addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_NOTREADY_LID_SPACE_USED_LIDS, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_REMOVED_LID_SPACE_LID_BLOAT_FACTOR.average()); addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_REMOVED_LID_SPACE_LID_FRAGMENTATION_FACTOR.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_REMOVED_LID_SPACE_LID_LIMIT.last()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_REMOVED_LID_SPACE_HIGHEST_USED_LID.last()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_REMOVED_LID_SPACE_USED_LIDS.last()); + addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_REMOVED_LID_SPACE_LID_LIMIT, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last + addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_REMOVED_LID_SPACE_HIGHEST_USED_LID, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last + addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_REMOVED_LID_SPACE_USED_LIDS, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last // bucket move - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_BUCKET_MOVE_BUCKETS_PENDING.last()); + addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_BUCKET_MOVE_BUCKETS_PENDING, EnumSet.of(sum, last)); // TODO: Vespa 9: Remove last // resource usage addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_RESOURCE_USAGE_DISK.average()); @@ -432,7 +434,7 @@ public class VespaMetricSet { addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_RESOURCE_USAGE_MEMORY_USAGE_TRANSIENT.max()); addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_RESOURCE_USAGE_MEMORY_MAPPINGS.max()); addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_RESOURCE_USAGE_OPEN_FILE_DESCRIPTORS.max()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_RESOURCE_USAGE_FEEDING_BLOCKED.max()); + addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_RESOURCE_USAGE_FEEDING_BLOCKED, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_RESOURCE_USAGE_MALLOC_ARENA.max()); addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_ATTRIBUTE_RESOURCE_USAGE_ADDRESS_SPACE.max()); addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_ATTRIBUTE_RESOURCE_USAGE_FEEDING_BLOCKED.max()); @@ -447,7 +449,7 @@ public class VespaMetricSet { // transaction log addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_TRANSACTIONLOG_ENTRIES.average()); addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_TRANSACTIONLOG_DISK_USAGE.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_TRANSACTIONLOG_REPLAY_TIME.last()); + addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_TRANSACTIONLOG_REPLAY_TIME, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last // document store addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_READY_DOCUMENT_STORE_DISK_USAGE.average()); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/Cgroup.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/Cgroup.java index e40e3c6c003..9079aa6fc3f 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/Cgroup.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/Cgroup.java @@ -24,7 +24,7 @@ import java.util.logging.Logger; public class Cgroup { private static final Logger logger = Logger.getLogger(Cgroup.class.getName()); - private static Map<String, Consumer<UnixPath>> cgroupDirectoryCallbacks = new HashMap<>(); + private static final Map<String, Consumer<UnixPath>> cgroupDirectoryCallbacks = new HashMap<>(); private final Path root; private final Path relativePath; @@ -135,6 +135,9 @@ public class Cgroup { /** Returns the memory controller of this cgroup (memory.* files). */ public MemoryController memory() { return new MemoryController(this); } + /** Returns the IO controller of this cgroup (io.* files). */ + public IoController io() { return new IoController(this); } + /** * Wraps {@code command} to ensure it is executed in this cgroup. * diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/IoController.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/IoController.java new file mode 100644 index 00000000000..5bbdd5c3b70 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/IoController.java @@ -0,0 +1,111 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.cgroup; + +import ai.vespa.validation.Validation; +import com.yahoo.vespa.hosted.node.admin.component.TaskContext; +import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; + +import java.util.Map; +import java.util.Optional; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +import static java.lang.Integer.parseInt; + +/** + * Represents a cgroup v2 IO controller, i.e. all io.* files. + * + * @author freva + */ +public class IoController { + private static final Logger logger = Logger.getLogger(IoController.class.getName()); + private final Cgroup cgroup; + + IoController(Cgroup cgroup) { + this.cgroup = cgroup; + } + + public record Device(int major, int minor) implements Comparable<Device> { + public Device { + // https://www.halolinux.us/kernel-architecture/representation-of-major-and-minor-numbers.html + Validation.requireInRange(major, "device major", 0, 0xFFF); + Validation.requireInRange(minor, "device minor", 0, 0xFFFFF); + } + + private String toFileContent() { return major + ":" + minor; } + private static Device fromString(String device) { + String[] parts = device.split(":"); + return new Device(parseInt(parts[0]), parseInt(parts[1])); + } + + @Override + public int compareTo(Device o) { + return major != o.major ? Integer.compare(major, o.major) : Integer.compare(minor, o.minor); + } + } + + /** + * Defines max allowed IO: + * <ul> + * <li><b>rbps</b>: Read bytes per seconds</li> + * <li><b>riops</b>: Read IO operations per seconds</li> + * <li><b>wbps</b>: Write bytes per seconds</li> + * <li><b>wiops</b>: Write IO operations per seconds</li> + * </ul>. + */ + public record Max(Size rbps, Size wbps, Size riops, Size wiops) { + public static Max UNLIMITED = new Max(Size.max(), Size.max(), Size.max(), Size.max()); + + // Keys can be specified in any order, this is the order they are outputted in from io.max + // https://github.com/torvalds/linux/blob/c1a515d3c0270628df8ae5f5118ba859b85464a2/block/blk-throttle.c#L1541 + private String toFileContent() { return "rbps=%s wbps=%s riops=%s wiops=%s".formatted(rbps, wbps, riops, wiops); } + + public static Max fromString(String max) { + String[] parts = max.split(" "); + Size rbps = Size.max(), riops = Size.max(), wbps = Size.max(), wiops = Size.max(); + for (String part : parts) { + if (part.isEmpty()) continue; + String[] kv = part.split("="); + if (kv.length != 2) throw new IllegalArgumentException("Invalid io.max format: " + max); + switch (kv[0]) { + case "rbps" -> rbps = Size.from(kv[1]); + case "riops" -> riops = Size.from(kv[1]); + case "wbps" -> wbps = Size.from(kv[1]); + case "wiops" -> wiops = Size.from(kv[1]); + default -> throw new IllegalArgumentException("Unknown key " + kv[0]); + } + } + return new Max(rbps, wbps, riops, wiops); + } + } + + /** + * Returns the maximum allowed IO usage, by device, or empty if cgroup is not found. + * + * @see Max + */ + public Optional<Map<Device, Max>> readMax() { + return cgroup.readIfExists("io.max") + .map(content -> content + .lines() + .map(line -> { + String[] parts = line.strip().split(" ", 2); + return Map.entry(Device.fromString(parts[0]), Max.fromString(parts[1])); + }) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))); + } + + public boolean updateMax(TaskContext context, Device device, Max max) { + Max prevMax = readMax() + .map(maxByDevice -> maxByDevice.get(device)) + .orElse(Max.UNLIMITED); + if (prevMax.equals(max)) return false; + + UnixPath path = cgroup.unixPath().resolve("io.max"); + context.recordSystemModification(logger, "Updating %s for device %s from '%s' to '%s'", + path, device.toFileContent(), prevMax.toFileContent(), max.toFileContent()); + path.writeUtf8File(device.toFileContent() + ' ' + max.toFileContent() + '\n'); + return true; + } + +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/Size.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/Size.java index 5e6ca7de8bd..a8cbe2e8afe 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/Size.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/Size.java @@ -10,12 +10,13 @@ import java.util.Objects; */ public class Size { private static final String MAX = "max"; + private static final Size MAX_SIZE = new Size(true, 0); private final boolean max; private final long value; public static Size max() { - return new Size(true, 0); + return MAX_SIZE; } public static Size from(long value) { @@ -23,7 +24,7 @@ public class Size { } public static Size from(String value) { - return value.equals(MAX) ? new Size(true, 0) : new Size(false, Long.parseLong(value)); + return value.equals(MAX) ? MAX_SIZE : new Size(false, Long.parseLong(value)); } private Size(boolean max, long value) { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperations.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperations.java index 264035b86a1..fa933e9622a 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperations.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperations.java @@ -36,7 +36,7 @@ public class ContainerOperations { public ContainerOperations(ContainerEngine containerEngine, Cgroup cgroup, FileSystem fileSystem, Timer timer) { this.containerEngine = Objects.requireNonNull(containerEngine); - this.imageDownloader = new ContainerImageDownloader(containerEngine); + this.imageDownloader = new ContainerImageDownloader(containerEngine, timer); this.imagePruner = new ContainerImagePruner(containerEngine, timer); this.containerStatsCollector = new ContainerStatsCollector(containerEngine, cgroup, fileSystem); } @@ -62,8 +62,8 @@ public class ContainerOperations { } /** Pull image asynchronously. Returns true if image is still downloading and false if download is complete */ - public boolean pullImageAsyncIfNeeded(TaskContext context, DockerImage dockerImage, RegistryCredentials registryCredentials) { - return !imageDownloader.get(context, dockerImage, registryCredentials); + public boolean pullImageAsyncIfNeeded(TaskContext context, DockerImage dockerImage, RegistryCredentialsProvider credentialsProvider) { + return !imageDownloader.get(context, dockerImage, credentialsProvider); } /** Executes a command inside container identified by given context. Does NOT throw on non-zero exit code */ diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImageDownloader.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImageDownloader.java index 1e37e080528..d3327bf5148 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImageDownloader.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImageDownloader.java @@ -3,10 +3,13 @@ package com.yahoo.vespa.hosted.node.admin.container.image; import com.yahoo.concurrent.DaemonThreadFactory; import com.yahoo.config.provision.DockerImage; +import com.yahoo.jdisc.Timer; import com.yahoo.vespa.hosted.node.admin.component.TaskContext; import com.yahoo.vespa.hosted.node.admin.container.ContainerEngine; -import com.yahoo.vespa.hosted.node.admin.container.RegistryCredentials; +import com.yahoo.vespa.hosted.node.admin.container.RegistryCredentialsProvider; +import java.time.Duration; +import java.time.Instant; import java.util.Collections; import java.util.HashSet; import java.util.Objects; @@ -26,13 +29,15 @@ public class ContainerImageDownloader { private static final Logger LOG = Logger.getLogger(ContainerImageDownloader.class.getName()); private final ContainerEngine containerEngine; + private final Timer timer; private final ExecutorService executorService = Executors.newSingleThreadExecutor( new DaemonThreadFactory("container-image-downloader")); // Download one image at a time private final Set<DockerImage> pendingDownloads = Collections.synchronizedSet(new HashSet<>()); - public ContainerImageDownloader(ContainerEngine containerEngine) { + public ContainerImageDownloader(ContainerEngine containerEngine, Timer timer) { this.containerEngine = Objects.requireNonNull(containerEngine); + this.timer = Objects.requireNonNull(timer); } /** @@ -40,12 +45,14 @@ public class ContainerImageDownloader { * * @return true if the image download has completed. */ - public boolean get(TaskContext context, DockerImage image, RegistryCredentials registryCredentials) { + public boolean get(TaskContext context, DockerImage image, RegistryCredentialsProvider credentialsProvider) { if (pendingDownloads.contains(image)) return false; if (containerEngine.hasImage(context, image)) return true; executorService.submit(() -> { try { - containerEngine.pullImage(context, image, registryCredentials); + Instant start = timer.currentTime(); + containerEngine.pullImage(context, image, credentialsProvider.get()); + LOG.log(Level.INFO, "Downloaded container image " + image + " in " + Duration.between(start, timer.currentTime())); } catch (RuntimeException e) { LOG.log(Level.SEVERE, "Failed to download container image " + image, e); } finally { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/metrics/Metrics.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/metrics/Metrics.java index 07a8d545178..e9dbfa0c524 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/metrics/Metrics.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/metrics/Metrics.java @@ -102,6 +102,17 @@ public class Metrics { } } + public void deleteMetricByName(String application, String metricName, DimensionType type) { + synchronized (monitor) { + Optional.ofNullable(metrics.get(type)) + .map(m -> m.get(application)) + .map(ApplicationMetrics::metricsByDimensions) + .ifPresent(dims -> + dims.values().forEach(metrics -> metrics.remove(metricName)) + ); + } + } + Map<Dimensions, Map<String, MetricValue>> getOrCreateApplicationMetrics(String application, DimensionType type) { return metrics.computeIfAbsent(type, m -> new HashMap<>()) .computeIfAbsent(application, app -> new ApplicationMetrics()) diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java index b6ec0ebbd94..830b7f4ed33 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java @@ -80,7 +80,6 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { private final String certificateDnsSuffix; private final ServiceIdentityProvider hostIdentityProvider; private final IdentityDocumentClient identityDocumentClient; - private final BooleanFlag tenantServiceIdentityFlag; // Used as an optimization to ensure ZTS is not DDoS'ed on continuously failing refresh attempts private final Map<ContainerName, Instant> lastRefreshAttempt = new ConcurrentHashMap<>(); @@ -89,7 +88,6 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { ConfigServerInfo configServerInfo, String certificateDnsSuffix, ServiceIdentityProvider hostIdentityProvider, - FlagSource flagSource, Timer timer) { this.ztsTrustStorePath = ztsTrustStorePath; this.certificateDnsSuffix = certificateDnsSuffix; @@ -99,7 +97,6 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { hostIdentityProvider, new AthenzIdentityVerifier(Set.of(configServerInfo.getConfigServerIdentity()))); this.timer = timer; - this.tenantServiceIdentityFlag = Flags.NODE_ADMIN_TENANT_SERVICE_REGISTRY.bindTo(flagSource); } public boolean converge(NodeAgentContext context) { @@ -109,11 +106,7 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { if (context.zone().getSystemName().isPublic()) return modified; - if (shouldWriteTenantServiceIdentity(context)) { - modified |= maintain(context, TENANT); - } else { - modified |= deleteTenantCredentials(context); - } + modified |= maintain(context, TENANT); return modified; } @@ -268,24 +261,6 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { return "node-certificate"; } - private boolean deleteTenantCredentials(NodeAgentContext context) { - var siaDirectory = context.paths().of(CONTAINER_SIA_DIRECTORY, context.users().vespa()); - var identityDocumentFile = siaDirectory.resolve(TENANT.getIdentityDocument()); - if (!Files.exists(identityDocumentFile)) return false; - return getAthenzIdentity(context, TENANT, identityDocumentFile).map(athenzIdentity -> { - var privateKeyFile = (ContainerPath) SiaUtils.getPrivateKeyFile(siaDirectory, athenzIdentity); - var certificateFile = (ContainerPath) SiaUtils.getCertificateFile(siaDirectory, athenzIdentity); - try { - var modified = Files.deleteIfExists(identityDocumentFile); - modified |= Files.deleteIfExists(privateKeyFile); - modified |= Files.deleteIfExists(certificateFile); - return modified; - } catch (IOException e) { - throw new UncheckedIOException(e); - } - }).orElse(false); - } - private boolean shouldRefreshCredentials(Duration age) { return age.compareTo(REFRESH_PERIOD) >= 0; } @@ -399,16 +374,6 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { } } - private boolean shouldWriteTenantServiceIdentity(NodeAgentContext context) { - var version = context.node().currentVespaVersion() - .orElse(context.node().wantedVespaVersion().orElse(Version.emptyVersion)); - var appId = context.node().owner().orElse(ApplicationId.defaultId()); - return tenantServiceIdentityFlag - .with(FetchVector.Dimension.VESPA_VERSION, version.toFullString()) - .with(FetchVector.Dimension.APPLICATION_ID, appId.serializedForm()) - .value(); - } - private void copyCredsToLegacyPath(NodeAgentContext context, ContainerPath privateKeyFile, ContainerPath certificateFile) throws IOException { var legacySiaDirectory = context.paths().of(LEGACY_SIA_DIRECTORY, context.users().vespa()); var keysDirectory = legacySiaDirectory.resolve("keys"); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfo.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfo.java index c743f1c8c85..feafe9fddc9 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfo.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfo.java @@ -81,6 +81,9 @@ public class SyncFileInfo { remoteFilename = rotatedOnly && filename.endsWith(".0.log") ? "zookeeper.log" : "zookeeper.log-" + DATE_TIME_FORMATTER.format(new UnixPath(logFile).getLastModifiedTime()); minDurationBetweenSync = filename.endsWith(".0.log") ? rotatedOnly ? Duration.ofHours(1) : Duration.ZERO : null; + } else if (filename.startsWith("start-services.out-")) { + compression = Compression.ZSTD; + dir = "logs/start-services/"; } else { compression = filename.endsWith(".zst") ? Compression.NONE : Compression.ZSTD; if (rotatedOnly && compression != Compression.NONE) diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java index 4c17bfbe039..284306e1e8c 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java @@ -17,11 +17,9 @@ import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeState; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.reports.DropDocumentsReport; import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Orchestrator; -import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.OrchestratorException; import com.yahoo.vespa.hosted.node.admin.container.Container; import com.yahoo.vespa.hosted.node.admin.container.ContainerOperations; import com.yahoo.vespa.hosted.node.admin.container.ContainerResources; -import com.yahoo.vespa.hosted.node.admin.container.RegistryCredentials; import com.yahoo.vespa.hosted.node.admin.container.RegistryCredentialsProvider; import com.yahoo.vespa.hosted.node.admin.maintenance.ContainerWireguardTask; import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer; @@ -431,9 +429,8 @@ public class NodeAgentImpl implements NodeAgent { NodeSpec node = context.node(); if (node.wantedDockerImage().equals(container.map(c -> c.image()))) return false; - RegistryCredentials credentials = registryCredentialsProvider.get(); return node.wantedDockerImage() - .map(image -> containerOperations.pullImageAsyncIfNeeded(context, image, credentials)) + .map(image -> containerOperations.pullImageAsyncIfNeeded(context, image, registryCredentialsProvider)) .orElse(false); } @@ -486,18 +483,20 @@ public class NodeAgentImpl implements NodeAgent { lastNode = node; } + // Run this here and now, even though we may immediately remove the container below. + // This ensures these maintainers are run even if something fails or returns early. + // These maintainers should also run immediately after starting the container (see below). + container.ifPresent(c -> runImportantContainerMaintainers(context, c)); + switch (node.state()) { - case ready: - case reserved: - case failed: - case inactive: - case parked: + case ready, reserved, failed, inactive, parked -> { storageMaintainer.syncLogs(context, true); + if (node.state() == NodeState.reserved) downloadImageIfNeeded(context, container); removeContainerIfNeededUpdateContainerState(context, container); updateNodeRepoWithCurrentAttributes(context, Optional.empty()); stopServicesIfNeeded(context); - break; - case active: + } + case active -> { storageMaintainer.syncLogs(context, true); storageMaintainer.cleanDiskIfFull(context); storageMaintainer.handleCoreDumpsForContainer(context, container, false); @@ -513,13 +512,11 @@ public class NodeAgentImpl implements NodeAgent { containerState = STARTING; container = Optional.of(startContainer(context)); containerState = UNKNOWN; + runImportantContainerMaintainers(context, container.get()); } else { container = Optional.of(updateContainerIfNeeded(context, container.get())); } - aclMaintainer.ifPresent(maintainer -> maintainer.converge(context)); - final Optional<Container> finalContainer = container; - wireguardTasks.forEach(task -> task.converge(context, finalContainer.get().id())); startServicesIfNeeded(context); resumeNodeIfNeeded(context); if (healthChecker.isPresent()) { @@ -550,11 +547,8 @@ public class NodeAgentImpl implements NodeAgent { orchestrator.resume(context.hostname().value()); suspendedInOrchestrator = false; } - break; - case provisioned: - nodeRepository.setNodeState(context.hostname().value(), NodeState.ready); - break; - case dirty: + } + case dirty -> { removeContainerIfNeededUpdateContainerState(context, container); context.log(logger, "State is " + node.state() + ", will delete application storage and mark node as ready"); credentialsMaintainers.forEach(maintainer -> maintainer.clearCredentials(context)); @@ -562,12 +556,16 @@ public class NodeAgentImpl implements NodeAgent { storageMaintainer.archiveNodeStorage(context); updateNodeRepoWithCurrentAttributes(context, Optional.empty()); nodeRepository.setNodeState(context.hostname().value(), NodeState.ready); - break; - default: - throw ConvergenceException.ofError("UNKNOWN STATE " + node.state().name()); + } + default -> throw ConvergenceException.ofError("Unexpected state " + node.state().name()); } } + private void runImportantContainerMaintainers(NodeAgentContext context, Container container) { + aclMaintainer.ifPresent(maintainer -> maintainer.converge(context)); + wireguardTasks.forEach(task -> task.converge(context, container.id())); + } + private static void logChangesToNodeSpec(NodeAgentContext context, NodeSpec lastNode, NodeSpec node) { StringBuilder builder = new StringBuilder(); appendIfDifferent(builder, "state", lastNode, node, NodeSpec::state); @@ -609,23 +607,8 @@ public class NodeAgentImpl implements NodeAgent { if (context.node().state() != NodeState.active) return; context.log(logger, "Ask Orchestrator for permission to suspend node"); - try { - orchestrator.suspend(context.hostname().value()); - suspendedInOrchestrator = true; - } catch (OrchestratorException e) { - // Ensure the ACLs are up to date: The reason we're unable to suspend may be because some other - // node is unable to resume because the ACL rules of SOME Docker container is wrong... - // Same can happen with stale WireGuard config, so update that too - try { - aclMaintainer.ifPresent(maintainer -> maintainer.converge(context)); - wireguardTasks.forEach(task -> getContainer(context).ifPresent(c -> task.converge(context, c.id()))); - } catch (RuntimeException suppressed) { - logger.log(Level.WARNING, "Suppressing ACL update failure: " + suppressed); - e.addSuppressed(suppressed); - } - - throw e; - } + orchestrator.suspend(context.hostname().value()); + suspendedInOrchestrator = true; } protected void writeContainerData(NodeAgentContext context, ContainerData containerData) { } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributes.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributes.java index 332b4e61dc1..c638fe98cdf 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributes.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributes.java @@ -13,36 +13,13 @@ import java.util.Set; * * @author hakonhall */ -public class FileAttributes { - - private final Instant lastModifiedTime; - private final int ownerId; - private final int groupId; - private final String permissions; - private final boolean isRegularFile; - private final boolean isDirectory; - private final long size; - - public FileAttributes(Instant lastModifiedTime, int ownerId, int groupId, String permissions, boolean isRegularFile, boolean isDirectory, long size) { - this.lastModifiedTime = lastModifiedTime; - this.ownerId = ownerId; - this.groupId = groupId; - this.permissions = permissions; - this.isRegularFile = isRegularFile; - this.isDirectory = isDirectory; - this.size = size; - } - - public Instant lastModifiedTime() { return lastModifiedTime; } - public int ownerId() { return ownerId; } - public int groupId() { return groupId; } - public String permissions() { return permissions; } - public boolean isRegularFile() { return isRegularFile; } - public boolean isDirectory() { return isDirectory; } - public long size() { return size; } +public record FileAttributes(Instant lastModifiedTime, int ownerId, int groupId, String permissions, + boolean isRegularFile, boolean isDirectory, long size, int deviceMajor, int deviceMinor) { @SuppressWarnings("unchecked") static FileAttributes fromAttributes(Map<String, Object> attributes) { + long dev_t = (long) attributes.get("dev"); + return new FileAttributes( ((FileTime) attributes.get("lastModifiedTime")).toInstant(), (int) attributes.get("uid"), @@ -50,6 +27,11 @@ public class FileAttributes { PosixFilePermissions.toString(((Set<PosixFilePermission>) attributes.get("permissions"))), (boolean) attributes.get("isRegularFile"), (boolean) attributes.get("isDirectory"), - (long) attributes.get("size")); + (long) attributes.get("size"), + deviceMajor(dev_t), deviceMinor(dev_t)); } + + // Encoded as MMMM Mmmm mmmM MMmm, where M is a hex digit of the major number and m is a hex digit of the minor number. + static int deviceMajor(long dev_t) { return (int) (((dev_t & 0xFFFFF00000000000L) >> 32) | ((dev_t & 0xFFF00) >> 8)); } + static int deviceMinor(long dev_t) { return (int) (((dev_t & 0x00000FFFFFF00000L) >> 12) | (dev_t & 0x000FF)); } } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/cgroup/CgroupTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/cgroup/CgroupTest.java index 27580082020..c93d90329f7 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/cgroup/CgroupTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/cgroup/CgroupTest.java @@ -1,6 +1,4 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.cgroup; import com.yahoo.vespa.hosted.node.admin.container.ContainerId; @@ -10,7 +8,6 @@ import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; import com.yahoo.vespa.test.file.TestFileSystem; import org.junit.jupiter.api.Test; -import java.io.IOException; import java.nio.file.FileSystem; import java.util.Map; import java.util.Optional; @@ -23,6 +20,8 @@ import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.StatField.T import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.StatField.USER_USAGE_USEC; import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.sharesToWeight; import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.weightToShares; +import static com.yahoo.vespa.hosted.node.admin.cgroup.IoController.Device; +import static com.yahoo.vespa.hosted.node.admin.cgroup.IoController.Max; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -75,35 +74,39 @@ public class CgroupTest { } @Test - public void reads_cpu_stats() throws IOException { - cgroupRoot.resolve("cpu.stat").writeUtf8File("usage_usec 17794243\n" + - "user_usec 16099205\n" + - "system_usec 1695038\n" + - "nr_periods 12465\n" + - "nr_throttled 25\n" + - "throttled_usec 14256\n"); + public void reads_cpu_stats() { + cgroupRoot.resolve("cpu.stat").writeUtf8File(""" + usage_usec 17794243 + user_usec 16099205 + system_usec 1695038 + nr_periods 12465 + nr_throttled 25 + throttled_usec 14256 + """); assertEquals(Map.of(TOTAL_USAGE_USEC, 17794243L, USER_USAGE_USEC, 16099205L, SYSTEM_USAGE_USEC, 1695038L, TOTAL_PERIODS, 12465L, THROTTLED_PERIODS, 25L, THROTTLED_TIME_USEC, 14256L), containerCgroup.cpu().readStats()); } @Test - public void reads_memory_metrics() throws IOException { + public void reads_memory_metrics() { cgroupRoot.resolve("memory.current").writeUtf8File("2525093888\n"); assertEquals(2525093888L, containerCgroup.memory().readCurrent().value()); cgroupRoot.resolve("memory.max").writeUtf8File("4322885632\n"); assertEquals(4322885632L, containerCgroup.memory().readMax().value()); - cgroupRoot.resolve("memory.stat").writeUtf8File("anon 3481600\n" + - "file 69206016\n" + - "kernel_stack 73728\n" + - "slab 3552304\n" + - "percpu 262336\n" + - "sock 73728\n" + - "shmem 8380416\n" + - "file_mapped 1081344\n" + - "file_dirty 135168\n"); + cgroupRoot.resolve("memory.stat").writeUtf8File(""" + anon 3481600 + file 69206016 + kernel_stack 73728 + slab 3552304 + percpu 262336 + sock 73728 + shmem 8380416 + file_mapped 1081344 + file_dirty 135168 + """); assertEquals(69206016L, containerCgroup.memory().readFileSystemCache().value()); } @@ -117,4 +120,37 @@ public class CgroupTest { () -> "Original shares: " + originalShares + ", round trip shares: " + roundTripShares + ", diff: " + diff); } } + + @Test + void reads_io_max() { + assertEquals(Optional.empty(), containerCgroup.io().readMax()); + + cgroupRoot.resolve("io.max").writeUtf8File(""); + assertEquals(Optional.of(Map.of()), containerCgroup.io().readMax()); + + cgroupRoot.resolve("io.max").writeUtf8File(""" + 253:1 rbps=11 wbps=max riops=22 wiops=33 + 253:0 rbps=max wbps=44 riops=max wiops=55 + """); + assertEquals(Map.of(new Device(253, 1), new Max(Size.from(11), Size.max(), Size.from(22), Size.from(33)), + new Device(253, 0), new Max(Size.max(), Size.from(44), Size.max(), Size.from(55))), + containerCgroup.io().readMax().orElseThrow()); + } + + @Test + void writes_io_max() { + Device device = new Device(253, 0); + Max initial = new Max(Size.max(), Size.from(44), Size.max(), Size.from(55)); + assertTrue(containerCgroup.io().updateMax(context, device, initial)); + assertEquals("253:0 rbps=max wbps=44 riops=max wiops=55\n", cgroupRoot.resolve("io.max").readUtf8File()); + + cgroupRoot.resolve("io.max").writeUtf8File(""" + 253:1 rbps=11 wbps=max riops=22 wiops=33 + 253:0 rbps=max wbps=44 riops=max wiops=55 + """); + assertFalse(containerCgroup.io().updateMax(context, device, initial)); + + cgroupRoot.resolve("io.max").writeUtf8File(""); + assertFalse(containerCgroup.io().updateMax(context, device, Max.UNLIMITED)); + } } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/cgroup/IoControllerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/cgroup/IoControllerTest.java new file mode 100644 index 00000000000..71a05eb4571 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/cgroup/IoControllerTest.java @@ -0,0 +1,19 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.cgroup; + +import org.junit.jupiter.api.Test; + +import static com.yahoo.vespa.hosted.node.admin.cgroup.IoController.Max; +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * @author freva + */ +class IoControllerTest { + + @Test + void parse_io_max() { + assertEquals(Max.UNLIMITED, Max.fromString("")); + assertEquals(new Max(Size.from(1), Size.max(), Size.max(), Size.max()), Max.fromString("rbps=1 wiops=max")); + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImageDownloaderTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImageDownloaderTest.java index 9fd14e7e665..7f002eee315 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImageDownloaderTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImageDownloaderTest.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.node.admin.container.image; import com.yahoo.config.provision.DockerImage; +import com.yahoo.jdisc.test.TestTimer; import com.yahoo.vespa.hosted.node.admin.component.TaskContext; import com.yahoo.vespa.hosted.node.admin.component.TestTaskContext; import com.yahoo.vespa.hosted.node.admin.container.ContainerEngineMock; @@ -21,15 +22,15 @@ public class ContainerImageDownloaderTest { @Timeout(5_000) void test_download() { ContainerEngineMock podman = new ContainerEngineMock().asyncImageDownload(true); - ContainerImageDownloader downloader = new ContainerImageDownloader(podman); + ContainerImageDownloader downloader = new ContainerImageDownloader(podman, new TestTimer()); TaskContext context = new TestTaskContext(); DockerImage image = DockerImage.fromString("registry.example.com/repo/vespa:7.42"); - assertFalse(downloader.get(context, image, RegistryCredentials.none), "Download started"); - assertFalse(downloader.get(context, image, RegistryCredentials.none), "Download pending"); + assertFalse(downloader.get(context, image, () -> RegistryCredentials.none), "Download started"); + assertFalse(downloader.get(context, image, () -> RegistryCredentials.none), "Download pending"); podman.completeDownloadOf(image); boolean downloadCompleted; - while (!(downloadCompleted = downloader.get(context, image, RegistryCredentials.none))) ; + while (!(downloadCompleted = downloader.get(context, image, () -> RegistryCredentials.none))) ; assertTrue(downloadCompleted, "Download completed"); } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java index b7aee6706b1..f10f3db9b59 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java @@ -38,6 +38,8 @@ public class SyncFileInfoTest { private static final Path vespaLogPath2 = fileSystem.getPath("/opt/vespa/logs/vespa.log-2021-02-12"); private static final Path zkLogPath0 = fileSystem.getPath("/opt/vespa/logs/zookeeper.configserver.0.log"); private static final Path zkLogPath1 = fileSystem.getPath("/opt/vespa/logs/zookeeper.configserver.1.log"); + private static final Path startServicesPath1 = fileSystem.getPath("/opt/vespa/logs/start-services.out"); + private static final Path startServicesPath2 = fileSystem.getPath("/opt/vespa/logs/start-services.out-20230808100143"); @Test void access_logs() { @@ -93,6 +95,12 @@ public class SyncFileInfoTest { assertForLogFile(zkLogPath1, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/zookeeper/zookeeper.log-2022-05-09.14-22-11.zst", ZSTD, false); } + @Test + void start_services() { + assertForLogFile(startServicesPath1, null, null, true); + assertForLogFile(startServicesPath2, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/start-services/start-services.out-20230808100143.zst", ZSTD, true); + } + private static void assertForLogFile(Path srcPath, String destination, SyncFileInfo.Compression compression, boolean rotatedOnly) { assertForLogFile(srcPath, destination, compression, null, rotatedOnly); } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java index 0913e1d040a..ef4d6d849f6 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java @@ -487,20 +487,6 @@ public class NodeAgentImplTest { } @Test - void provisionedNodeIsMarkedAsReady() { - final NodeSpec node = nodeBuilder(NodeState.provisioned) - .wantedDockerImage(dockerImage) - .build(); - - NodeAgentContext context = createContext(node); - NodeAgentImpl nodeAgent = makeNodeAgent(null, false); - when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node)); - - nodeAgent.doConverge(context); - verify(nodeRepository, times(1)).setNodeState(eq(hostName), eq(NodeState.ready)); - } - - @Test void testRestartDeadContainerAfterNodeAdminRestart() { final NodeSpec node = nodeBuilder(NodeState.active) .currentDockerImage(dockerImage).wantedDockerImage(dockerImage) diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributesCacheTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributesCacheTest.java index 8c9188a9409..1b68d1d10a3 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributesCacheTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributesCacheTest.java @@ -3,9 +3,12 @@ package com.yahoo.vespa.hosted.node.admin.task.util.file; import org.junit.jupiter.api.Test; +import java.time.Instant; import java.util.Optional; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -23,7 +26,8 @@ public class FileAttributesCacheTest { verify(unixPath, times(1)).getAttributesIfExists(); verifyNoMoreInteractions(unixPath); - FileAttributes attributes = mock(FileAttributes.class); + FileAttributes attributes = new FileAttributes(Instant.EPOCH, 0, 0, "", false, false, 0, 0, 0); + when(unixPath.getAttributesIfExists()).thenReturn(Optional.of(attributes)); when(unixPath.getAttributesIfExists()).thenReturn(Optional.of(attributes)); assertTrue(cache.get().isPresent()); verify(unixPath, times(1 + 1)).getAttributesIfExists(); @@ -32,4 +36,4 @@ public class FileAttributesCacheTest { assertEquals(attributes, cache.getOrThrow()); verifyNoMoreInteractions(unixPath); } -}
\ No newline at end of file +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributesTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributesTest.java new file mode 100644 index 00000000000..ddcd225a871 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileAttributesTest.java @@ -0,0 +1,20 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.file; + +import org.junit.jupiter.api.Test; + +import static com.yahoo.vespa.hosted.node.admin.task.util.file.FileAttributes.deviceMajor; +import static com.yahoo.vespa.hosted.node.admin.task.util.file.FileAttributes.deviceMinor; +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * @author freva + */ +class FileAttributesTest { + + @Test + void parse_dev_t() { + assertEquals(0x12345BCD, deviceMajor(0x1234567890ABCDEFL)); + assertEquals(0x67890AEF, deviceMinor(0x1234567890ABCDEFL)); + } +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningThrottler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningThrottler.java index cfdc3e8dbe8..1cb7daae33f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningThrottler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningThrottler.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; +import ai.vespa.metrics.ConfigServerMetrics; import com.yahoo.jdisc.Metric; import com.yahoo.vespa.flags.IntFlag; import com.yahoo.vespa.flags.PermanentFlags; @@ -22,7 +23,7 @@ import java.util.logging.Logger; public class ProvisioningThrottler { /** Metric that indicates whether throttling is active where 1 means active and 0 means inactive */ - private static final String throttlingActiveMetric = "throttledHostProvisioning"; + private static final String throttlingActiveMetric = ConfigServerMetrics.THROTTLED_HOST_PROVISIONING.baseName(); private static final Logger LOG = Logger.getLogger(ProvisioningThrottler.class.getName()); private static final Duration WINDOW = Duration.ofDays(1); diff --git a/opennlp-linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpLinguistics.java b/opennlp-linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpLinguistics.java index 1d96d8a0cdf..1e610202ccb 100644 --- a/opennlp-linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpLinguistics.java +++ b/opennlp-linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpLinguistics.java @@ -8,7 +8,7 @@ import com.yahoo.language.process.Tokenizer; import com.yahoo.language.simple.SimpleLinguistics; /** - * Returns a linguistics implementation based on OpenNlp. + * A linguistics implementation based on OpenNlp. * * @author bratseth * @author jonmv diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java index 49fab7522ba..20269951a1e 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java @@ -84,6 +84,8 @@ public class OrchestratorImpl implements Orchestrator { Clock.systemUTC(), new ApplicationApiFactory(configServerConfig.zookeeperserver().size(), orchestratorConfig.numProxies(), + orchestratorConfig.numProxiesAllowedDown(), + orchestratorConfig.numProxiesAllowedDownRatio(), Clock.systemUTC()), orchestratorConfig.serviceMonitorConvergenceLatencySeconds()); } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiFactory.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiFactory.java index 0ad236c5a23..832b00f2294 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiFactory.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiFactory.java @@ -18,8 +18,10 @@ public class ApplicationApiFactory { private final OrchestrationParams orchestrationParams; private final Clock clock; - public ApplicationApiFactory(int numberOfConfigServers, int numProxies, Clock clock) { - this.orchestrationParams = HostedVespaOrchestration.create(numberOfConfigServers, numProxies); + public ApplicationApiFactory(int numberOfConfigServers, int numProxies, int numProxiesAllowedDown, + double numProxiesAllowedDownRatio, Clock clock) { + this.orchestrationParams = HostedVespaOrchestration.create(numberOfConfigServers, numProxies, numProxiesAllowedDown, + numProxiesAllowedDownRatio); this.clock = clock; } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApi.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApi.java index cb2a2fe5f62..7fa3bd45b4c 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApi.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApi.java @@ -30,10 +30,19 @@ public interface ClusterApi { boolean noServicesOutsideGroupIsDown() throws HostStateChangeDeniedException; - int percentageOfServicesDownOutsideGroup(); - int percentageOfServicesDownIfGroupIsAllowedToBeDown(); + /** Returns the number of services currently in the cluster, plus the number of missing services. */ + int size(); + + int servicesDownOutsideGroup(); + default int percentageOfServicesDownOutsideGroup() { return sizePercentageOf(servicesDownOutsideGroup()); } + int servicesDownIfGroupIsAllowedToBeDown(); + default int percentageOfServicesDownIfGroupIsAllowedToBeDown() { return sizePercentageOf(servicesDownIfGroupIsAllowedToBeDown()); } + + ClusterPolicyOverride clusterPolicyOverride(); Optional<StorageNode> storageNodeInGroup(); String downDescription(); + + private int sizePercentageOf(int count) { return (int) Math.round(100.0 * count / size()); } } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApiImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApiImpl.java index 024a3bc58db..6240761dd6b 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApiImpl.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApiImpl.java @@ -41,6 +41,7 @@ class ClusterApiImpl implements ClusterApi { private final Clock clock; private final Set<ServiceInstance> servicesInGroup; private final Set<ServiceInstance> servicesNotInGroup; + private final ClusterPolicyOverride clusterPolicyOverride; /** Lazily initialized in servicesDownAndNotInGroup(), do not access directly. */ private Set<ServiceInstance> servicesDownAndNotInGroup = null; @@ -71,6 +72,10 @@ class ClusterApiImpl implements ClusterApi { this.hostInfos = hostInfos; this.clusterControllerClientFactory = clusterControllerClientFactory; this.clock = clock; + this.clusterPolicyOverride = new ClusterPolicyOverride(serviceCluster.serviceInstances().size(), + clusterParams.size(), + clusterParams.allowedDown(), + clusterParams.allowedDownRatio()); Map<Boolean, Set<ServiceInstance>> serviceInstancesByLocality = serviceCluster.serviceInstances().stream() @@ -168,15 +173,21 @@ class ClusterApiImpl implements ClusterApi { } @Override - public int percentageOfServicesDownOutsideGroup() { - int numberOfServicesDown = servicesDownAndNotInGroup().size() + missingServices; - return numberOfServicesDown * 100 / (serviceCluster.serviceInstances().size() + missingServices); + public int size() { return serviceCluster.serviceInstances().size() + missingServices; } + + @Override + public int servicesDownOutsideGroup() { + return servicesDownAndNotInGroup().size() + missingServices; + } + + @Override + public int servicesDownIfGroupIsAllowedToBeDown() { + return servicesDownAndNotInGroup().size() + servicesInGroup.size() + missingServices; } @Override - public int percentageOfServicesDownIfGroupIsAllowedToBeDown() { - int numberOfServicesDown = servicesDownAndNotInGroup().size() + missingServices + servicesInGroup.size(); - return numberOfServicesDown * 100 / (serviceCluster.serviceInstances().size() + missingServices); + public ClusterPolicyOverride clusterPolicyOverride() { + return clusterPolicyOverride; } /** @@ -201,7 +212,7 @@ class ClusterApiImpl implements ClusterApi { if (suspended.size() > nodeLimit) { description.append(" and " + (suspended.size() - nodeLimit) + " others"); } - description.append(" are suspended."); + description.append(" " + isOrAre(suspended.size()) + " suspended."); } Set<ServiceInstance> downElsewhere = servicesDownAndNotInGroup().stream() @@ -223,12 +234,14 @@ class ClusterApiImpl implements ClusterApi { if (downElsewhereTotal > serviceLimit) { description.append(" and " + (downElsewhereTotal - serviceLimit) + " others"); } - description.append(" are down."); + description.append(" " + isOrAre(downElsewhereTotal) + " down."); } return description.toString(); } + private static String isOrAre(int count) { return count == 1 ? "is" : "are"; } + private Optional<StorageNode> storageNodeInGroup(Predicate<ServiceInstance> storageServicePredicate) { if (!VespaModelUtil.isStorage(serviceCluster)) { return Optional.empty(); diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterPolicyOverride.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterPolicyOverride.java new file mode 100644 index 00000000000..f724a4da9cb --- /dev/null +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterPolicyOverride.java @@ -0,0 +1,50 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.orchestrator.model; + +import com.yahoo.vespa.orchestrator.policy.SuspensionLimit; + +import java.util.Optional; +import java.util.OptionalDouble; +import java.util.OptionalInt; + +/** + * @author hakonhall + */ +public record ClusterPolicyOverride(int deployedSize, OptionalInt expectedSize, OptionalInt allowedDown, OptionalDouble allowedDownRatio) { + public ClusterPolicyOverride { + if (deployedSize <= 0) + throw new IllegalArgumentException("deployedSize must be positive"); + + if (expectedSize.isPresent() && expectedSize.getAsInt() <= 0) + throw new IllegalArgumentException("expectedSize must be positive"); + + if (allowedDown.isPresent()) { + if (allowedDown.getAsInt() <= 0) + throw new IllegalArgumentException("allowedDown must be positive: " + allowedDown.getAsInt()); + if (expectedSize.isPresent() && allowedDown.getAsInt() > expectedSize.getAsInt()) + throw new IllegalArgumentException("allowedDown must be less than or equal to expectedSize (" + expectedSize.getAsInt() + + "): " + allowedDown.getAsInt()); + } + + if (allowedDownRatio.isPresent() && (allowedDownRatio.getAsDouble() < 0.0 || allowedDownRatio.getAsDouble() > 1.0)) + throw new IllegalArgumentException("allowedDownRatio must be between 0.0 and 1.0: " + allowedDownRatio.getAsDouble()); + + } + + public static ClusterPolicyOverride fromDeployedSize(int deployedSize) { + return new ClusterPolicyOverride(deployedSize, OptionalInt.empty(), OptionalInt.empty(), OptionalDouble.empty()); + } + + public Optional<SuspensionLimit> getSuspensionLimit() { + return allowedDown.isPresent() || allowedDownRatio.isPresent() ? + Optional.of(new SuspensionLimit(allowedDown.orElse(0), allowedDownRatio.orElse(0.0))) : + Optional.empty(); + } + + public OptionalInt allowedDownPercentage() { + return allowedDownRatio.isPresent() ? + OptionalInt.of((int) Math.round(allowedDownRatio.getAsDouble() * 100.0)) : + OptionalInt.empty(); + } + +} diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/ClusterParams.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/ClusterParams.java index ab010da50ad..4048a27fda0 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/ClusterParams.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/ClusterParams.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.orchestrator.policy; import java.util.Objects; +import java.util.OptionalDouble; import java.util.OptionalInt; /** @@ -14,9 +15,13 @@ public class ClusterParams { private static final ClusterParams DEFAULT = new ClusterParams.Builder().build(); private final int size; + private final int allowedDown; + private final double allowedDownRatio; public static class Builder { - private int size = 0; + private int size = -1; + private int allowedDown = -1; + private double allowedDownRatio = -1.0; public Builder() {} @@ -25,8 +30,18 @@ public class ClusterParams { return this; } + public Builder setAllowedDown(int allowedDown) { + this.allowedDown = allowedDown; + return this; + } + + public Builder setAllowedDownRatio(double allowedDownRatio) { + this.allowedDownRatio = allowedDownRatio; + return this; + } + public ClusterParams build() { - return new ClusterParams(size); + return new ClusterParams(size, allowedDown, allowedDownRatio); } } @@ -34,8 +49,10 @@ public class ClusterParams { return DEFAULT; } - private ClusterParams(int size) { + private ClusterParams(int size, int allowedDown, double allowedDownRatio) { this.size = size; + this.allowedDown = allowedDown; + this.allowedDownRatio = allowedDownRatio; } /** @@ -46,16 +63,28 @@ public class ClusterParams { return size > 0 ? OptionalInt.of(size) : OptionalInt.empty(); } + /** The number of services that are allowed to be down. */ + public OptionalInt allowedDown() { + return allowedDown > 0 ? OptionalInt.of(allowedDown) : OptionalInt.empty(); + } + + /** The ratio of services that are allowed to be down. */ + public OptionalDouble allowedDownRatio() { + return 0.0 <= allowedDownRatio && allowedDownRatio <= 1.0 ? + OptionalDouble.of(allowedDownRatio) : + OptionalDouble.empty(); + } + @Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; ClusterParams that = (ClusterParams) o; - return size == that.size; + return size == that.size && allowedDown == that.allowedDown && Double.compare(that.allowedDownRatio, allowedDownRatio) == 0; } @Override public int hashCode() { - return Objects.hash(size); + return Objects.hash(size, allowedDown, allowedDownRatio); } } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicy.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicy.java index 5d553c86c50..88b339e15f3 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicy.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicy.java @@ -37,10 +37,11 @@ public class HostedVespaClusterPolicy implements ClusterPolicy { return SuspensionReasons.nothingNoteworthy(); } - int percentageOfServicesAllowedToBeDown = getConcurrentSuspensionLimit(clusterApi).asPercentage(); - if (clusterApi.percentageOfServicesDownIfGroupIsAllowedToBeDown() <= percentageOfServicesAllowedToBeDown) { + SuspensionLimit limit = getConcurrentSuspensionLimit(clusterApi); + if (clusterApi.servicesDownIfGroupIsAllowedToBeDown() <= limit.allowedDown()) + return SuspensionReasons.nothingNoteworthy(); + if (clusterApi.percentageOfServicesDownIfGroupIsAllowedToBeDown() <= limit.allowedDownPercentage()) return SuspensionReasons.nothingNoteworthy(); - } // Be a bit more cautious when removing nodes permanently if (!permanent) { @@ -50,19 +51,39 @@ public class HostedVespaClusterPolicy implements ClusterPolicy { } } - String message = percentageOfServicesAllowedToBeDown <= 0 - ? clusterApi.percentageOfServicesDownOutsideGroup() + "% of the " + clusterApi.serviceDescription(true) - + " are down or suspended already:" + clusterApi.downDescription() - : "The percentage of downed or suspended " + clusterApi.serviceDescription(true) - + " would increase from " + clusterApi.percentageOfServicesDownOutsideGroup() + "% to " - + clusterApi.percentageOfServicesDownIfGroupIsAllowedToBeDown() + "% (limit is " - + percentageOfServicesAllowedToBeDown + "%):" + clusterApi.downDescription(); + final String message; + if (limit.allowedDownPercentage() > 0) { + final String numberDescription; + final String fromDescription; + final String toDescription; + final String limitDescription; + if (limit.allowedDown() > 1) { + numberDescription = "number (percentage)"; + fromDescription = clusterApi.servicesDownOutsideGroup() + " (" + clusterApi.percentageOfServicesDownOutsideGroup() + "%)"; + toDescription = clusterApi.servicesDownIfGroupIsAllowedToBeDown() + " (" + clusterApi.percentageOfServicesDownIfGroupIsAllowedToBeDown() + "%)"; + limitDescription = limit.allowedDown() + " (" + limit.allowedDownPercentage() + "%)"; + } else { + numberDescription = "percentage"; + fromDescription = clusterApi.percentageOfServicesDownOutsideGroup() + "%"; + toDescription = clusterApi.percentageOfServicesDownIfGroupIsAllowedToBeDown() + "%"; + limitDescription = limit.allowedDownPercentage() + "%"; + } - throw new HostStateChangeDeniedException(clusterApi.getNodeGroup(), ENOUGH_SERVICES_UP_CONSTRAINT, message); + message = "The %s of %s that are down would increase from %s to %s which is beyond the limit of %s" + .formatted(numberDescription, clusterApi.serviceDescription(true), fromDescription, toDescription, limitDescription); + } else { + message = "%d %s %s already down".formatted(clusterApi.servicesDownOutsideGroup(), + clusterApi.serviceDescription(false), + clusterApi.servicesDownOutsideGroup() == 1 ? "is" : "are"); + } + + throw new HostStateChangeDeniedException(clusterApi.getNodeGroup(), + ENOUGH_SERVICES_UP_CONSTRAINT, + message + ":" + clusterApi.downDescription()); } // Non-private for testing purposes - ConcurrentSuspensionLimitForCluster getConcurrentSuspensionLimit(ClusterApi clusterApi) { + SuspensionLimit getConcurrentSuspensionLimit(ClusterApi clusterApi) { // Possible service clusters on a node as of 2021-01-22: // // CLUSTER ID SERVICE TYPE HEALTH ASSOCIATION @@ -102,45 +123,50 @@ public class HostedVespaClusterPolicy implements ClusterPolicy { // H proxy (same as B) // I proxy host + Optional<SuspensionLimit> override = clusterApi.clusterPolicyOverride().getSuspensionLimit(); + if (override.isPresent()) { + return override.get(); + } + if (clusterApi.serviceType().equals(ServiceType.CLUSTER_CONTROLLER)) { - return ConcurrentSuspensionLimitForCluster.ONE_NODE; + return SuspensionLimit.fromAllowedDown(1); } if (Set.of(ServiceType.STORAGE, ServiceType.SEARCH, ServiceType.DISTRIBUTOR, ServiceType.TRANSACTION_LOG_SERVER) .contains(clusterApi.serviceType())) { // Delegate to the cluster controller - return ConcurrentSuspensionLimitForCluster.ALL_NODES; + return SuspensionLimit.fromAllowedDownRatio(1); } if (clusterApi.serviceType().equals(ServiceType.CONTAINER)) { - return ConcurrentSuspensionLimitForCluster.TEN_PERCENT; + return SuspensionLimit.fromAllowedDownRatio(0.1); } if (VespaModelUtil.ADMIN_CLUSTER_ID.equals(clusterApi.clusterId())) { if (ServiceType.SLOBROK.equals(clusterApi.serviceType())) { - return ConcurrentSuspensionLimitForCluster.ONE_NODE; + return SuspensionLimit.fromAllowedDown(1); } - return ConcurrentSuspensionLimitForCluster.ALL_NODES; + return SuspensionLimit.fromAllowedDownRatio(1); } else if (ServiceType.METRICS_PROXY.equals(clusterApi.serviceType())) { - return ConcurrentSuspensionLimitForCluster.ALL_NODES; + return SuspensionLimit.fromAllowedDownRatio(1); } if (Set.of(ServiceType.CONFIG_SERVER, ServiceType.CONTROLLER).contains(clusterApi.serviceType())) { - return ConcurrentSuspensionLimitForCluster.ONE_NODE; + return SuspensionLimit.fromAllowedDown(1); } if (clusterApi.serviceType().equals(ServiceType.HOST_ADMIN)) { if (Set.of(ClusterId.CONFIG_SERVER_HOST, ClusterId.CONTROLLER_HOST).contains(clusterApi.clusterId())) { - return ConcurrentSuspensionLimitForCluster.ONE_NODE; + return SuspensionLimit.fromAllowedDown(1); } return zone.system().isCd() - ? ConcurrentSuspensionLimitForCluster.FIFTY_PERCENT - : ConcurrentSuspensionLimitForCluster.TWENTY_PERCENT; + ? SuspensionLimit.fromAllowedDownRatio(0.5) + : SuspensionLimit.fromAllowedDownRatio(0.2); } // The above should cover all cases, but if not we'll return a reasonable default: - return ConcurrentSuspensionLimitForCluster.TEN_PERCENT; + return SuspensionLimit.fromAllowedDownRatio(0.1); } } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaOrchestration.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaOrchestration.java index eb2b863aefb..54d440f751f 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaOrchestration.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaOrchestration.java @@ -11,7 +11,8 @@ import com.yahoo.vespa.applicationmodel.ServiceType; * @author hakonhall */ public class HostedVespaOrchestration { - public static OrchestrationParams create(int numConfigServers, int numProxies) { + public static OrchestrationParams create(int numConfigServers, int numProxies, int numProxiesAllowedDown, + double numProxiesAllowedDownRatio) { // We'll create parameters for both the controller and config server applications, even though // only one of them is present, as (a) no harm is done by having the extra parameters, and // (b) it leads to simpler code below. @@ -75,6 +76,8 @@ public class HostedVespaOrchestration { new ClusterParams .Builder() .setSize(numProxies) + .setAllowedDown(numProxiesAllowedDown) + .setAllowedDownRatio(numProxiesAllowedDownRatio) .build()) .build()) @@ -87,6 +90,8 @@ public class HostedVespaOrchestration { new ClusterParams .Builder() .setSize(numProxies) + .setAllowedDown(numProxiesAllowedDown) + .setAllowedDownRatio(numProxiesAllowedDownRatio) .build()) .build()) diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/SuspensionLimit.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/SuspensionLimit.java new file mode 100644 index 00000000000..8a3d62dcc9c --- /dev/null +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/SuspensionLimit.java @@ -0,0 +1,29 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.orchestrator.policy; + +/** + * @author hakonhall + * + * @param allowedDown the maximum number of services (nodes) that are allowed to be down. + * @param allowedDownRatio the maximum ratio of services (nodes) that are allowed to be down. + */ +public record SuspensionLimit(int allowedDown, double allowedDownRatio) { + public SuspensionLimit { + if (allowedDown < 0) + throw new IllegalArgumentException("allowedDown cannot be negative: " + allowedDown); + if (allowedDownRatio < 0.0 || allowedDownRatio > 1.0) + throw new IllegalArgumentException("allowedDownRatio must be between 0.0 and 1.0: " + allowedDownRatio); + } + + public static SuspensionLimit fromAllowedDown(int allowedDown) { + return new SuspensionLimit(allowedDown, 0); + } + + public static SuspensionLimit fromAllowedDownRatio(double allowedDownRatio) { + return new SuspensionLimit(0, allowedDownRatio); + } + + public int allowedDownPercentage() { + return (int) Math.round(allowedDownRatio * 100.0); + } +} diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java index 70a8381c9ac..3eb6bf6bc5e 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java @@ -77,7 +77,7 @@ public class OrchestratorImplTest { private static final Zone zone = Zone.defaultZone(); private final ManualClock clock = new ManualClock(); - private final ApplicationApiFactory applicationApiFactory = new ApplicationApiFactory(3, 5, clock); + private final ApplicationApiFactory applicationApiFactory = new ApplicationApiFactory(3, 5, 1, 0.1, clock); private final InMemoryFlagSource flagSource = new InMemoryFlagSource(); private final MockCurator curator = new MockCurator(); private final ZkStatusService statusService = new ZkStatusService( diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorTest.java index 73774321ffb..ee62ffabd30 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorTest.java @@ -57,7 +57,7 @@ public class OrchestratorTest { var flagSource = new InMemoryFlagSource(); var timer = new TestTimer(); var clustercontroller = new ClusterControllerClientFactoryMock(); - var applicationApiFactory = new ApplicationApiFactory(3, 5, timer.toUtcClock()); + var applicationApiFactory = new ApplicationApiFactory(3, 5, 1, 0.1, timer.toUtcClock()); var clusterPolicy = new HostedVespaClusterPolicy(flagSource, zone); var policy = new HostedVespaPolicy(clusterPolicy, clustercontroller, applicationApiFactory, flagSource); var zone = new Zone(SystemName.cd, Environment.prod, RegionName.from("cd-us-east-1")); @@ -103,7 +103,7 @@ public class OrchestratorTest { fail(); } catch (HostStateChangeDeniedException e) { assertTrue(e.getMessage().contains("Changing the state of cfg2 would violate enough-services-up")); - assertTrue(e.getMessage().contains("[cfg1] are suspended.")); + assertTrue(e.getMessage().contains("[cfg1] is suspended.")); } // cfg1 is removed from the application @@ -115,7 +115,7 @@ public class OrchestratorTest { fail(); } catch (HostStateChangeDeniedException e) { assertTrue(e.getMessage().contains("Changing the state of cfg2 would violate enough-services-up")); - assertTrue(e.getMessage().contains("[1 missing config server] are down.")); + assertTrue(e.getMessage().contains("[1 missing config server] is down.")); } // cfg1 is reprovisioned, added to the node repo, and activated @@ -130,7 +130,7 @@ public class OrchestratorTest { fail(); } catch (HostStateChangeDeniedException e) { assertTrue(e.getMessage().contains("Changing the state of cfg1 would violate enough-services-up")); - assertTrue(e.getMessage().contains("[cfg2] are suspended")); + assertTrue(e.getMessage().contains("[cfg2] is suspended")); } // etc (should be the same as for cfg1) diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java index b073f546cce..49978f824c4 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java @@ -33,7 +33,6 @@ import java.util.EnumSet; import java.util.List; import java.util.Optional; import java.util.Set; -import java.util.stream.Collectors; import java.util.stream.IntStream; import static org.junit.Assert.assertEquals; @@ -95,7 +94,7 @@ public class ClusterApiImplTest { assertFalse(clusterApi.isStorageCluster()); assertEquals(" [host3, host4] are suspended. [ServiceInstance{configId=service-2, hostName=host2, " + "serviceStatus=ServiceStatusInfo{status=DOWN, since=Optional.empty, lastChecked=Optional.empty}}] " + - "are down.", + "is down.", clusterApi.downDescription()); assertEquals(60, clusterApi.percentageOfServicesDownOutsideGroup()); assertEquals(80, clusterApi.percentageOfServicesDownIfGroupIsAllowedToBeDown()); @@ -178,8 +177,9 @@ public class ClusterApiImplTest { policy.verifyGroupGoingDownIsFine(clusterApi); fail(); } catch (HostStateChangeDeniedException e) { - assertTrue(e.getMessage().contains("Changing the state of cfg1 would violate enough-services-up: 33% of the config " + - "servers are down or suspended already: [1 missing config server] are down.")); + assertEquals("Changing the state of cfg1 would violate enough-services-up: 1 config server is already down: " + + "[1 missing config server] is down.", + e.getMessage()); } } @@ -197,8 +197,9 @@ public class ClusterApiImplTest { policy.verifyGroupGoingDownIsFine(clusterApi); fail(); } catch (HostStateChangeDeniedException e) { - assertTrue(e.getMessage().contains("Changing the state of cfg1 would violate enough-services-up: 33% of the config " + - "server hosts are down or suspended already: [1 missing config server host] are down.")); + assertEquals("Changing the state of cfg1 would violate enough-services-up: 1 config server host is already down: " + + "[1 missing config server host] is down.", + e.getMessage()); } } @@ -212,8 +213,9 @@ public class ClusterApiImplTest { policy.verifyGroupGoingDownIsFine(clusterApi); fail(); } catch (HostStateChangeDeniedException e) { - assertTrue(e.getMessage().contains("Changing the state of cfg1 would violate enough-services-up: 33% of the config " + - "servers are down or suspended already: [1 missing config server] are down.")); + assertEquals("Changing the state of cfg1 would violate enough-services-up: 1 config server is already down: " + + "[1 missing config server] is down.", + e.getMessage()); } } diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ModelTestUtils.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ModelTestUtils.java index f2e2972ae9f..32aeebb2c74 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ModelTestUtils.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ModelTestUtils.java @@ -64,8 +64,11 @@ class ModelTestUtils { public static final int NUMBER_OF_CONFIG_SERVERS = 3; public static final int NUMBER_OF_PROXIES = 5; + public static final int NUMBER_OF_PROXIES_ALLOWED_DOWN = 1; + public static final double NUMBER_OF_PROXIES_ALLOWED_DOWN_RATIO = 0.1; public static final OrchestrationParams ORCHESTRATION_PARAMS = - HostedVespaOrchestration.create(NUMBER_OF_CONFIG_SERVERS, NUMBER_OF_PROXIES); + HostedVespaOrchestration.create(NUMBER_OF_CONFIG_SERVERS, NUMBER_OF_PROXIES, NUMBER_OF_PROXIES_ALLOWED_DOWN, + NUMBER_OF_PROXIES_ALLOWED_DOWN_RATIO); public static final ApplicationParams APPLICATION_PARAMS = ORCHESTRATION_PARAMS .getApplicationParams(OrchestratorUtil.toApplicationId( new ApplicationInstanceReference(TENANT_ID, APPLICATION_INSTANCE_ID))); @@ -93,7 +96,8 @@ class ModelTestUtils { private final ManualClock clock = new ManualClock(); ApplicationApiFactory applicationApiFactory() { - return new ApplicationApiFactory(NUMBER_OF_CONFIG_SERVERS, NUMBER_OF_PROXIES, clock); + return new ApplicationApiFactory(NUMBER_OF_CONFIG_SERVERS, NUMBER_OF_PROXIES, NUMBER_OF_PROXIES_ALLOWED_DOWN, + NUMBER_OF_PROXIES_ALLOWED_DOWN_RATIO, clock); } HostInfos getHostInfos() { diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicyTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicyTest.java index eb70d809855..47bdcd4e68e 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicyTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicyTest.java @@ -10,6 +10,7 @@ import com.yahoo.vespa.applicationmodel.ServiceType; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.orchestrator.model.ApplicationApi; import com.yahoo.vespa.orchestrator.model.ClusterApi; +import com.yahoo.vespa.orchestrator.model.ClusterPolicyOverride; import com.yahoo.vespa.orchestrator.model.NodeGroup; import com.yahoo.vespa.orchestrator.model.VespaModelUtil; import org.junit.Before; @@ -36,6 +37,7 @@ public class HostedVespaClusterPolicyTest { @Before public void setUp() { when(clusterApi.getApplication()).thenReturn(applicationApi); + when(clusterApi.clusterPolicyOverride()).thenReturn(ClusterPolicyOverride.fromDeployedSize(3)); when(zone.system()).thenReturn(SystemName.main); NodeGroup nodeGroup = mock(NodeGroup.class); @@ -62,24 +64,24 @@ public class HostedVespaClusterPolicyTest { public void testSlobrokSuspensionLimit() { when(clusterApi.clusterId()).thenReturn(VespaModelUtil.ADMIN_CLUSTER_ID); when(clusterApi.serviceType()).thenReturn(ServiceType.SLOBROK); - assertEquals(ConcurrentSuspensionLimitForCluster.ONE_NODE, - policy.getConcurrentSuspensionLimit(clusterApi)); + assertEquals(SuspensionLimit.fromAllowedDown(1), + policy.getConcurrentSuspensionLimit(clusterApi)); } @Test public void testAdminSuspensionLimit() { when(clusterApi.clusterId()).thenReturn(VespaModelUtil.ADMIN_CLUSTER_ID); when(clusterApi.serviceType()).thenReturn(new ServiceType("non-slobrok-service-type")); - assertEquals(ConcurrentSuspensionLimitForCluster.ALL_NODES, - policy.getConcurrentSuspensionLimit(clusterApi)); + assertEquals(SuspensionLimit.fromAllowedDownRatio(1.0), + policy.getConcurrentSuspensionLimit(clusterApi)); } @Test public void testStorageSuspensionLimit() { when(clusterApi.serviceType()).thenReturn(ServiceType.STORAGE); when(clusterApi.clusterId()).thenReturn(new ClusterId("some-cluster-id")); - assertEquals(ConcurrentSuspensionLimitForCluster.ALL_NODES, - policy.getConcurrentSuspensionLimit(clusterApi)); + assertEquals(SuspensionLimit.fromAllowedDownRatio(1.0), + policy.getConcurrentSuspensionLimit(clusterApi)); } @Test @@ -87,12 +89,12 @@ public class HostedVespaClusterPolicyTest { when(applicationApi.applicationId()).thenReturn(VespaModelUtil.TENANT_HOST_APPLICATION_ID); when(clusterApi.clusterId()).thenReturn(ClusterId.TENANT_HOST); when(clusterApi.serviceType()).thenReturn(ServiceType.HOST_ADMIN); - assertEquals(ConcurrentSuspensionLimitForCluster.TWENTY_PERCENT, + assertEquals(SuspensionLimit.fromAllowedDownRatio(0.2), policy.getConcurrentSuspensionLimit(clusterApi)); when(zone.system()).thenReturn(SystemName.cd); - assertEquals(ConcurrentSuspensionLimitForCluster.FIFTY_PERCENT, + assertEquals(SuspensionLimit.fromAllowedDownRatio(0.5), policy.getConcurrentSuspensionLimit(clusterApi)); } @@ -101,7 +103,7 @@ public class HostedVespaClusterPolicyTest { when(applicationApi.applicationId()).thenReturn(ApplicationId.fromSerializedForm("a:b:c")); when(clusterApi.clusterId()).thenReturn(new ClusterId("some-cluster-id")); when(clusterApi.serviceType()).thenReturn(new ServiceType("some-service-type")); - assertEquals(ConcurrentSuspensionLimitForCluster.TEN_PERCENT, + assertEquals(SuspensionLimit.fromAllowedDownRatio(0.1), policy.getConcurrentSuspensionLimit(clusterApi)); } @@ -132,12 +134,14 @@ public class HostedVespaClusterPolicyTest { boolean expectSuccess) throws HostStateChangeDeniedException { when(clusterApi.noServicesOutsideGroupIsDown()).thenReturn(noServicesOutsideGroupIsDown); when(clusterApi.allServicesDown()).thenReturn(noServicesInGroupIsUp); + when(clusterApi.servicesDownIfGroupIsAllowedToBeDown()).thenReturn(20); when(clusterApi.percentageOfServicesDownIfGroupIsAllowedToBeDown()).thenReturn(20); - doReturn(ConcurrentSuspensionLimitForCluster.TEN_PERCENT).when(policy).getConcurrentSuspensionLimit(clusterApi); + doReturn(SuspensionLimit.fromAllowedDownRatio(0.1)).when(policy).getConcurrentSuspensionLimit(clusterApi); when(applicationApi.applicationId()).thenReturn(ApplicationId.fromSerializedForm("a:b:c")); when(clusterApi.serviceType()).thenReturn(new ServiceType("service-type")); when(clusterApi.serviceDescription(true)).thenReturn("services of {service-type,cluster-id}"); + when(clusterApi.servicesDownOutsideGroup()).thenReturn(5); when(clusterApi.percentageOfServicesDownOutsideGroup()).thenReturn(5); when(clusterApi.percentageOfServicesDownIfGroupIsAllowedToBeDown()).thenReturn(percentageOfServicesDownIfGroupIsAllowedToBeDown); when(clusterApi.downDescription()).thenReturn(" Down description"); @@ -153,9 +157,9 @@ public class HostedVespaClusterPolicyTest { } } catch (HostStateChangeDeniedException e) { if (!expectSuccess) { - assertEquals("Changing the state of node-group would violate enough-services-up: The percentage of downed " + - "or suspended services of {service-type,cluster-id} would increase from 5% to 13% (limit is 10%): " + - "Down description", + assertEquals("Changing the state of node-group would violate enough-services-up: The percentage of " + + "services of {service-type,cluster-id} that are down would increase from 5% to 13% " + + "which is beyond the limit of 10%: Down description", e.getMessage()); assertEquals("enough-services-up", e.getConstraintName()); } diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicyTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicyTest.java index a622142b873..97adeea9835 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicyTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicyTest.java @@ -37,7 +37,7 @@ public class HostedVespaPolicyTest { private final ClusterControllerClientFactory clientFactory = mock(ClusterControllerClientFactory.class); private final ClusterControllerClient client = mock(ClusterControllerClient.class); private final ManualClock clock = new ManualClock(); - private final ApplicationApiFactory applicationApiFactory = new ApplicationApiFactory(3, 5, clock); + private final ApplicationApiFactory applicationApiFactory = new ApplicationApiFactory(3, 5, 1, 0.1, clock); private final InMemoryFlagSource flagSource = new InMemoryFlagSource(); @Before diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostRequestHandlerTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostRequestHandlerTest.java index b9dab4b3aeb..465804a9406 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostRequestHandlerTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostRequestHandlerTest.java @@ -79,7 +79,7 @@ class HostRequestHandlerTest { private static final ServiceMonitor serviceMonitor = mock(ServiceMonitor.class); private static final StatusService EVERY_HOST_IS_UP_HOST_STATUS_SERVICE = new ZkStatusService( new MockCurator(), mock(Metric.class), new TestTimer(), new DummyAntiServiceMonitor()); - private static final ApplicationApiFactory applicationApiFactory = new ApplicationApiFactory(3, 5, clock); + private static final ApplicationApiFactory applicationApiFactory = new ApplicationApiFactory(3, 5, 1, 0.1, clock); static { when(serviceMonitor.getApplication(any(HostName.class))) diff --git a/parent/pom.xml b/parent/pom.xml index df20b94ec79..56c896d57cc 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -1161,6 +1161,21 @@ <artifactId>checker-qual</artifactId> <version>3.30.0</version> </dependency> + <dependency> + <groupId>com.google.http-client</groupId> + <artifactId>google-http-client-apache-v2</artifactId> + <version>1.43.3</version> + </dependency> + <dependency> + <groupId>com.google.http-client</groupId> + <artifactId>google-http-client</artifactId> + <version>1.43.3</version> + </dependency> + <dependency> + <groupId>com.google.auth</groupId> + <artifactId>google-auth-library-oauth2-http</artifactId> + <version>1.19.0</version> + </dependency> </dependencies> </dependencyManagement> diff --git a/persistence/src/vespa/persistence/spi/clusterstate.cpp b/persistence/src/vespa/persistence/spi/clusterstate.cpp index ad5039fade1..e6708192d47 100644 --- a/persistence/src/vespa/persistence/spi/clusterstate.cpp +++ b/persistence/src/vespa/persistence/spi/clusterstate.cpp @@ -97,7 +97,7 @@ void ClusterState::serialize(vespalib::nbostream& o) const { assert(_distribution); assert(_state); vespalib::asciistream tmp; - _state->serialize(tmp, false); + _state->serialize(tmp); o << tmp.str() << _nodeIndex; o << _distribution->serialize(); } diff --git a/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp b/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp index aedfde2521c..4a4a021d6d5 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp @@ -25,6 +25,7 @@ LOG_SETUP(".proton.matching.matcher"); using search::fef::Properties; using namespace search::fef::indexproperties::matching; +using namespace search::fef::indexproperties; using namespace search::engine; using namespace search::grouping; using search::DocumentMetaData; @@ -122,7 +123,7 @@ Matcher::Matcher(const search::index::Schema &schema, Properties props, const ve _rankSetup(), _viewResolver(ViewResolver::createFromSchema(schema)), _statsLock(), - _stats(), + _stats(softtimeout::Factor::lookup(_indexEnv.getProperties())), _startTime(my_clock::now()), _clock(clock), _queryLimiter(queryLimiter), @@ -149,9 +150,6 @@ Matcher::getStats() return stats; } -using search::fef::indexproperties::softtimeout::Enabled; -using search::fef::indexproperties::softtimeout::Factor; - std::unique_ptr<MatchToolsFactory> Matcher::create_match_tools_factory(const search::engine::Request &request, ISearchContext &searchContext, IAttributeContext &attrContext, const search::IDocumentMetaStore &metaStore, @@ -160,11 +158,11 @@ Matcher::create_match_tools_factory(const search::engine::Request &request, ISea bool is_search) const { const Properties & rankProperties = request.propertiesMap.rankProperties(); - bool softTimeoutEnabled = Enabled::lookup(rankProperties, _rankSetup->getSoftTimeoutEnabled()); - bool hasFactorOverride = Factor::isPresent(rankProperties); + bool softTimeoutEnabled = softtimeout::Enabled::lookup(rankProperties, _rankSetup->getSoftTimeoutEnabled()); + bool hasFactorOverride = softtimeout::Factor::isPresent(rankProperties); double factor = softTimeoutEnabled ? ( hasFactorOverride - ? Factor::lookup(rankProperties, _stats.softDoomFactor()) + ? softtimeout::Factor::lookup(rankProperties, _stats.softDoomFactor()) : _stats.softDoomFactor()) : 0.95; vespalib::duration safeLeft = std::chrono::duration_cast<vespalib::duration>(request.getTimeLeft() * factor); diff --git a/searchcore/src/vespa/searchcore/proton/matching/matching_stats.cpp b/searchcore/src/vespa/searchcore/proton/matching/matching_stats.cpp index 86fb3cf8107..47c0fbc8c55 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/matching_stats.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/matching_stats.cpp @@ -19,7 +19,7 @@ constexpr double MAX_CHANGE_FACTOR = 5; } // namespace proton::matching::<unnamed> -MatchingStats::MatchingStats(double prev_soft_doom_factor) +MatchingStats::MatchingStats(double prev_soft_doom_factor) noexcept : _queries(0), _limited_queries(0), _docidSpaceCovered(0), @@ -57,7 +57,7 @@ MatchingStats::merge_partition(const Partition &partition, size_t id) } MatchingStats & -MatchingStats::add(const MatchingStats &rhs) +MatchingStats::add(const MatchingStats &rhs) noexcept { _queries += rhs._queries; _limited_queries += rhs._limited_queries; diff --git a/searchcore/src/vespa/searchcore/proton/matching/matching_stats.h b/searchcore/src/vespa/searchcore/proton/matching/matching_stats.h index 4139bfbaf66..a9f7d3258d9 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/matching_stats.h +++ b/searchcore/src/vespa/searchcore/proton/matching/matching_stats.h @@ -24,21 +24,21 @@ private: double _min; double _max; public: - Avg() : _value(0.0), _count(0), _min(0.0), _max(0.0) {} - Avg & set(double value) { + Avg() noexcept : _value(0.0), _count(0), _min(0.0), _max(0.0) {} + Avg & set(double value) noexcept { _value = value; _count = 1; _min = value; _max = value; return *this; } - double avg() const { + double avg() const noexcept { return (_count > 0) ? (_value / _count) : 0; } - size_t count() const { return _count; } - double min() const { return _min; } - double max() const { return _max; } - void add(const Avg &other) { + size_t count() const noexcept { return _count; } + double min() const noexcept { return _min; } + double max() const noexcept { return _max; } + void add(const Avg &other) noexcept { if (_count == 0) { _min = other._min; _max = other._max; @@ -78,31 +78,31 @@ public: _active_time(), _wait_time() { } - Partition &docsCovered(size_t value) { _docsCovered = value; return *this; } - size_t docsCovered() const { return _docsCovered; } - Partition &docsMatched(size_t value) { _docsMatched = value; return *this; } - size_t docsMatched() const { return _docsMatched; } - Partition &docsRanked(size_t value) { _docsRanked = value; return *this; } - size_t docsRanked() const { return _docsRanked; } - Partition &docsReRanked(size_t value) { _docsReRanked = value; return *this; } - size_t docsReRanked() const { return _docsReRanked; } - Partition &softDoomed(bool v) { _softDoomed += v ? 1 : 0; return *this; } - size_t softDoomed() const { return _softDoomed; } - Partition & doomOvertime(vespalib::duration overtime) { _doomOvertime.set(vespalib::to_s(overtime)); return *this; } - vespalib::duration doomOvertime() const { return vespalib::from_s(_doomOvertime.max()); } - - Partition &active_time(double time_s) { _active_time.set(time_s); return *this; } - double active_time_avg() const { return _active_time.avg(); } - size_t active_time_count() const { return _active_time.count(); } - double active_time_min() const { return _active_time.min(); } - double active_time_max() const { return _active_time.max(); } - Partition &wait_time(double time_s) { _wait_time.set(time_s); return *this; } - double wait_time_avg() const { return _wait_time.avg(); } - size_t wait_time_count() const { return _wait_time.count(); } - double wait_time_min() const { return _wait_time.min(); } - double wait_time_max() const { return _wait_time.max(); } - - Partition &add(const Partition &rhs) { + Partition &docsCovered(size_t value) noexcept { _docsCovered = value; return *this; } + size_t docsCovered() const noexcept { return _docsCovered; } + Partition &docsMatched(size_t value) noexcept { _docsMatched = value; return *this; } + size_t docsMatched() const noexcept { return _docsMatched; } + Partition &docsRanked(size_t value) noexcept { _docsRanked = value; return *this; } + size_t docsRanked() const noexcept { return _docsRanked; } + Partition &docsReRanked(size_t value) noexcept { _docsReRanked = value; return *this; } + size_t docsReRanked() const noexcept { return _docsReRanked; } + Partition &softDoomed(bool v) noexcept { _softDoomed += v ? 1 : 0; return *this; } + size_t softDoomed() const noexcept { return _softDoomed; } + Partition & doomOvertime(vespalib::duration overtime) noexcept { _doomOvertime.set(vespalib::to_s(overtime)); return *this; } + vespalib::duration doomOvertime() const noexcept { return vespalib::from_s(_doomOvertime.max()); } + + Partition &active_time(double time_s) noexcept { _active_time.set(time_s); return *this; } + double active_time_avg() const noexcept { return _active_time.avg(); } + size_t active_time_count() const noexcept { return _active_time.count(); } + double active_time_min() const noexcept { return _active_time.min(); } + double active_time_max() const noexcept { return _active_time.max(); } + Partition &wait_time(double time_s) noexcept { _wait_time.set(time_s); return *this; } + double wait_time_avg() const noexcept { return _wait_time.avg(); } + size_t wait_time_count() const noexcept { return _wait_time.count(); } + double wait_time_min() const noexcept { return _wait_time.min(); } + double wait_time_max() const noexcept { return _wait_time.max(); } + + Partition &add(const Partition &rhs) noexcept { _docsCovered += rhs.docsCovered(); _docsMatched += rhs._docsMatched; _docsRanked += rhs._docsRanked; @@ -138,9 +138,10 @@ public: static constexpr double INITIAL_SOFT_DOOM_FACTOR = 0.5; MatchingStats(const MatchingStats &) = delete; MatchingStats & operator = (const MatchingStats &) = delete; - MatchingStats(MatchingStats &&) = default; - MatchingStats & operator = (MatchingStats &&) = default; - MatchingStats(double prev_soft_doom_factor = INITIAL_SOFT_DOOM_FACTOR); + MatchingStats(MatchingStats &&) noexcept = default; + MatchingStats & operator = (MatchingStats &&) noexcept = default; + MatchingStats() noexcept : MatchingStats(INITIAL_SOFT_DOOM_FACTOR) {} + MatchingStats(double prev_soft_doom_factor) noexcept; ~MatchingStats(); MatchingStats &queries(size_t value) { _queries = value; return *this; } @@ -206,7 +207,7 @@ public: const Partition &getPartition(size_t index) const { return _partitions[index]; } // used to aggregate accross searches (and configurations) - MatchingStats &add(const MatchingStats &rhs); + MatchingStats &add(const MatchingStats &rhs) noexcept; }; } diff --git a/searchlib/src/vespa/searchlib/expression/resultnodes.cpp b/searchlib/src/vespa/searchlib/expression/resultnodes.cpp index 7fb3ab1b6cf..8f9f1b7ca06 100644 --- a/searchlib/src/vespa/searchlib/expression/resultnodes.cpp +++ b/searchlib/src/vespa/searchlib/expression/resultnodes.cpp @@ -438,7 +438,9 @@ void RawResultNode::setBuffer(const void *buf, size_t sz) { _value.resize(sz + 1); - memcpy(_value.data(), buf, sz); + if (sz > 0) { + memcpy(_value.data(), buf, sz); + } _value.back() = 0; _value.resize(sz); } diff --git a/searchlib/src/vespa/searchlib/features/tensor_factory_blueprint.cpp b/searchlib/src/vespa/searchlib/features/tensor_factory_blueprint.cpp index 3e5d1da6a1a..7c267413a86 100644 --- a/searchlib/src/vespa/searchlib/features/tensor_factory_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/features/tensor_factory_blueprint.cpp @@ -36,7 +36,8 @@ TensorFactoryBlueprint::TensorFactoryBlueprint(const vespalib::string &baseName) : Blueprint(baseName), _sourceType(), _sourceParam(), - _dimension("0") // default dimension is set to the source param if not specified. + _dimension("0"), // default dimension is set to the source param if not specified. + _valueType(vespalib::eval::ValueType::error_type()) { } diff --git a/searchlib/src/vespa/searchlib/features/tensor_factory_blueprint.h b/searchlib/src/vespa/searchlib/features/tensor_factory_blueprint.h index 26fcc79b6f5..47ccb038ac7 100644 --- a/searchlib/src/vespa/searchlib/features/tensor_factory_blueprint.h +++ b/searchlib/src/vespa/searchlib/features/tensor_factory_blueprint.h @@ -4,6 +4,7 @@ #include <vespa/searchlib/fef/blueprint.h> #include <vespa/vespalib/stllike/string.h> +#include <vespa/eval/eval/value_type.h> namespace search::features { @@ -19,6 +20,7 @@ protected: vespalib::string _sourceType; vespalib::string _sourceParam; vespalib::string _dimension; + vespalib::eval::ValueType _valueType; bool extractSource(const vespalib::string &source); TensorFactoryBlueprint(const vespalib::string &baseName); diff --git a/searchlib/src/vespa/searchlib/features/tensor_from_attribute_executor.h b/searchlib/src/vespa/searchlib/features/tensor_from_attribute_executor.h index 5a3fede76e8..7b04d10cea2 100644 --- a/searchlib/src/vespa/searchlib/features/tensor_from_attribute_executor.h +++ b/searchlib/src/vespa/searchlib/features/tensor_from_attribute_executor.h @@ -28,9 +28,9 @@ private: public: TensorFromAttributeExecutor(const search::attribute::IAttributeVector *attribute, - const vespalib::string &dimension) + const vespalib::eval::ValueType &valueType) : _attribute(attribute), - _type(vespalib::eval::ValueType::make_type(CellType::DOUBLE, {{dimension}})), + _type(valueType), _attrBuffer(), _addr_ref(), _tensor() diff --git a/searchlib/src/vespa/searchlib/features/tensor_from_labels_feature.cpp b/searchlib/src/vespa/searchlib/features/tensor_from_labels_feature.cpp index b72a75bd19f..f36c1dbfdaa 100644 --- a/searchlib/src/vespa/searchlib/features/tensor_from_labels_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/tensor_from_labels_feature.cpp @@ -41,15 +41,23 @@ TensorFromLabelsBlueprint::setup(const search::fef::IIndexEnvironment &env, // _params[0] = source ('attribute(name)' OR 'query(param)'); // _params[1] = dimension (optional); bool validSource = extractSource(params[0].getValue()); + if (! validSource) { + return fail("invalid source: '%s'", params[0].getValue().c_str()); + } if (params.size() == 2) { _dimension = params[1].getValue(); } else { _dimension = _sourceParam; } + auto vt = ValueType::make_type(CellType::DOUBLE, {{_dimension}}); + _valueType = ValueType::from_spec(vt.to_spec()); + if (_valueType.is_error()) { + return fail("invalid dimension name: '%s'", _dimension.c_str()); + } describeOutput("tensor", "The tensor created from the given source (attribute field or query parameter)", - FeatureType::object(ValueType::make_type(CellType::DOUBLE, {{_dimension}}))); - return validSource; + FeatureType::object(_valueType)); + return true; } namespace { @@ -57,23 +65,24 @@ namespace { FeatureExecutor & createAttributeExecutor(const search::fef::IQueryEnvironment &env, const vespalib::string &attrName, - const vespalib::string &dimension, vespalib::Stash &stash) + const ValueType &valueType, + vespalib::Stash &stash) { const IAttributeVector *attribute = env.getAttributeContext().getAttribute(attrName); if (attribute == NULL) { Issue::report("tensor_from_labels feature: The attribute vector '%s' was not found." " Returning empty tensor.", attrName.c_str()); - return ConstantTensorExecutor::createEmpty(ValueType::make_type(CellType::DOUBLE, {{dimension}}), stash); + return ConstantTensorExecutor::createEmpty(valueType, stash); } if (attribute->isFloatingPointType()) { Issue::report("tensor_from_labels feature: The attribute vector '%s' must have basic type string or integer." " Returning empty tensor.", attrName.c_str()); - return ConstantTensorExecutor::createEmpty(ValueType::make_type(CellType::DOUBLE, {{dimension}}), stash); + return ConstantTensorExecutor::createEmpty(valueType, stash); } if (attribute->getCollectionType() == search::attribute::CollectionType::WSET) { Issue::report("tensor_from_labels feature: The attribute vector '%s' is a weighted set - use tensorFromWeightedSet instead." " Returning empty tensor.", attrName.c_str()); - return ConstantTensorExecutor::createEmpty(ValueType::make_type(CellType::DOUBLE, {{dimension}}), stash); + return ConstantTensorExecutor::createEmpty(valueType, stash); } // Note that for array attribute vectors the default weight is 1.0 for all values. // This means we can get the attribute content as weighted content and build @@ -81,25 +90,25 @@ createAttributeExecutor(const search::fef::IQueryEnvironment &env, if (attribute->isIntegerType()) { // Using WeightedStringContent ensures that the integer values are converted // to strings while extracting them from the attribute. - return stash.create<TensorFromAttributeExecutor<WeightedStringContent>>(attribute, dimension); + return stash.create<TensorFromAttributeExecutor<WeightedStringContent>>(attribute, valueType); } // When the underlying attribute is of type string we can reference these values // using WeightedConstCharContent. - return stash.create<TensorFromAttributeExecutor<WeightedConstCharContent>>(attribute, dimension); + return stash.create<TensorFromAttributeExecutor<WeightedConstCharContent>>(attribute, valueType); } FeatureExecutor & createQueryExecutor(const search::fef::IQueryEnvironment &env, const vespalib::string &queryKey, - const vespalib::string &dimension, vespalib::Stash &stash) + const ValueType &valueType, + vespalib::Stash &stash) { - ValueType type = ValueType::make_type(CellType::DOUBLE, {{dimension}}); search::fef::Property prop = env.getProperties().lookup(queryKey); if (prop.found() && !prop.get().empty()) { std::vector<vespalib::string> vector; ArrayParser::parse(prop.get(), vector); auto factory = FastValueBuilderFactory::get(); - auto builder = factory.create_value_builder<double>(type, 1, 1, vector.size()); + auto builder = factory.create_value_builder<double>(valueType, 1, 1, vector.size()); std::vector<vespalib::stringref> addr_ref; for (const auto &elem : vector) { addr_ref.clear(); @@ -109,7 +118,7 @@ createQueryExecutor(const search::fef::IQueryEnvironment &env, } return ConstantTensorExecutor::create(builder->build(std::move(builder)), stash); } - return ConstantTensorExecutor::createEmpty(type, stash); + return ConstantTensorExecutor::createEmpty(valueType, stash); } } @@ -118,11 +127,11 @@ FeatureExecutor & TensorFromLabelsBlueprint::createExecutor(const search::fef::IQueryEnvironment &env, vespalib::Stash &stash) const { if (_sourceType == ATTRIBUTE_SOURCE) { - return createAttributeExecutor(env, _sourceParam, _dimension, stash); + return createAttributeExecutor(env, _sourceParam, _valueType, stash); } else if (_sourceType == QUERY_SOURCE) { - return createQueryExecutor(env, _sourceParam, _dimension, stash); + return createQueryExecutor(env, _sourceParam, _valueType, stash); } - return ConstantTensorExecutor::createEmpty(ValueType::make_type(CellType::DOUBLE, {{_dimension}}), stash); + return ConstantTensorExecutor::createEmpty(_valueType, stash); } } // namespace features diff --git a/searchlib/src/vespa/searchlib/features/tensor_from_weighted_set_feature.cpp b/searchlib/src/vespa/searchlib/features/tensor_from_weighted_set_feature.cpp index cbe262a0cbd..312f9ee2bc6 100644 --- a/searchlib/src/vespa/searchlib/features/tensor_from_weighted_set_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/tensor_from_weighted_set_feature.cpp @@ -54,15 +54,23 @@ TensorFromWeightedSetBlueprint::setup(const search::fef::IIndexEnvironment &env, // _params[0] = source ('attribute(name)' OR 'query(param)'); // _params[1] = dimension (optional); bool validSource = extractSource(params[0].getValue()); + if (! validSource) { + return fail("invalid source: '%s'", params[0].getValue().c_str()); + } if (params.size() == 2) { _dimension = params[1].getValue(); } else { _dimension = _sourceParam; } + auto vt = ValueType::make_type(CellType::DOUBLE, {{_dimension}}); + _valueType = ValueType::from_spec(vt.to_spec()); + if (_valueType.is_error()) { + return fail("invalid dimension name: '%s'", _dimension.c_str()); + } describeOutput("tensor", "The tensor created from the given weighted set source (attribute field or query parameter)", - FeatureType::object(ValueType::make_type(CellType::DOUBLE, {{_dimension}}))); - return validSource; + FeatureType::object(_valueType)); + return true; } namespace { @@ -70,45 +78,45 @@ namespace { FeatureExecutor & createAttributeExecutor(const search::fef::IQueryEnvironment &env, const vespalib::string &attrName, - const vespalib::string &dimension, + const ValueType &valueType, vespalib::Stash &stash) { const IAttributeVector *attribute = env.getAttributeContext().getAttribute(attrName); if (attribute == NULL) { Issue::report("tensor_from_weighted_set feature: The attribute vector '%s' was not found." " Returning empty tensor.", attrName.c_str()); - return ConstantTensorExecutor::createEmpty(ValueType::make_type(CellType::DOUBLE, {{dimension}}), stash); + return ConstantTensorExecutor::createEmpty(valueType, stash); } if (attribute->getCollectionType() != search::attribute::CollectionType::WSET || attribute->isFloatingPointType()) { Issue::report("tensor_from_weighted_set feature: The attribute vector '%s' is NOT of type weighted set of string or integer." " Returning empty tensor.", attrName.c_str()); - return ConstantTensorExecutor::createEmpty(ValueType::make_type(CellType::DOUBLE, {{dimension}}), stash); + return ConstantTensorExecutor::createEmpty(valueType, stash); } if (attribute->isIntegerType()) { // Using WeightedStringContent ensures that the integer values are converted // to strings while extracting them from the attribute. - return stash.create<TensorFromAttributeExecutor<WeightedStringContent>>(attribute, dimension); + return stash.create<TensorFromAttributeExecutor<WeightedStringContent>>(attribute, valueType); } // When the underlying attribute is of type string we can reference these values // using WeightedConstCharContent. - return stash.create<TensorFromAttributeExecutor<WeightedConstCharContent>>(attribute, dimension); + return stash.create<TensorFromAttributeExecutor<WeightedConstCharContent>>(attribute, valueType); } FeatureExecutor & createQueryExecutor(const search::fef::IQueryEnvironment &env, const vespalib::string &queryKey, - const vespalib::string &dimension, vespalib::Stash &stash) + const ValueType &valueType, + vespalib::Stash &stash) { - ValueType type = ValueType::make_type(CellType::DOUBLE, {{dimension}}); search::fef::Property prop = env.getProperties().lookup(queryKey); if (prop.found() && !prop.get().empty()) { WeightedStringVector vector; WeightedSetParser::parse(prop.get(), vector); auto factory = FastValueBuilderFactory::get(); size_t sz = vector._data.size(); - auto builder = factory.create_value_builder<double>(type, 1, 1, sz); + auto builder = factory.create_value_builder<double>(valueType, 1, 1, sz); std::vector<vespalib::stringref> addr_ref; for (const auto &elem : vector._data) { addr_ref.clear(); @@ -118,7 +126,7 @@ createQueryExecutor(const search::fef::IQueryEnvironment &env, } return ConstantTensorExecutor::create(builder->build(std::move(builder)), stash); } - return ConstantTensorExecutor::createEmpty(type, stash); + return ConstantTensorExecutor::createEmpty(valueType, stash); } } @@ -127,11 +135,11 @@ FeatureExecutor & TensorFromWeightedSetBlueprint::createExecutor(const search::fef::IQueryEnvironment &env, vespalib::Stash &stash) const { if (_sourceType == ATTRIBUTE_SOURCE) { - return createAttributeExecutor(env, _sourceParam, _dimension, stash); + return createAttributeExecutor(env, _sourceParam, _valueType, stash); } else if (_sourceType == QUERY_SOURCE) { - return createQueryExecutor(env, _sourceParam, _dimension, stash); + return createQueryExecutor(env, _sourceParam, _valueType, stash); } - return ConstantTensorExecutor::createEmpty(ValueType::make_type(CellType::DOUBLE, {{_dimension}}), stash); + return ConstantTensorExecutor::createEmpty(_valueType, stash); } } // namespace features diff --git a/searchlib/src/vespa/searchlib/fef/ranksetup.cpp b/searchlib/src/vespa/searchlib/fef/ranksetup.cpp index 4c676d5ba5c..823e39199df 100644 --- a/searchlib/src/vespa/searchlib/fef/ranksetup.cpp +++ b/searchlib/src/vespa/searchlib/fef/ranksetup.cpp @@ -66,7 +66,6 @@ RankSetup::RankSetup(const BlueprintFactory &factory, const IIndexEnvironment &i _diversityCutoffStrategy("loose"), _softTimeoutEnabled(false), _softTimeoutTailCost(0.1), - _softTimeoutFactor(0.5), _global_filter_lower_limit(0.0), _global_filter_upper_limit(1.0), _mutateOnMatch(), @@ -120,7 +119,6 @@ RankSetup::configure() setRankScoreDropLimit(hitcollector::RankScoreDropLimit::lookup(_indexEnv.getProperties())); setSoftTimeoutEnabled(softtimeout::Enabled::lookup(_indexEnv.getProperties())); setSoftTimeoutTailCost(softtimeout::TailCost::lookup(_indexEnv.getProperties())); - setSoftTimeoutFactor(softtimeout::Factor::lookup(_indexEnv.getProperties())); set_global_filter_lower_limit(matching::GlobalFilterLowerLimit::lookup(_indexEnv.getProperties())); set_global_filter_upper_limit(matching::GlobalFilterUpperLimit::lookup(_indexEnv.getProperties())); _mutateOnMatch._attribute = mutate::on_match::Attribute::lookup(_indexEnv.getProperties()); diff --git a/searchlib/src/vespa/searchlib/fef/ranksetup.h b/searchlib/src/vespa/searchlib/fef/ranksetup.h index 783c1506ff0..832b86d042a 100644 --- a/searchlib/src/vespa/searchlib/fef/ranksetup.h +++ b/searchlib/src/vespa/searchlib/fef/ranksetup.h @@ -74,7 +74,6 @@ private: vespalib::string _diversityCutoffStrategy; bool _softTimeoutEnabled; double _softTimeoutTailCost; - double _softTimeoutFactor; double _global_filter_lower_limit; double _global_filter_upper_limit; MutateOperation _mutateOnMatch; @@ -211,11 +210,6 @@ public: **/ uint32_t getArraySize() const { return _arraySize; } - /** whether match phase should do graceful degradation */ - bool hasMatchPhaseDegradation() const { - return (_degradationAttribute.size() > 0); - } - /** get name of attribute to use for graceful degradation in match phase */ vespalib::string getDegradationAttribute() const { return _degradationAttribute; @@ -390,20 +384,10 @@ public: **/ void setIgnoreDefaultRankFeatures(bool flag) { _ignoreDefaultRankFeatures = flag; } - /** - * Get the flag indicating whether we should ignore the default - * rank features (the ones specified by the plugins themselves) - * - * @return true means ignore default rank features - **/ - bool getIgnoreDefaultRankFeatures() { return _ignoreDefaultRankFeatures; } - void setSoftTimeoutEnabled(bool v) { _softTimeoutEnabled = v; } bool getSoftTimeoutEnabled() const { return _softTimeoutEnabled; } void setSoftTimeoutTailCost(double v) { _softTimeoutTailCost = v; } double getSoftTimeoutTailCost() const { return _softTimeoutTailCost; } - void setSoftTimeoutFactor(double v) { _softTimeoutFactor = v; } - double getSoftTimeoutFactor() const { return _softTimeoutFactor; } void set_global_filter_lower_limit(double v) { _global_filter_lower_limit = v; } double get_global_filter_lower_limit() const { return _global_filter_lower_limit; } diff --git a/searchsummary/src/vespa/juniper/sumdesc.cpp b/searchsummary/src/vespa/juniper/sumdesc.cpp index aa6aededa0c..e88f7971666 100644 --- a/searchsummary/src/vespa/juniper/sumdesc.cpp +++ b/searchsummary/src/vespa/juniper/sumdesc.cpp @@ -623,9 +623,8 @@ int SummaryDesc::complete_extended_token(unsigned char* start, ssize_t length, /* Return a highlight tagged summary string from this summary * description */ -std::string SummaryDesc::get_summary(const char* buffer, size_t bytes, - const SummaryConfig* sumconf, - size_t& char_size) +std::string +SummaryDesc::get_summary(const char* buffer, size_t bytes, const SummaryConfig* sumconf, size_t& char_size) { std::vector<char> s; ssize_t prev_end = 0; @@ -759,7 +758,7 @@ std::string SummaryDesc::get_summary(const char* buffer, size_t bytes, s.size(), a.charLen()); _sumconf = NULL; // Not valid after this call. char_size = a.charLen(); - return std::string(&s[0], s.size()); + return std::string(s.begin(), s.end()); } diff --git a/storage/src/tests/distributor/bucketdbmetricupdatertest.cpp b/storage/src/tests/distributor/bucketdbmetricupdatertest.cpp index c4536c6fa2c..4d04e3ca51a 100644 --- a/storage/src/tests/distributor/bucketdbmetricupdatertest.cpp +++ b/storage/src/tests/distributor/bucketdbmetricupdatertest.cpp @@ -7,7 +7,6 @@ #include <vespa/vespalib/util/memoryusage.h> #include <vespa/vespalib/gtest/gtest.h> #include <string> -#include <sstream> namespace storage::distributor { @@ -16,19 +15,16 @@ using namespace ::testing; struct BucketDBMetricUpdaterTest : Test { void visitBucketWith2Copies1Trusted(BucketDBMetricUpdater& metricUpdater); - void visitBucketWith2CopiesBothTrusted( - BucketDBMetricUpdater& metricUpdater); + void visitBucketWith2CopiesBothTrusted(BucketDBMetricUpdater& metricUpdater); void visitBucketWith1Copy(BucketDBMetricUpdater& metricUpdater); - using NodeToReplicasMap = std::unordered_map<uint16_t, uint32_t>; + using NodeToReplicasMap = MinReplicaMap; NodeToReplicasMap replicaStatsOf(BucketDBMetricUpdater& metricUpdater); BucketDBMetricUpdaterTest(); }; -BucketDBMetricUpdaterTest::BucketDBMetricUpdaterTest() -{ -} +BucketDBMetricUpdaterTest::BucketDBMetricUpdaterTest() = default; namespace { @@ -38,8 +34,6 @@ void addNode(BucketInfo& info, uint16_t node, uint32_t crc) { info.addNode(BucketCopy(1234, node, apiInfo), order); } -using Trusted = bool; - BucketInfo makeInfo(uint32_t copy0Crc) { @@ -271,8 +265,7 @@ TEST_F(BucketDBMetricUpdaterTest, complete_round_clears_working_state) { // Replicas on nodes 0 and 1. void -BucketDBMetricUpdaterTest::visitBucketWith2Copies1Trusted( - BucketDBMetricUpdater& metricUpdater) +BucketDBMetricUpdaterTest::visitBucketWith2Copies1Trusted(BucketDBMetricUpdater& metricUpdater) { BucketInfo info; addNode(info, 0, 100); @@ -283,8 +276,7 @@ BucketDBMetricUpdaterTest::visitBucketWith2Copies1Trusted( // Replicas on nodes 0 and 2. void -BucketDBMetricUpdaterTest::visitBucketWith2CopiesBothTrusted( - BucketDBMetricUpdater& metricUpdater) +BucketDBMetricUpdaterTest::visitBucketWith2CopiesBothTrusted(BucketDBMetricUpdater& metricUpdater) { BucketInfo info; addNode(info, 0, 200); diff --git a/storage/src/tests/distributor/distributor_bucket_space_test.cpp b/storage/src/tests/distributor/distributor_bucket_space_test.cpp index 41e0dafdaaf..3ea4c1ca3c2 100644 --- a/storage/src/tests/distributor/distributor_bucket_space_test.cpp +++ b/storage/src/tests/distributor/distributor_bucket_space_test.cpp @@ -102,17 +102,17 @@ DistributorBucketSpaceTest::count_service_layer_buckets(const std::vector<Bucket CountVector result(3); std::vector<uint16_t> ideal_nodes; for (auto& bucket : buckets) { - auto &ideal_nodes_bundle = bucket_space.get_ideal_service_layer_nodes_bundle(bucket); + const auto & ideal_nodes_bundle = bucket_space.get_ideal_service_layer_nodes_bundle(bucket); for (uint32_t i = 0; i < 3; ++i) { switch (i) { case 0: - ideal_nodes = ideal_nodes_bundle.get_available_nodes(); + ideal_nodes = ideal_nodes_bundle.available_nodes(); break; case 1: - ideal_nodes = ideal_nodes_bundle.get_available_nonretired_nodes(); + ideal_nodes = ideal_nodes_bundle.available_nonretired_nodes(); break; case 2: - ideal_nodes = ideal_nodes_bundle.get_available_nonretired_or_maintenance_nodes(); + ideal_nodes = ideal_nodes_bundle.available_nonretired_or_maintenance_nodes(); break; default: ; diff --git a/storage/src/tests/distributor/distributor_host_info_reporter_test.cpp b/storage/src/tests/distributor/distributor_host_info_reporter_test.cpp index 6dfab5abc21..a72dfec2d94 100644 --- a/storage/src/tests/distributor/distributor_host_info_reporter_test.cpp +++ b/storage/src/tests/distributor/distributor_host_info_reporter_test.cpp @@ -14,7 +14,7 @@ namespace storage::distributor { using End = vespalib::JsonStream::End; using File = vespalib::File; -using MinReplicaStats = std::unordered_map<uint16_t, uint32_t>; +using MinReplicaStats = MinReplicaMap; using Object = vespalib::JsonStream::Object; using PerNodeBucketSpacesStats = BucketSpacesStatsProvider::PerNodeBucketSpacesStats; using BucketSpacesStats = BucketSpacesStatsProvider::BucketSpacesStats; @@ -36,7 +36,7 @@ struct MockedMinReplicaProvider : MinReplicaProvider MinReplicaStats minReplica; ~MockedMinReplicaProvider() override; - std::unordered_map<uint16_t, uint32_t> getMinReplica() const override { + MinReplicaMap getMinReplica() const override { return minReplica; } }; diff --git a/storage/src/tests/distributor/distributor_stripe_test.cpp b/storage/src/tests/distributor/distributor_stripe_test.cpp index e963b505bc2..566fb704105 100644 --- a/storage/src/tests/distributor/distributor_stripe_test.cpp +++ b/storage/src/tests/distributor/distributor_stripe_test.cpp @@ -1043,13 +1043,13 @@ TEST_F(DistributorStripeTest, enable_condition_probing_config_is_propagated_to_i { setup_stripe(Redundancy(1), NodeCount(1), "distributor:1 storage:1"); - EXPECT_FALSE(getConfig().enable_condition_probing()); - - configure_enable_condition_probing(true); EXPECT_TRUE(getConfig().enable_condition_probing()); configure_enable_condition_probing(false); EXPECT_FALSE(getConfig().enable_condition_probing()); + + configure_enable_condition_probing(true); + EXPECT_TRUE(getConfig().enable_condition_probing()); } } diff --git a/storage/src/tests/distributor/distributor_stripe_test_util.cpp b/storage/src/tests/distributor/distributor_stripe_test_util.cpp index 7a64eda28ff..6ececa39583 100644 --- a/storage/src/tests/distributor/distributor_stripe_test_util.cpp +++ b/storage/src/tests/distributor/distributor_stripe_test_util.cpp @@ -40,34 +40,22 @@ DistributorStripeTestUtil::createLinks() _node = std::make_unique<TestDistributorApp>(_config.getConfigId()); _metrics = std::make_shared<DistributorMetricSet>(); _ideal_state_metrics = std::make_shared<IdealStateMetricSet>(); - _stripe = std::make_unique<DistributorStripe>(_node->getComponentRegister(), - *_metrics, - *_ideal_state_metrics, - _node->node_identity(), - _messageSender, - *this, - _done_initializing); + _stripe = std::make_unique<DistributorStripe>(_node->getComponentRegister(), *_metrics, *_ideal_state_metrics, + _node->node_identity(), _messageSender, *this, _done_initializing); } void -DistributorStripeTestUtil::setup_stripe(int redundancy, - int nodeCount, - const std::string& systemState, - uint32_t earlyReturn, - bool requirePrimaryToBeWritten) +DistributorStripeTestUtil::setup_stripe(int redundancy, int nodeCount, const std::string& systemState, + uint32_t earlyReturn, bool requirePrimaryToBeWritten) { setup_stripe(redundancy, nodeCount, lib::ClusterStateBundle(lib::ClusterState(systemState)), earlyReturn, requirePrimaryToBeWritten); } void -DistributorStripeTestUtil::setup_stripe(int redundancy, - int node_count, - const lib::ClusterStateBundle& state, - uint32_t early_return, - bool require_primary_to_be_written) +DistributorStripeTestUtil::setup_stripe(int redundancy, int node_count, const lib::ClusterStateBundle& state, + uint32_t early_return, bool require_primary_to_be_written) { - lib::Distribution::DistributionConfigBuilder config( - lib::Distribution::getDefaultDistributionConfig(redundancy, node_count).get()); + lib::Distribution::DistributionConfigBuilder config(lib::Distribution::getDefaultDistributionConfig(redundancy, node_count).get()); config.redundancy = redundancy; config.initialRedundancy = early_return; config.ensurePrimaryPersisted = require_primary_to_be_written; @@ -93,8 +81,7 @@ DistributorStripeTestUtil::setup_stripe(int redundancy, void DistributorStripeTestUtil::set_redundancy(uint32_t redundancy) { - auto distribution = std::make_shared<lib::Distribution>( - lib::Distribution::getDefaultDistributionConfig(redundancy, 100)); + auto distribution = std::make_shared<lib::Distribution>(lib::Distribution::getDefaultDistributionConfig(redundancy, 100)); // Same rationale for not triggering a full distribution change as // in setup_stripe() above _node->getComponentRegister().setDistribution(distribution); @@ -217,8 +204,7 @@ DistributorStripeTestUtil::getIdealStr(document::BucketId id, const lib::Cluster } std::vector<uint16_t> nodes; - getDistribution().getIdealNodes( - lib::NodeType::STORAGE, state, id, nodes); + getDistribution().getIdealNodes(lib::NodeType::STORAGE, state, id, nodes, "uim"); std::sort(nodes.begin(), nodes.end()); std::ostringstream ost; ost << id << ": " << dumpVector(nodes); @@ -226,8 +212,7 @@ DistributorStripeTestUtil::getIdealStr(document::BucketId id, const lib::Cluster } void -DistributorStripeTestUtil::addIdealNodes(const lib::ClusterState& state, - const document::BucketId& id) +DistributorStripeTestUtil::addIdealNodes(const lib::ClusterState& state, const document::BucketId& id) { BucketDatabase::Entry entry = getBucket(id); @@ -236,15 +221,11 @@ DistributorStripeTestUtil::addIdealNodes(const lib::ClusterState& state, } std::vector<uint16_t> res; - getDistribution().getIdealNodes( - lib::NodeType::STORAGE, state, id, res); + getDistribution().getIdealNodes(lib::NodeType::STORAGE, state, id, res, "uim"); for (uint32_t i = 0; i < res.size(); ++i) { - if (state.getNodeState(lib::Node(lib::NodeType::STORAGE, res[i])).getState() != - lib::State::MAINTENANCE) - { - entry->addNode(BucketCopy(0, res[i], api::BucketInfo(1,1,1)), - toVector<uint16_t>(0)); + if (state.getNodeState(lib::Node(lib::NodeType::STORAGE, res[i])).getState() != lib::State::MAINTENANCE) { + entry->addNode(BucketCopy(0, res[i], api::BucketInfo(1,1,1)), toVector<uint16_t>(0)); } } @@ -292,10 +273,7 @@ DistributorStripeTestUtil::addNodesToBucketDB(const document::Bucket& bucket, co } uint16_t idx = atoi(tok2[0].data()); - BucketCopy node( - 0, - idx, - info); + BucketCopy node(0, idx, info); // Allow user to manually override trusted and active. if (tok3.size() > flagsIdx && tok3[flagsIdx] == "t") { @@ -309,44 +287,32 @@ DistributorStripeTestUtil::addNodesToBucketDB(const document::Bucket& bucket, co } void -DistributorStripeTestUtil::addNodesToBucketDB(const document::BucketId& id, - const std::string& nodeStr) -{ +DistributorStripeTestUtil::addNodesToBucketDB(const document::BucketId& id, const std::string& nodeStr) { addNodesToBucketDB(document::Bucket(makeBucketSpace(), id), nodeStr); } void -DistributorStripeTestUtil::removeFromBucketDB(const document::BucketId& id) -{ +DistributorStripeTestUtil::removeFromBucketDB(const document::BucketId& id) { getBucketDatabase().remove(id); } void -DistributorStripeTestUtil::addIdealNodes(const document::BucketId& id) -{ +DistributorStripeTestUtil::addIdealNodes(const document::BucketId& id) { // TODO STRIPE roundabout way of getting state bundle..! addIdealNodes(*operation_context().cluster_state_bundle().getBaselineClusterState(), id); } void -DistributorStripeTestUtil::insertBucketInfo(document::BucketId id, - uint16_t node, - uint32_t checksum, - uint32_t count, - uint32_t size, - bool trusted, - bool active) +DistributorStripeTestUtil::insertBucketInfo(document::BucketId id, uint16_t node, uint32_t checksum, + uint32_t count, uint32_t size, bool trusted, bool active) { api::BucketInfo info(checksum, count, size); insertBucketInfo(id, node, info, trusted, active); } void -DistributorStripeTestUtil::insertBucketInfo(document::BucketId id, - uint16_t node, - const api::BucketInfo& info, - bool trusted, - bool active) +DistributorStripeTestUtil::insertBucketInfo(document::BucketId id, uint16_t node, const api::BucketInfo& info, + bool trusted, bool active) { BucketDatabase::Entry entry = getBucketDatabase().get(id); if (!entry.valid()) { @@ -358,9 +324,7 @@ DistributorStripeTestUtil::insertBucketInfo(document::BucketId id, info2.setActive(); } BucketCopy copy(operation_context().generate_unique_timestamp(), node, info2); - entry->addNode(copy.setTrusted(trusted), toVector<uint16_t>(0)); - getBucketDatabase().update(entry); } @@ -371,9 +335,7 @@ DistributorStripeTestUtil::dumpBucket(const document::BucketId& bid) } void -DistributorStripeTestUtil::sendReply(Operation& op, - int idx, - api::ReturnCode::Result result) +DistributorStripeTestUtil::sendReply(Operation& op, int idx, api::ReturnCode::Result result) { if (idx == -1) { idx = _sender.commands().size() - 1; @@ -387,20 +349,17 @@ DistributorStripeTestUtil::sendReply(Operation& op, } BucketDatabase::Entry -DistributorStripeTestUtil::getBucket(const document::Bucket& bucket) const -{ +DistributorStripeTestUtil::getBucket(const document::Bucket& bucket) const { return getBucketDatabase(bucket.getBucketSpace()).get(bucket.getBucketId()); } BucketDatabase::Entry -DistributorStripeTestUtil::getBucket(const document::BucketId& bId) const -{ +DistributorStripeTestUtil::getBucket(const document::BucketId& bId) const { return getBucketDatabase().get(bId); } void -DistributorStripeTestUtil::disableBucketActivationInConfig(bool disable) -{ +DistributorStripeTestUtil::disableBucketActivationInConfig(bool disable) { ConfigBuilder builder; builder.disableBucketActivation = disable; configure_stripe(builder); @@ -437,14 +396,12 @@ DistributorStripeTestUtil::doc_selection_parser() const { } DistributorMetricSet& -DistributorStripeTestUtil::metrics() -{ +DistributorStripeTestUtil::metrics() { return *_metrics; } bool -DistributorStripeTestUtil::tick() -{ +DistributorStripeTestUtil::tick() { return _stripe->tick(); } @@ -553,8 +510,7 @@ DistributorStripeTestUtil::getBucketSpaces() const void DistributorStripeTestUtil::enable_cluster_state(vespalib::stringref state) { - getBucketDBUpdater().simulate_cluster_state_bundle_activation( - lib::ClusterStateBundle(lib::ClusterState(state))); + getBucketDBUpdater().simulate_cluster_state_bundle_activation(lib::ClusterStateBundle(lib::ClusterState(state))); } void diff --git a/storage/src/tests/distributor/mock_tickable_stripe.h b/storage/src/tests/distributor/mock_tickable_stripe.h index ec2f978c029..77a6f537d28 100644 --- a/storage/src/tests/distributor/mock_tickable_stripe.h +++ b/storage/src/tests/distributor/mock_tickable_stripe.h @@ -23,7 +23,7 @@ struct MockTickableStripe : TickableStripe { const lib::Distribution&, const lib::ClusterState&, const char*, - const std::unordered_set<uint16_t>&, + const OutdatedNodes &, const std::vector<dbtransition::Entry>&) override { abort(); diff --git a/storage/src/tests/distributor/multi_thread_stripe_access_guard_test.cpp b/storage/src/tests/distributor/multi_thread_stripe_access_guard_test.cpp index 6bc98ef022e..db89b30efb2 100644 --- a/storage/src/tests/distributor/multi_thread_stripe_access_guard_test.cpp +++ b/storage/src/tests/distributor/multi_thread_stripe_access_guard_test.cpp @@ -25,7 +25,7 @@ struct AggregationTestingMockTickableStripe : MockTickableStripe { } void merge_entries_into_db(document::BucketSpace, api::Timestamp, const lib::Distribution&, - const lib::ClusterState&, const char*, const std::unordered_set<uint16_t>&, + const lib::ClusterState&, const char*, const OutdatedNodes &, const std::vector<dbtransition::Entry>& entries_in) override { entries = entries_in; } diff --git a/storage/src/tests/distributor/pendingmessagetrackertest.cpp b/storage/src/tests/distributor/pendingmessagetrackertest.cpp index 3bfa1027a82..8277281206d 100644 --- a/storage/src/tests/distributor/pendingmessagetrackertest.cpp +++ b/storage/src/tests/distributor/pendingmessagetrackertest.cpp @@ -162,10 +162,14 @@ TEST_F(PendingMessageTrackerTest, simple) { clock.setAbsoluteTimeInSeconds(1); PendingMessageTracker tracker(compReg, 0); + std::ostringstream dummy; // Enable time tracking + tracker.reportStatus(dummy, framework::HttpUrlPath("/pendingmessages?order=bucket")); + auto remove = std::make_shared<api::RemoveCommand>( makeDocumentBucket(document::BucketId(16, 1234)), document::DocumentId("id:footype:testdoc:n=1234:foo"), 1001); remove->setAddress(makeStorageAddress(0)); + tracker.insert(remove); { @@ -238,6 +242,8 @@ TEST_F(PendingMessageTrackerTest, multiple_messages) { compReg.setClock(clock); clock.setAbsoluteTimeInSeconds(1); PendingMessageTracker tracker(compReg, 0); + std::ostringstream dummy; // Enable time tracking + tracker.reportStatus(dummy, framework::HttpUrlPath("/pendingmessages?order=bucket")); insertMessages(tracker); diff --git a/storage/src/tests/distributor/simplemaintenancescannertest.cpp b/storage/src/tests/distributor/simplemaintenancescannertest.cpp index 723b0baa6cd..b5dc72d995b 100644 --- a/storage/src/tests/distributor/simplemaintenancescannertest.cpp +++ b/storage/src/tests/distributor/simplemaintenancescannertest.cpp @@ -165,7 +165,7 @@ TEST_F(SimpleMaintenanceScannerTest, pending_maintenance_operation_statistics) { "split bucket: 0, join bucket: 0, " "set bucket state: 0, garbage collection: 0"); { - auto stats(_scanner->getPendingMaintenanceStats()); + const auto & stats = _scanner->getPendingMaintenanceStats(); EXPECT_EQ(expectedEmpty, stringifyGlobalPendingStats(stats)); } @@ -173,7 +173,7 @@ TEST_F(SimpleMaintenanceScannerTest, pending_maintenance_operation_statistics) { // All mock operations generated have the merge type. { - auto stats(_scanner->getPendingMaintenanceStats()); + const auto & stats = _scanner->getPendingMaintenanceStats(); std::string expected("delete bucket: 0, merge bucket: 2, " "split bucket: 0, join bucket: 0, " "set bucket state: 0, garbage collection: 0"); @@ -182,7 +182,7 @@ TEST_F(SimpleMaintenanceScannerTest, pending_maintenance_operation_statistics) { _scanner->reset(); { - auto stats(_scanner->getPendingMaintenanceStats()); + const auto & stats = _scanner->getPendingMaintenanceStats(); EXPECT_EQ(expectedEmpty, stringifyGlobalPendingStats(stats)); } } @@ -191,14 +191,14 @@ TEST_F(SimpleMaintenanceScannerTest, per_node_maintenance_stats_are_tracked) { addBucketToDb(1); addBucketToDb(3); { - auto stats(_scanner->getPendingMaintenanceStats()); + const auto & stats = _scanner->getPendingMaintenanceStats(); NodeMaintenanceStats emptyStats; EXPECT_EQ(emptyStats, stats.perNodeStats.forNode(0, makeBucketSpace())); } ASSERT_TRUE(scanEntireDatabase(2)); // Mock is currently hardwired to increment movingOut for node 1 and // copyingIn for node 2 per bucket iterated (we've got 2). - auto stats(_scanner->getPendingMaintenanceStats()); + const auto & stats = _scanner->getPendingMaintenanceStats(); { NodeMaintenanceStats wantedNode1Stats; wantedNode1Stats.movingOut = 2; diff --git a/storage/src/tests/distributor/statecheckerstest.cpp b/storage/src/tests/distributor/statecheckerstest.cpp index 16854cd63c6..4ca4d70a816 100644 --- a/storage/src/tests/distributor/statecheckerstest.cpp +++ b/storage/src/tests/distributor/statecheckerstest.cpp @@ -1383,9 +1383,8 @@ std::string StateCheckersTest::testGarbageCollection( getBucketDatabase().update(e); NodeMaintenanceStatsTracker statsTracker; - StateChecker::Context c(node_context(), operation_context(), - getDistributorBucketSpace(), statsTracker, - makeDocumentBucket(e.getBucketId())); + StateChecker::Context c(node_context(), operation_context(), getDistributorBucketSpace(), + statsTracker, makeDocumentBucket(e.getBucketId())); getClock().setAbsoluteTimeInSeconds(nowTimestamp); return testStateChecker(checker, c, false, PendingMessage(), includePriority, includeSchedulingPri); } @@ -1394,38 +1393,29 @@ TEST_F(StateCheckersTest, garbage_collection) { // BucketId(17, 0) has id (and thus 'hash') 0x4400000000000000. With a // check interval modulo of 3600, this implies a start point of 848. - EXPECT_EQ("NO OPERATIONS GENERATED", - testGarbageCollection(900, 3600 + 847, 3600)); + EXPECT_EQ("NO OPERATIONS GENERATED", testGarbageCollection(900, 3600 + 847, 3600)); - EXPECT_EQ("[Needs garbage collection: Last check at 900, current time 4448, " - "configured interval 3600]", + EXPECT_EQ("[Needs garbage collection: Last check at 900, current time 4448, configured interval 3600]", testGarbageCollection(900, 3600 + 848, 3600)); - EXPECT_EQ("[Needs garbage collection: Last check at 3, current time 4000, " - "configured interval 3600]", + EXPECT_EQ("[Needs garbage collection: Last check at 3, current time 4000, configured interval 3600]", testGarbageCollection(3, 4000, 3600)); // GC start point 3648. - EXPECT_EQ("NO OPERATIONS GENERATED", - testGarbageCollection(3, 3647, 8000)); + EXPECT_EQ("NO OPERATIONS GENERATED", testGarbageCollection(3, 3647, 8000)); - EXPECT_EQ("[Needs garbage collection: Last check at 3, current time 4000, " - "configured interval 3600]", + EXPECT_EQ("[Needs garbage collection: Last check at 3, current time 4000, configured interval 3600]", testGarbageCollection(3, 4000, 3600)); // GC explicitly disabled. - EXPECT_EQ("NO OPERATIONS GENERATED", - testGarbageCollection(3, 4000, 0)); + EXPECT_EQ("NO OPERATIONS GENERATED", testGarbageCollection(3, 4000, 0)); - EXPECT_EQ("NO OPERATIONS GENERATED", - testGarbageCollection(3, 3, 1)); + EXPECT_EQ("NO OPERATIONS GENERATED", testGarbageCollection(3, 3, 1)); - EXPECT_EQ("[Needs garbage collection: Last check at 3, current time 4000, " - "configured interval 300] (pri 200)", + EXPECT_EQ("[Needs garbage collection: Last check at 3, current time 4000, configured interval 300] (pri 200)", testGarbageCollection(3, 4000, 300, 1, true)); - EXPECT_EQ("NO OPERATIONS GENERATED", - testGarbageCollection(3850, 4000, 300, 1)); + EXPECT_EQ("NO OPERATIONS GENERATED", testGarbageCollection(3850, 4000, 300, 1)); } TEST_F(StateCheckersTest, gc_ops_are_prioritized_with_low_priority_category) { @@ -1597,11 +1587,10 @@ TEST_F(StateCheckersTest, context_populates_ideal_state_containers) { StateChecker::Context c(node_context(), operation_context(), getDistributorBucketSpace(), statsTracker, makeDocumentBucket({17, 0})); - ASSERT_THAT(c.idealState, ElementsAre(1, 3)); - // TODO replace with UnorderedElementsAre once we can build gmock without issues - std::vector<uint16_t> ideal_state(c.unorderedIdealState.begin(), c.unorderedIdealState.end()); - std::sort(ideal_state.begin(), ideal_state.end()); - ASSERT_THAT(ideal_state, ElementsAre(1, 3)); + ASSERT_THAT(c.idealState(), ElementsAre(1, 3)); + for (uint16_t node : c.idealState()) { + ASSERT_TRUE(c.idealStateBundle.is_nonretired_or_maintenance(node)); + } } namespace { @@ -1616,8 +1605,7 @@ public: explicit StateCheckerRunner(StateCheckersTest& fixture); ~StateCheckerRunner(); - StateCheckerRunner& addToDb(const document::BucketId& bid, - const std::string& bucketInfo) + StateCheckerRunner& addToDb(const document::BucketId& bid, const std::string& bucketInfo) { _fixture.addNodesToBucketDB(bid, bucketInfo); return *this; @@ -1652,8 +1640,7 @@ public: Checker checker; StateChecker::Context c(_fixture.node_context(), _fixture.operation_context(), _fixture.getDistributorBucketSpace(), _statsTracker, makeDocumentBucket(bid)); - _result = _fixture.testStateChecker( - checker, c, false, StateCheckersTest::PendingMessage(), false); + _result = _fixture.testStateChecker(checker, c, false, StateCheckersTest::PendingMessage(), false); } const std::string& result() const { return _result; } diff --git a/storage/src/tests/distributor/top_level_bucket_db_updater_test.cpp b/storage/src/tests/distributor/top_level_bucket_db_updater_test.cpp index 567e0a947da..7eb9dfe6269 100644 --- a/storage/src/tests/distributor/top_level_bucket_db_updater_test.cpp +++ b/storage/src/tests/distributor/top_level_bucket_db_updater_test.cpp @@ -65,12 +65,9 @@ public: close(); } - std::shared_ptr<RequestBucketInfoReply> make_fake_bucket_reply( - const lib::ClusterState& state, - const RequestBucketInfoCommand& cmd, - int storageIndex, - uint32_t bucketCount, - uint32_t invalidBucketCount = 0) + std::shared_ptr<RequestBucketInfoReply> + make_fake_bucket_reply(const lib::ClusterState& state, const RequestBucketInfoCommand& cmd, + int storageIndex, uint32_t bucketCount,uint32_t invalidBucketCount = 0) { auto sreply = std::make_shared<RequestBucketInfoReply>(cmd); sreply->setAddress(storage_address(storageIndex)); @@ -84,19 +81,14 @@ public: } std::vector<uint16_t> nodes; - distributor_bucket_space(bucket).getDistribution().getIdealNodes( - lib::NodeType::STORAGE, state, bucket, nodes); + distributor_bucket_space(bucket).getDistribution().getIdealNodes(lib::NodeType::STORAGE, state, bucket, nodes, "uim"); for (uint32_t j = 0; j < nodes.size(); ++j) { if (nodes[j] == storageIndex) { if (i >= bucketCount) { - vec.push_back(api::RequestBucketInfoReply::Entry( - document::BucketId(16, i), - api::BucketInfo())); + vec.emplace_back(document::BucketId(16, i), api::BucketInfo()); } else { - vec.push_back(api::RequestBucketInfoReply::Entry( - document::BucketId(16, i), - api::BucketInfo(10,1,1))); + vec.emplace_back(document::BucketId(16, i), api::BucketInfo(10,1,1)); } } } @@ -105,45 +97,34 @@ public: return sreply; } - void fake_bucket_reply(const lib::ClusterState &state, - const api::StorageCommand &cmd, - uint32_t bucket_count, - uint32_t invalid_bucket_count = 0) + void fake_bucket_reply(const lib::ClusterState &state, const api::StorageCommand &cmd, + uint32_t bucket_count, uint32_t invalid_bucket_count = 0) { ASSERT_EQ(cmd.getType(), MessageType::REQUESTBUCKETINFO); const api::StorageMessageAddress& address(*cmd.getAddress()); bucket_db_updater().onRequestBucketInfoReply( - make_fake_bucket_reply(state, - dynamic_cast<const RequestBucketInfoCommand &>(cmd), - address.getIndex(), - bucket_count, - invalid_bucket_count)); + make_fake_bucket_reply(state, dynamic_cast<const RequestBucketInfoCommand &>(cmd), + address.getIndex(), bucket_count, invalid_bucket_count)); } - void fake_bucket_reply(const lib::ClusterState &state, - const api::StorageCommand &cmd, - uint32_t bucket_count, + void fake_bucket_reply(const lib::ClusterState &state, const api::StorageCommand &cmd, uint32_t bucket_count, const std::function<void(api::RequestBucketInfoReply&)>& reply_decorator) { ASSERT_EQ(cmd.getType(), MessageType::REQUESTBUCKETINFO); const api::StorageMessageAddress& address(*cmd.getAddress()); - auto reply = make_fake_bucket_reply(state, - dynamic_cast<const RequestBucketInfoCommand &>(cmd), - address.getIndex(), - bucket_count, 0); + auto reply = make_fake_bucket_reply(state, dynamic_cast<const RequestBucketInfoCommand &>(cmd), + address.getIndex(), bucket_count, 0); reply_decorator(*reply); bucket_db_updater().onRequestBucketInfoReply(reply); } - void send_fake_reply_for_single_bucket_request( - const api::RequestBucketInfoCommand& rbi) + void send_fake_reply_for_single_bucket_request(const api::RequestBucketInfoCommand& rbi) { ASSERT_EQ(size_t(1), rbi.getBuckets().size()); const document::BucketId& bucket(rbi.getBuckets()[0]); auto reply = std::make_shared<api::RequestBucketInfoReply>(rbi); - reply->getBucketInfo().push_back( - api::RequestBucketInfoReply::Entry(bucket, api::BucketInfo(20, 10, 12, 50, 60, true, true))); + reply->getBucketInfo().emplace_back(bucket, api::BucketInfo(20, 10, 12, 50, 60, true, true)); stripe_of_bucket(bucket).bucket_db_updater().onRequestBucketInfoReply(reply); } @@ -154,15 +135,11 @@ public: } std::vector<uint16_t> nodes; - distributor_bucket_space(id).getDistribution().getIdealNodes( - lib::NodeType::STORAGE, state, document::BucketId(id), nodes); + distributor_bucket_space(id).getDistribution().getIdealNodes(lib::NodeType::STORAGE, state, document::BucketId(id), nodes, "uim"); if (nodes.size() != entry->getNodeCount()) { - return vespalib::make_string("Bucket Id %s has %d nodes in " - "ideal state, but has only %d in DB", - id.toString().c_str(), - (int)nodes.size(), - (int)entry->getNodeCount()); + return vespalib::make_string("Bucket Id %s has %d nodes in ideal state, but has only %d in DB", + id.toString().c_str(), (int)nodes.size(), (int)entry->getNodeCount()); } for (uint32_t i = 0; i<nodes.size(); i++) { @@ -175,10 +152,7 @@ public: } if (!found) { - return vespalib::make_string( - "Bucket Id %s has no copy from node %d", - id.toString().c_str(), - nodes[i]); + return vespalib::make_string("Bucket Id %s has no copy from node %d", id.toString().c_str(), nodes[i]); } } @@ -188,13 +162,11 @@ public: struct OrderByIncreasingNodeIndex { template <typename T> bool operator()(const T& lhs, const T& rhs) { - return (lhs->getAddress()->getIndex() - < rhs->getAddress()->getIndex()); + return (lhs->getAddress()->getIndex() < rhs->getAddress()->getIndex()); } }; - void sort_sent_messages_by_index(DistributorMessageSenderStub& sender, - size_t sortFromOffset = 0) + void sort_sent_messages_by_index(DistributorMessageSenderStub& sender, size_t sortFromOffset = 0) { std::sort(sender.commands().begin() + sortFromOffset, sender.commands().end(), diff --git a/storage/src/tests/distributor/top_level_distributor_test.cpp b/storage/src/tests/distributor/top_level_distributor_test.cpp index dad6f477d83..94f8821f9c8 100644 --- a/storage/src/tests/distributor/top_level_distributor_test.cpp +++ b/storage/src/tests/distributor/top_level_distributor_test.cpp @@ -92,7 +92,7 @@ struct TopLevelDistributorTest : Test, TopLevelDistributorTestUtil { return _distributor->getBucketSpacesStats(); } - std::unordered_map<uint16_t, uint32_t> distributor_min_replica_stats() { + MinReplicaMap distributor_min_replica_stats() { return _distributor->getMinReplica(); } @@ -504,7 +504,7 @@ void assert_invalid_bucket_stats_for_all_spaces( ASSERT_FALSE(space_iter->second.valid()); } -void assert_min_replica_stats_zeroed(const std::unordered_map<uint16_t, uint32_t>& stats, uint16_t node_index) { +void assert_min_replica_stats_zeroed(const MinReplicaMap & stats, uint16_t node_index) { auto iter = stats.find(node_index); ASSERT_TRUE(iter != stats.cend()); EXPECT_EQ(iter->second, 0); diff --git a/storage/src/tests/distributor/top_level_distributor_test_util.cpp b/storage/src/tests/distributor/top_level_distributor_test_util.cpp index 9677ea568e8..9859a6fb237 100644 --- a/storage/src/tests/distributor/top_level_distributor_test_util.cpp +++ b/storage/src/tests/distributor/top_level_distributor_test_util.cpp @@ -187,7 +187,7 @@ TopLevelDistributorTestUtil::get_ideal_str(document::BucketId id, const lib::Clu return id.toString(); } std::vector<uint16_t> nodes; - _component->getDistribution()->getIdealNodes(lib::NodeType::STORAGE, state, id, nodes); + _component->getDistribution()->getIdealNodes(lib::NodeType::STORAGE, state, id, nodes, "uim"); std::sort(nodes.begin(), nodes.end()); std::ostringstream ost; ost << id << ": " << dumpVector(nodes); @@ -205,14 +205,11 @@ TopLevelDistributorTestUtil::add_ideal_nodes(const lib::ClusterState& state, con std::vector<uint16_t> res; assert(_component.get()); - _component->getDistribution()->getIdealNodes(lib::NodeType::STORAGE, state, id, res); + _component->getDistribution()->getIdealNodes(lib::NodeType::STORAGE, state, id, res, "uim"); for (uint32_t i = 0; i < res.size(); ++i) { - if (state.getNodeState(lib::Node(lib::NodeType::STORAGE, res[i])).getState() != - lib::State::MAINTENANCE) - { - entry->addNode(BucketCopy(0, res[i], api::BucketInfo(1,1,1)), - toVector<uint16_t>(0)); + if (state.getNodeState(lib::Node(lib::NodeType::STORAGE, res[i])).getState() != lib::State::MAINTENANCE) { + entry->addNode(BucketCopy(0, res[i], api::BucketInfo(1,1,1)), toVector<uint16_t>(0)); } } diff --git a/storage/src/vespa/storage/bucketdb/bucketcopy.h b/storage/src/vespa/storage/bucketdb/bucketcopy.h index e8d1db1d824..ca629a6cd8e 100644 --- a/storage/src/vespa/storage/bucketdb/bucketcopy.h +++ b/storage/src/vespa/storage/bucketdb/bucketcopy.h @@ -7,10 +7,10 @@ namespace storage { class BucketCopy { private: - uint64_t _timestamp; + uint64_t _timestamp; api::BucketInfo _info; - uint16_t _flags; - uint16_t _node; + uint16_t _flags; + uint16_t _node; public: static const int TRUSTED = 1; @@ -18,9 +18,7 @@ public: BucketCopy() noexcept : _timestamp(0), _flags(0), _node(0xffff) {} - BucketCopy(uint64_t timestamp, - uint16_t nodeIdx, - const api::BucketInfo& info) noexcept + BucketCopy(uint64_t timestamp, uint16_t nodeIdx, const api::BucketInfo& info) noexcept : _timestamp(timestamp), _info(info), _flags(0), @@ -76,16 +74,14 @@ public: _info.setActive(setactive); } - bool consistentWith(const BucketCopy& other, - bool countInvalidAsConsistent = false) const noexcept - { + bool consistentWith(const BucketCopy& other) const noexcept { // If both are valid, check checksum and doc count. if (valid() && other.valid()) { return (getChecksum() == other.getChecksum() && getDocumentCount() == other.getDocumentCount()); } - return countInvalidAsConsistent; + return false; } void print(std::ostream&, bool verbose, const std::string& indent) const; @@ -93,9 +89,7 @@ public: std::string toString() const; bool operator==(const BucketCopy& other) const noexcept { - return - getBucketInfo() == other.getBucketInfo() && - _flags == other._flags; + return (getBucketInfo() == other.getBucketInfo()) && (_flags == other._flags); } }; diff --git a/storage/src/vespa/storage/bucketdb/bucketinfo.cpp b/storage/src/vespa/storage/bucketdb/bucketinfo.cpp index dcf49b4d022..a8c21efa793 100644 --- a/storage/src/vespa/storage/bucketdb/bucketinfo.cpp +++ b/storage/src/vespa/storage/bucketdb/bucketinfo.cpp @@ -9,9 +9,9 @@ namespace storage { template class BucketInfoBase<std::vector<BucketCopy>>; template class BucketInfoBase<vespalib::ConstArrayRef<BucketCopy>>; -BucketInfo::BucketInfo() : BucketInfoBase() {} +BucketInfo::BucketInfo() noexcept : BucketInfoBase() {} -BucketInfo::BucketInfo(uint32_t lastGarbageCollection, std::vector<BucketCopy> nodes) +BucketInfo::BucketInfo(uint32_t lastGarbageCollection, std::vector<BucketCopy> nodes) noexcept : BucketInfoBase(lastGarbageCollection, std::move(nodes)) {} @@ -23,7 +23,7 @@ BucketInfo::BucketInfo(BucketInfo&&) noexcept = default; BucketInfo& BucketInfo::operator=(BucketInfo&&) noexcept = default; void -BucketInfo::updateTrusted() { +BucketInfo::updateTrusted() noexcept { if (validAndConsistent()) { for (uint32_t i = 0; i < _nodes.size(); i++) { _nodes[i].setTrusted(); @@ -51,7 +51,7 @@ BucketInfo::updateTrusted() { } void -BucketInfo::resetTrusted() { +BucketInfo::resetTrusted() noexcept { for (uint32_t i = 0; i < _nodes.size(); i++) { _nodes[i].clearTrusted(); } @@ -63,10 +63,10 @@ namespace { struct Sorter { const std::vector<uint16_t>& _order; - Sorter(const std::vector<uint16_t>& recommendedOrder) : + Sorter(const std::vector<uint16_t>& recommendedOrder) noexcept : _order(recommendedOrder) {} - bool operator() (const BucketCopy& a, const BucketCopy& b) { + bool operator() (const BucketCopy& a, const BucketCopy& b) noexcept { int order_a = -1; for (uint32_t i = 0; i < _order.size(); i++) { if (_order[i] == a.getNode()) { @@ -119,8 +119,7 @@ BucketInfo::addNodes(const std::vector<BucketCopy>& newCopies, if (found) { if (found->getTimestamp() < newCopies[i].getTimestamp()) { - found->setBucketInfo(newCopies[i].getTimestamp(), - newCopies[i].getBucketInfo()); + found->setBucketInfo(newCopies[i].getTimestamp(), newCopies[i].getBucketInfo()); } } else { _nodes.push_back(newCopies[i]); @@ -135,19 +134,15 @@ BucketInfo::addNodes(const std::vector<BucketCopy>& newCopies, } void -BucketInfo::addNode(const BucketCopy& newCopy, - const std::vector<uint16_t>& recommendedOrder) +BucketInfo::addNode(const BucketCopy& newCopy, const std::vector<uint16_t>& recommendedOrder) { - addNodes(toVector<BucketCopy>(newCopy), - recommendedOrder); + addNodes(toVector<BucketCopy>(newCopy), recommendedOrder); } bool BucketInfo::removeNode(unsigned short node, TrustedUpdate update) { - for (std::vector<BucketCopy>::iterator iter = _nodes.begin(); - iter != _nodes.end(); - iter++) { + for (auto iter = _nodes.begin(); iter != _nodes.end(); iter++) { if (iter->getNode() == node) { _nodes.erase(iter); if (update == TrustedUpdate::UPDATE) { @@ -162,11 +157,9 @@ BucketInfo::removeNode(unsigned short node, TrustedUpdate update) BucketCopy* BucketInfo::getNodeInternal(uint16_t node) { - for (std::vector<BucketCopy>::iterator iter = _nodes.begin(); - iter != _nodes.end(); - iter++) { - if (iter->getNode() == node) { - return &*iter; + for (BucketCopy & copy : _nodes) { + if (copy.getNode() == node) { + return © } } return 0; diff --git a/storage/src/vespa/storage/bucketdb/bucketinfo.h b/storage/src/vespa/storage/bucketdb/bucketinfo.h index 1870d4c91d4..219d0335966 100644 --- a/storage/src/vespa/storage/bucketdb/bucketinfo.h +++ b/storage/src/vespa/storage/bucketdb/bucketinfo.h @@ -25,15 +25,15 @@ protected: uint32_t _lastGarbageCollection; NodeSeq _nodes; public: - BucketInfoBase() + BucketInfoBase() noexcept : _lastGarbageCollection(0), _nodes() {} - BucketInfoBase(uint32_t lastGarbageCollection, const NodeSeq& nodes) + BucketInfoBase(uint32_t lastGarbageCollection, const NodeSeq& nodes) noexcept : _lastGarbageCollection(lastGarbageCollection), _nodes(nodes) {} - BucketInfoBase(uint32_t lastGarbageCollection, NodeSeq&& nodes) + BucketInfoBase(uint32_t lastGarbageCollection, NodeSeq&& nodes) noexcept : _lastGarbageCollection(lastGarbageCollection), _nodes(std::move(nodes)) {} @@ -47,28 +47,28 @@ public: /** * @return Returns the last time when this bucket was "garbage collected". */ - uint32_t getLastGarbageCollectionTime() const { return _lastGarbageCollection; } + uint32_t getLastGarbageCollectionTime() const noexcept { return _lastGarbageCollection; } /** True if the bucket contains no documents and is consistent. */ - bool emptyAndConsistent() const; + bool emptyAndConsistent() const noexcept; /** Check that all copies have complete bucket information and are consistent with eachother. */ - bool validAndConsistent() const; + bool validAndConsistent() const noexcept; /** * True if the bucket contains at least one invalid copy */ - bool hasInvalidCopy() const; + bool hasInvalidCopy() const noexcept; /** * Returns the number of trusted nodes this entry has. */ - uint16_t getTrustedCount() const; + uint16_t getTrustedCount() const noexcept; - bool hasTrusted() const { + bool hasTrusted() const noexcept { return getTrustedCount() != 0; } @@ -78,14 +78,14 @@ public: * @param countInCompleteAsInconsistent If false, nodes that are incomplete * are always counted as consistent with complete nodes. */ - bool consistentNodes(bool countInvalidAsConsistent = false) const; + bool consistentNodes() const noexcept; void print(std::ostream&, bool verbose, const std::string& indent) const; /** * Returns the bucket copy struct for the given node, null if nonexisting */ - const BucketCopy* getNode(uint16_t node) const; + const BucketCopy* getNode(uint16_t node) const noexcept; /** * Returns the number of nodes this entry has. @@ -95,14 +95,14 @@ public: /** * Returns a list of the nodes this entry has. */ - std::vector<uint16_t> getNodes() const; + std::vector<uint16_t> getNodes() const noexcept; /** Returns a reference to the node with the given index in the node array. This operation has undefined behaviour if the index given is not within the node count. */ - const BucketCopy& getNodeRef(uint16_t idx) const { + const BucketCopy& getNodeRef(uint16_t idx) const noexcept { return _nodes[idx]; } @@ -117,14 +117,14 @@ public: std::string toString() const; - uint32_t getHighestDocumentCount() const; - uint32_t getHighestTotalDocumentSize() const; - uint32_t getHighestMetaCount() const; - uint32_t getHighestUsedFileSize() const; + uint32_t getHighestDocumentCount() const noexcept; + uint32_t getHighestTotalDocumentSize() const noexcept; + uint32_t getHighestMetaCount() const noexcept; + uint32_t getHighestUsedFileSize() const noexcept; - bool hasRecentlyCreatedEmptyCopy() const; + bool hasRecentlyCreatedEmptyCopy() const noexcept; - bool operator==(const BucketInfoBase& other) const; + bool operator==(const BucketInfoBase& other) const noexcept; }; template <typename NodeSeq> @@ -140,8 +140,8 @@ public: class BucketInfo : public BucketInfoBase<std::vector<BucketCopy>> { public: - BucketInfo(); - BucketInfo(uint32_t lastGarbageCollection, std::vector<BucketCopy> nodes); + BucketInfo() noexcept; + BucketInfo(uint32_t lastGarbageCollection, std::vector<BucketCopy> nodes) noexcept; ~BucketInfo(); BucketInfo(const BucketInfo&); @@ -152,20 +152,20 @@ public: /** * Sets the last time the bucket was "garbage collected". */ - void setLastGarbageCollectionTime(uint32_t timestamp) { + void setLastGarbageCollectionTime(uint32_t timestamp) noexcept { _lastGarbageCollection = timestamp; } /** Update trusted flags if bucket is now complete and consistent. */ - void updateTrusted(); + void updateTrusted() noexcept; /** Removes any historical information on trustedness, and sets the bucket copies to trusted if they are now complete and consistent. */ - void resetTrusted(); + void resetTrusted() noexcept; /** Adds the given node. @@ -184,8 +184,7 @@ public: /** Simplified API for the common case of inserting one node. See addNodes(). */ - void addNode(const BucketCopy& newCopy, - const std::vector<uint16_t>& recommendedOrder); + void addNode(const BucketCopy& newCopy, const std::vector<uint16_t>& recommendedOrder); /** Updates bucket information for a node. Does nothing if the node diff --git a/storage/src/vespa/storage/bucketdb/bucketinfo.hpp b/storage/src/vespa/storage/bucketdb/bucketinfo.hpp index b7e8c5925c5..ce7adc8af67 100644 --- a/storage/src/vespa/storage/bucketdb/bucketinfo.hpp +++ b/storage/src/vespa/storage/bucketdb/bucketinfo.hpp @@ -9,16 +9,18 @@ namespace storage { template <typename NodeSeq> -std::string BucketInfoBase<NodeSeq>::toString() const { +std::string +BucketInfoBase<NodeSeq>::toString() const { std::ostringstream ost; print(ost, true, ""); return ost.str(); } template <typename NodeSeq> -bool BucketInfoBase<NodeSeq>::emptyAndConsistent() const { - for (uint32_t i = 0; i < _nodes.size(); i++) { - if (!_nodes[i].empty()) { +bool +BucketInfoBase<NodeSeq>::emptyAndConsistent() const noexcept { + for (const auto & n : _nodes) { + if (!n.empty()) { return false; } } @@ -26,9 +28,10 @@ bool BucketInfoBase<NodeSeq>::emptyAndConsistent() const { } template <typename NodeSeq> -bool BucketInfoBase<NodeSeq>::validAndConsistent() const { - for (uint32_t i = 0; i < _nodes.size(); i++) { - if (!_nodes[i].valid()) { +bool +BucketInfoBase<NodeSeq>::validAndConsistent() const noexcept { + for (const auto & n : _nodes) { + if (!n.valid()) { return false; } } @@ -36,9 +39,10 @@ bool BucketInfoBase<NodeSeq>::validAndConsistent() const { } template <typename NodeSeq> -bool BucketInfoBase<NodeSeq>::hasInvalidCopy() const { - for (uint32_t i = 0; i < _nodes.size(); i++) { - if (!_nodes[i].valid()) { +bool +BucketInfoBase<NodeSeq>::hasInvalidCopy() const noexcept { + for (const auto & n : _nodes){ + if (!n.valid()) { return true; } } @@ -46,10 +50,11 @@ bool BucketInfoBase<NodeSeq>::hasInvalidCopy() const { } template <typename NodeSeq> -uint16_t BucketInfoBase<NodeSeq>::getTrustedCount() const { +uint16_t +BucketInfoBase<NodeSeq>::getTrustedCount() const noexcept { uint32_t trustedCount = 0; - for (uint32_t i = 0; i < _nodes.size(); i++) { - if (_nodes[i].trusted()) { + for (const auto & n : _nodes) { + if (n.trusted()) { trustedCount++; } } @@ -57,11 +62,11 @@ uint16_t BucketInfoBase<NodeSeq>::getTrustedCount() const { } template <typename NodeSeq> -bool BucketInfoBase<NodeSeq>::consistentNodes(bool countInvalidAsConsistent) const { +bool +BucketInfoBase<NodeSeq>::consistentNodes() const noexcept { int compareIndex = 0; for (uint32_t i = 1; i < _nodes.size(); i++) { - if (!_nodes[i].consistentWith(_nodes[compareIndex], - countInvalidAsConsistent)) return false; + if (!_nodes[i].consistentWith(_nodes[compareIndex])) return false; } return true; } @@ -90,14 +95,16 @@ struct ReplicaMetadata { }; }; -constexpr bool is_majority(size_t n, size_t m) { +constexpr bool +is_majority(size_t n, size_t m) noexcept { return (n >= (m / 2) + 1); } } template <typename NodeSeq> -api::BucketInfo BucketInfoBase<NodeSeq>::majority_consistent_bucket_info() const noexcept { +api::BucketInfo +BucketInfoBase<NodeSeq>::majority_consistent_bucket_info() const noexcept { if (_nodes.size() < 3) { return {}; } @@ -116,7 +123,8 @@ api::BucketInfo BucketInfoBase<NodeSeq>::majority_consistent_bucket_info() const } template <typename NodeSeq> -void BucketInfoBase<NodeSeq>::print(std::ostream& out, bool verbose, const std::string& indent) const { +void +BucketInfoBase<NodeSeq>::print(std::ostream& out, bool verbose, const std::string& indent) const { if (_nodes.size() == 0) { out << "no nodes"; } @@ -129,7 +137,8 @@ void BucketInfoBase<NodeSeq>::print(std::ostream& out, bool verbose, const std:: } template <typename NodeSeq> -const BucketCopy* BucketInfoBase<NodeSeq>::getNode(uint16_t node) const { +const BucketCopy* +BucketInfoBase<NodeSeq>::getNode(uint16_t node) const noexcept { for (const auto& n : _nodes) { if (n.getNode() == node) { return &n; @@ -139,54 +148,61 @@ const BucketCopy* BucketInfoBase<NodeSeq>::getNode(uint16_t node) const { } template <typename NodeSeq> -std::vector<uint16_t> BucketInfoBase<NodeSeq>::getNodes() const { +std::vector<uint16_t> +BucketInfoBase<NodeSeq>::getNodes() const noexcept { std::vector<uint16_t> result; - for (uint32_t i = 0; i < _nodes.size(); i++) { - result.emplace_back(_nodes[i].getNode()); + result.reserve(_nodes.size()); + for (const auto & n : _nodes) { + result.emplace_back(n.getNode()); } return result; } template <typename NodeSeq> -uint32_t BucketInfoBase<NodeSeq>::getHighestDocumentCount() const { +uint32_t +BucketInfoBase<NodeSeq>::getHighestDocumentCount() const noexcept { uint32_t highest = 0; - for (uint32_t i = 0; i < _nodes.size(); ++i) { - highest = std::max(highest, _nodes[i].getDocumentCount()); + for (const auto & n : _nodes) { + highest = std::max(highest, n.getDocumentCount()); } return highest; } template <typename NodeSeq> -uint32_t BucketInfoBase<NodeSeq>::getHighestTotalDocumentSize() const { +uint32_t +BucketInfoBase<NodeSeq>::getHighestTotalDocumentSize() const noexcept { uint32_t highest = 0; - for (uint32_t i = 0; i < _nodes.size(); ++i) { - highest = std::max(highest, _nodes[i].getTotalDocumentSize()); + for (const auto & n : _nodes) { + highest = std::max(highest, n.getTotalDocumentSize()); } return highest; } template <typename NodeSeq> -uint32_t BucketInfoBase<NodeSeq>::getHighestMetaCount() const { +uint32_t +BucketInfoBase<NodeSeq>::getHighestMetaCount() const noexcept { uint32_t highest = 0; - for (uint32_t i = 0; i < _nodes.size(); ++i) { - highest = std::max(highest, _nodes[i].getMetaCount()); + for (const auto & n : _nodes) { + highest = std::max(highest, n.getMetaCount()); } return highest; } template <typename NodeSeq> -uint32_t BucketInfoBase<NodeSeq>::getHighestUsedFileSize() const { +uint32_t +BucketInfoBase<NodeSeq>::getHighestUsedFileSize() const noexcept { uint32_t highest = 0; - for (uint32_t i = 0; i < _nodes.size(); ++i) { - highest = std::max(highest, _nodes[i].getUsedFileSize()); + for (const auto & n : _nodes) { + highest = std::max(highest, n.getUsedFileSize()); } return highest; } template <typename NodeSeq> -bool BucketInfoBase<NodeSeq>::hasRecentlyCreatedEmptyCopy() const { - for (uint32_t i = 0; i < _nodes.size(); ++i) { - if (_nodes[i].wasRecentlyCreated()) { +bool +BucketInfoBase<NodeSeq>::hasRecentlyCreatedEmptyCopy() const noexcept { + for (const auto & n : _nodes) { + if (n.wasRecentlyCreated()) { return true; } } @@ -194,7 +210,8 @@ bool BucketInfoBase<NodeSeq>::hasRecentlyCreatedEmptyCopy() const { } template <typename NodeSeq> -bool BucketInfoBase<NodeSeq>::operator==(const BucketInfoBase<NodeSeq>& other) const { +bool +BucketInfoBase<NodeSeq>::operator==(const BucketInfoBase<NodeSeq>& other) const noexcept { if (_nodes.size() != other._nodes.size()) { return false; } @@ -210,6 +227,6 @@ bool BucketInfoBase<NodeSeq>::operator==(const BucketInfoBase<NodeSeq>& other) c } return true; -}; +} } diff --git a/storage/src/vespa/storage/config/stor-distributormanager.def b/storage/src/vespa/storage/config/stor-distributormanager.def index 9c8469f924c..95461eb5dc2 100644 --- a/storage/src/vespa/storage/config/stor-distributormanager.def +++ b/storage/src/vespa/storage/config/stor-distributormanager.def @@ -312,4 +312,4 @@ enable_two_phase_garbage_collection bool default=true ## If true, a conditional Put or Remove operation received for a bucket with inconsistent ## replicas will trigger an implicit distributed condition probe to resolve the outcome of ## the condition across all divergent replicas. -enable_condition_probing bool default=false +enable_condition_probing bool default=true diff --git a/storage/src/vespa/storage/distributor/activecopy.cpp b/storage/src/vespa/storage/distributor/activecopy.cpp index a4ee4a51135..5d59d1a838f 100644 --- a/storage/src/vespa/storage/distributor/activecopy.cpp +++ b/storage/src/vespa/storage/distributor/activecopy.cpp @@ -9,24 +9,28 @@ #include <cassert> namespace std { - template<typename T> - std::ostream& operator<<(std::ostream& out, const std::vector<T>& v) { - out << "["; - for (uint32_t i=0; i<v.size(); ++i) { - out << "\n " << v[i]; - } - if (!v.empty()) { - out << "\n"; - } - return out << "]"; + +template<typename T> +std::ostream& operator<<(std::ostream& out, const std::vector<T>& v) { + out << "["; + for (uint32_t i=0; i<v.size(); ++i) { + out << "\n " << v[i]; + } + if (!v.empty()) { + out << "\n"; } + return out << "]"; +} + } namespace storage::distributor { -ActiveCopy::ActiveCopy(uint16_t node, const BucketDatabase::Entry& e, const std::vector<uint16_t>& idealState) : - _nodeIndex(node), - _ideal(0xffff) +using IndexList = lib::Distribution::IndexList; + +ActiveCopy::ActiveCopy(uint16_t node, const BucketDatabase::Entry& e, const std::vector<uint16_t>& idealState) + : _nodeIndex(node), + _ideal(0xffff) { const BucketCopy* copy = e->getNode(node); assert(copy != nullptr); @@ -91,66 +95,61 @@ operator<<(std::ostream& out, const ActiveCopy & e) { namespace { - struct ActiveStateOrder { - bool operator()(const ActiveCopy & e1, const ActiveCopy & e2) { - if (e1._ready != e2._ready) { - return e1._ready; - } - if (e1._doc_count != e2._doc_count) { - return e1._doc_count > e2._doc_count; - } - if (e1._ideal != e2._ideal) { - return e1._ideal < e2._ideal; - } - if (e1._active != e2._active) { - return e1._active; - } - return e1._nodeIndex < e2._nodeIndex; +struct ActiveStateOrder { + bool operator()(const ActiveCopy & e1, const ActiveCopy & e2) noexcept { + if (e1._ready != e2._ready) { + return e1._ready; } - }; - - std::vector<uint16_t> - buildValidNodeIndexList(BucketDatabase::Entry& e) { - std::vector<uint16_t> result; - result.reserve(e->getNodeCount()); - for (uint32_t i=0, n=e->getNodeCount(); i < n; ++i) { - const BucketCopy& cp = e->getNodeRef(i); - if (!cp.valid()) { - continue; - } + if (e1._doc_count != e2._doc_count) { + return e1._doc_count > e2._doc_count; + } + if (e1._ideal != e2._ideal) { + return e1._ideal < e2._ideal; + } + if (e1._active != e2._active) { + return e1._active; + } + return e1._nodeIndex < e2._nodeIndex; + } +}; + +IndexList +buildValidNodeIndexList(BucketDatabase::Entry& e) { + IndexList result; + result.reserve(e->getNodeCount()); + for (uint32_t i=0, n=e->getNodeCount(); i < n; ++i) { + const BucketCopy& cp = e->getNodeRef(i); + if (cp.valid()) { result.push_back(cp.getNode()); } - return result; } + return result; +} - std::vector<ActiveCopy> - buildNodeList(BucketDatabase::Entry& e, - const std::vector<uint16_t>& nodeIndexes, - const std::vector<uint16_t>& idealState) - { - std::vector<ActiveCopy> result; - result.reserve(nodeIndexes.size()); - for (uint16_t nodeIndex : nodeIndexes) { - result.emplace_back(nodeIndex, e, idealState); - } - return result; +std::vector<ActiveCopy> +buildNodeList(BucketDatabase::Entry& e,vespalib::ConstArrayRef<uint16_t> nodeIndexes, const std::vector<uint16_t>& idealState) +{ + std::vector<ActiveCopy> result; + result.reserve(nodeIndexes.size()); + for (uint16_t nodeIndex : nodeIndexes) { + result.emplace_back(nodeIndex, e, idealState); } + return result; +} + } ActiveList -ActiveCopy::calculate(const std::vector<uint16_t>& idealState, - const lib::Distribution& distribution, - BucketDatabase::Entry& e, - uint32_t max_activation_inhibited_out_of_sync_groups) +ActiveCopy::calculate(const std::vector<uint16_t>& idealState, const lib::Distribution& distribution, + BucketDatabase::Entry& e, uint32_t max_activation_inhibited_out_of_sync_groups) { - std::vector<uint16_t> validNodesWithCopy = buildValidNodeIndexList(e); + IndexList validNodesWithCopy = buildValidNodeIndexList(e); if (validNodesWithCopy.empty()) { return ActiveList(); } - using IndexList = std::vector<uint16_t>; std::vector<IndexList> groups; if (distribution.activePerGroup()) { - groups = distribution.splitNodesIntoLeafGroups(std::move(validNodesWithCopy)); + groups = distribution.splitNodesIntoLeafGroups(validNodesWithCopy); } else { groups.push_back(std::move(validNodesWithCopy)); } diff --git a/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.cpp b/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.cpp index fc6c957b737..dfcbbf63946 100644 --- a/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.cpp +++ b/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.cpp @@ -6,19 +6,26 @@ namespace storage::distributor { -BucketDBMetricUpdater::Stats::Stats() +BucketDBMetricUpdater::Stats::Stats() noexcept : _docCount(0), _byteCount(0), _tooFewCopies(0), _tooManyCopies(0), _noTrusted(0), - _totalBuckets(0) + _totalBuckets(0), + _mutable_db_mem_usage(), + _read_only_db_mem_usage(), + _minBucketReplica() { } BucketDBMetricUpdater::Stats::Stats(const Stats &rhs) = default; +BucketDBMetricUpdater::Stats & BucketDBMetricUpdater::Stats::operator=(const Stats &rhs) = default; +BucketDBMetricUpdater::Stats::Stats(Stats &&rhs) noexcept = default; +BucketDBMetricUpdater::Stats & BucketDBMetricUpdater::Stats::operator=(Stats &&rhs) noexcept = default; +BucketDBMetricUpdater::Stats::~Stats() = default; -BucketDBMetricUpdater::BucketDBMetricUpdater() +BucketDBMetricUpdater::BucketDBMetricUpdater() noexcept : _workingStats(), _lastCompleteStats(), _replicaCountingMode(ReplicaCountingMode::TRUSTED), @@ -35,8 +42,7 @@ BucketDBMetricUpdater::resetStats() } void -BucketDBMetricUpdater::visit(const BucketDatabase::Entry& entry, - uint32_t redundancy) +BucketDBMetricUpdater::visit(const BucketDatabase::Entry& entry, uint32_t redundancy) { if (entry->getNodeCount() == 0) { // We used to have an assert on >0 but that caused some crashes, see @@ -90,9 +96,7 @@ BucketDBMetricUpdater::visit(const BucketDatabase::Entry& entry, } void -BucketDBMetricUpdater::updateMinReplicationStats( - const BucketDatabase::Entry& entry, - uint32_t trustedCopies) +BucketDBMetricUpdater::updateMinReplicationStats(const BucketDatabase::Entry& entry, uint32_t trustedCopies) { auto& minBucketReplica = _workingStats._minBucketReplica; for (uint32_t i = 0; i < entry->getNodeCount(); i++) { @@ -103,9 +107,9 @@ BucketDBMetricUpdater::updateMinReplicationStats( // sync across each other. // Regardless of counting mode we still have to take the minimum // replica count across all buckets present on any given node. - const uint32_t countedReplicas( - (_replicaCountingMode == ReplicaCountingMode::TRUSTED) - ? trustedCopies : entry->getNodeCount()); + const uint32_t countedReplicas = (_replicaCountingMode == ReplicaCountingMode::TRUSTED) + ? trustedCopies + : entry->getNodeCount(); auto it = minBucketReplica.find(node); if (it == minBucketReplica.end()) { minBucketReplica[node] = countedReplicas; @@ -118,17 +122,18 @@ BucketDBMetricUpdater::updateMinReplicationStats( void BucketDBMetricUpdater::completeRound(bool resetWorkingStats) { - _lastCompleteStats = _workingStats; + _hasCompleteStats = true; if (resetWorkingStats) { + _lastCompleteStats = std::move(_workingStats); resetStats(); + } else { + _lastCompleteStats = _workingStats; } } void -BucketDBMetricUpdater::Stats::propagateMetrics( - IdealStateMetricSet& idealStateMetrics, - DistributorMetricSet& distributorMetrics) +BucketDBMetricUpdater::Stats::propagateMetrics(IdealStateMetricSet& idealStateMetrics, DistributorMetricSet& distributorMetrics) const { distributorMetrics.docsStored.set(_docCount); distributorMetrics.bytesStored.set(_byteCount); diff --git a/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.h b/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.h index 2edb86cbaa2..366c2f2dc41 100644 --- a/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.h +++ b/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.h @@ -2,10 +2,11 @@ #pragma once +#include <vespa/storage/distributor/min_replica_provider.h> #include <vespa/storage/bucketdb/bucketdatabase.h> #include <vespa/storage/config/config-stor-distributormanager.h> #include <vespa/vespalib/util/memoryusage.h> -#include <unordered_map> +#include <vespa/vespalib/stllike/hash_map.h> namespace storage::distributor { @@ -25,11 +26,12 @@ public: vespalib::MemoryUsage _mutable_db_mem_usage; vespalib::MemoryUsage _read_only_db_mem_usage; - Stats(); + Stats() noexcept; + Stats(Stats &&rhs) noexcept; + Stats & operator=(Stats &&rhs) noexcept; Stats(const Stats &rhs); - ~Stats() = default; - - Stats &operator=(const Stats &rhs) = default; + Stats & operator=(const Stats &rhs); + ~Stats(); /** * For each node N, look at all the buckets that have or should have a @@ -47,24 +49,24 @@ public: * Note: If no buckets have been found for a node, that node is not in * this map. */ - std::unordered_map<uint16_t, uint32_t> _minBucketReplica; + MinReplicaMap _minBucketReplica; /** * Propagate state values to the appropriate metric values. */ - void propagateMetrics(IdealStateMetricSet&, DistributorMetricSet&); + void propagateMetrics(IdealStateMetricSet&, DistributorMetricSet&) const; }; using ReplicaCountingMode = vespa::config::content::core::StorDistributormanagerConfig::MinimumReplicaCountingMode; private: - Stats _workingStats; - Stats _lastCompleteStats; + Stats _workingStats; + Stats _lastCompleteStats; ReplicaCountingMode _replicaCountingMode; - bool _hasCompleteStats; + bool _hasCompleteStats; public: - BucketDBMetricUpdater(); + BucketDBMetricUpdater() noexcept; ~BucketDBMetricUpdater(); void setMinimumReplicaCountingMode(ReplicaCountingMode mode) noexcept { @@ -91,11 +93,11 @@ public: /** * Returns true iff completeRound() has been called at least once. */ - bool hasCompletedRound() const { + bool hasCompletedRound() const noexcept { return _hasCompleteStats; } - Stats getLastCompleteStats() const { + const Stats & getLastCompleteStats() const noexcept { return _lastCompleteStats; } diff --git a/storage/src/vespa/storage/distributor/distributor_bucket_space.cpp b/storage/src/vespa/storage/distributor/distributor_bucket_space.cpp index 5969ccad4cb..299aaffb569 100644 --- a/storage/src/vespa/storage/distributor/distributor_bucket_space.cpp +++ b/storage/src/vespa/storage/distributor/distributor_bucket_space.cpp @@ -108,9 +108,7 @@ DistributorBucketSpace::owns_bucket_in_state( } bool -DistributorBucketSpace::owns_bucket_in_state( - const lib::ClusterState& clusterState, - document::BucketId bucket) const +DistributorBucketSpace::owns_bucket_in_state(const lib::ClusterState& clusterState, document::BucketId bucket) const { return owns_bucket_in_state(*_distribution, clusterState, bucket); } @@ -152,16 +150,16 @@ DistributorBucketSpace::get_ideal_service_layer_nodes_bundle(document::BucketId setup_ideal_nodes_bundle(ideal_nodes_bundle, *_distribution, *_clusterState, bucket); return ideal_nodes_bundle; } - document::BucketId lookup_bucket(is_split_group_bucket(bucket) ? bucket.getUsedBits() : _distribution_bits, bucket.getId()); + document::BucketId lookup_bucket(_distribution_bits, bucket.getId()); auto itr = _ideal_nodes.find(lookup_bucket); if (itr != _ideal_nodes.end()) { - return itr->second; + return *itr->second; } - IdealServiceLayerNodesBundle ideal_nodes_bundle; - setup_ideal_nodes_bundle(ideal_nodes_bundle, *_distribution, *_clusterState, lookup_bucket); + auto ideal_nodes_bundle = std::make_unique<IdealServiceLayerNodesBundle>(); + setup_ideal_nodes_bundle(*ideal_nodes_bundle, *_distribution, *_clusterState, lookup_bucket); auto insres = _ideal_nodes.insert(std::make_pair(lookup_bucket, std::move(ideal_nodes_bundle))); assert(insres.second); - return insres.first->second; + return *insres.first->second; } BucketOwnershipFlags diff --git a/storage/src/vespa/storage/distributor/distributor_bucket_space.h b/storage/src/vespa/storage/distributor/distributor_bucket_space.h index f38556a664c..a66f0e5e983 100644 --- a/storage/src/vespa/storage/distributor/distributor_bucket_space.h +++ b/storage/src/vespa/storage/distributor/distributor_bucket_space.h @@ -41,7 +41,7 @@ class DistributorBucketSpace { std::shared_ptr<const lib::ClusterState> _pending_cluster_state; std::vector<bool> _available_nodes; mutable vespalib::hash_map<document::BucketId, BucketOwnershipFlags, document::BucketId::hash> _ownerships; - mutable vespalib::hash_map<document::BucketId, IdealServiceLayerNodesBundle, document::BucketId::hash> _ideal_nodes; + mutable vespalib::hash_map<document::BucketId, std::unique_ptr<IdealServiceLayerNodesBundle>, document::BucketId::hash> _ideal_nodes; void clear(); void enumerate_available_nodes(); diff --git a/storage/src/vespa/storage/distributor/distributor_host_info_reporter.cpp b/storage/src/vespa/storage/distributor/distributor_host_info_reporter.cpp index bb7e573c980..46c9a526a8d 100644 --- a/storage/src/vespa/storage/distributor/distributor_host_info_reporter.cpp +++ b/storage/src/vespa/storage/distributor/distributor_host_info_reporter.cpp @@ -3,15 +3,9 @@ #include "bucket_spaces_stats_provider.h" #include "distributor_host_info_reporter.h" #include "min_replica_provider.h" -#include "pendingmessagetracker.h" - #include <set> -using std::set; -using std::unordered_map; - -namespace storage { -namespace distributor { +namespace storage::distributor { using BucketSpacesStats = BucketSpacesStatsProvider::BucketSpacesStats; using PerNodeBucketSpacesStats = BucketSpacesStatsProvider::PerNodeBucketSpacesStats; @@ -48,10 +42,10 @@ writeBucketSpacesStats(vespalib::JsonStream& stream, void outputStorageNodes(vespalib::JsonStream& output, - const unordered_map<uint16_t, uint32_t>& minReplica, + const MinReplicaMap & minReplica, const PerNodeBucketSpacesStats& bucketSpacesStats) { - set<uint16_t> nodes; + std::set<uint16_t> nodes; for (const auto& element : minReplica) { nodes.insert(element.first); } @@ -104,6 +98,5 @@ DistributorHostInfoReporter::report(vespalib::JsonStream& output) output << End(); } -} // distributor -} // storage +} diff --git a/storage/src/vespa/storage/distributor/distributor_stripe.cpp b/storage/src/vespa/storage/distributor/distributor_stripe.cpp index 616fd77fdd7..37d81f45ac1 100644 --- a/storage/src/vespa/storage/distributor/distributor_stripe.cpp +++ b/storage/src/vespa/storage/distributor/distributor_stripe.cpp @@ -120,8 +120,7 @@ DistributorStripe::sendReply(const std::shared_ptr<api::StorageReply>& reply) } void DistributorStripe::send_shutdown_abort_reply(const std::shared_ptr<api::StorageMessage>& msg) { - api::StorageReply::UP reply( - std::dynamic_pointer_cast<api::StorageCommand>(msg)->makeReply()); + auto reply = std::dynamic_pointer_cast<api::StorageCommand>(msg)->makeReply(); reply->setResult(api::ReturnCode(api::ReturnCode::ABORTED, "Distributor is shutting down")); send_up_with_tracking(std::shared_ptr<api::StorageMessage>(reply.release())); } @@ -179,8 +178,7 @@ DistributorStripe::handle_or_enqueue_message(const std::shared_ptr<api::StorageM } void -DistributorStripe::handleCompletedMerge( - const std::shared_ptr<api::MergeBucketReply>& reply) +DistributorStripe::handleCompletedMerge(const std::shared_ptr<api::MergeBucketReply>& reply) { _maintenanceOperationOwner.handleReply(reply); } @@ -236,9 +234,7 @@ DistributorStripe::handleReply(const std::shared_ptr<api::StorageReply>& reply) } bool -DistributorStripe::generateOperation( - const std::shared_ptr<api::StorageMessage>& msg, - Operation::SP& operation) +DistributorStripe::generateOperation(const std::shared_ptr<api::StorageMessage>& msg, Operation::SP& operation) { return _externalOperationHandler.handleMessage(msg, operation); } @@ -277,7 +273,6 @@ DistributorStripe::getClusterStateBundle() const void DistributorStripe::enableClusterStateBundle(const lib::ClusterStateBundle& state) { - lib::Node my_node(lib::NodeType::DISTRIBUTOR, getDistributorIndex()); lib::ClusterStateBundle oldState = _clusterStateBundle; _clusterStateBundle = state; propagateClusterStates(); @@ -319,7 +314,7 @@ DistributorStripe::enterRecoveryMode() { LOG(debug, "Entering recovery mode"); _schedulingMode = MaintenanceScheduler::RECOVERY_SCHEDULING_MODE; - _scanner->reset(); + _scanner->reset(); // Just drop accumulated stat on the floor. // We enter recovery mode due to cluster state or distribution config changes. // Until we have completed a new DB scan round, we don't know the state of our // newly owned buckets and must not report stats for these out to the cluster @@ -415,7 +410,6 @@ public: bool check(uint32_t msgType, uint16_t node, uint8_t pri) override { (void) node; - (void) pri; if (msgType == api::MessageType::SPLITBUCKET_ID && pri <= maxPri) { found = true; return false; @@ -428,9 +422,7 @@ public: } void -DistributorStripe::checkBucketForSplit(document::BucketSpace bucketSpace, - const BucketDatabase::Entry& e, - uint8_t priority) +DistributorStripe::checkBucketForSplit(document::BucketSpace bucketSpace, const BucketDatabase::Entry& e, uint8_t priority) { if (!getConfig().doInlineSplit()) { return; @@ -440,16 +432,13 @@ DistributorStripe::checkBucketForSplit(document::BucketSpace bucketSpace, // appropriate priority. SplitChecker checker(priority); for (uint32_t i = 0; i < e->getNodeCount(); ++i) { - _pendingMessageTracker.checkPendingMessages(e->getNodeRef(i).getNode(), - document::Bucket(bucketSpace, e.getBucketId()), - checker); + _pendingMessageTracker.checkPendingMessages(e->getNodeRef(i).getNode(), document::Bucket(bucketSpace, e.getBucketId()), checker); if (checker.found) { return; } } - Operation::SP operation = - _idealStateManager.generateInterceptingSplit(bucketSpace, e, priority); + Operation::SP operation = _idealStateManager.generateInterceptingSplit(bucketSpace, e, priority); if (operation.get()) { _maintenanceOperationOwner.start(operation, priority); @@ -458,8 +447,7 @@ DistributorStripe::checkBucketForSplit(document::BucketSpace bucketSpace, // TODO STRIPE must be invoked by top-level bucket db updater probably void -DistributorStripe::propagateDefaultDistribution( - std::shared_ptr<const lib::Distribution> distribution) +DistributorStripe::propagateDefaultDistribution(std::shared_ptr<const lib::Distribution> distribution) { auto global_distr = GlobalBucketSpaceDistributionConverter::convert_to_global(*distribution); for (auto* repo : {_bucketSpaceRepo.get(), _readOnlyBucketSpaceRepo.get()}) { @@ -562,7 +550,7 @@ void DistributorStripe::startExternalOperations() { _fetchedMessages.clear(); } -std::unordered_map<uint16_t, uint32_t> +MinReplicaMap DistributorStripe::getMinReplica() const { std::lock_guard guard(_metricLock); @@ -655,7 +643,7 @@ DistributorStripe::updateInternalMetricsForCompletedScan() _bucketDBMetricUpdater.completeRound(); _bucketDbStats = _bucketDBMetricUpdater.getLastCompleteStats(); - _maintenanceStats = _scanner->getPendingMaintenanceStats(); + _maintenanceStats = _scanner->reset(); auto new_space_stats = toBucketSpacesStats(_maintenanceStats.perNodeStats); if (merge_no_longer_pending_edge(_bucketSpacesStats, new_space_stats)) { _must_send_updated_host_info = true; @@ -696,12 +684,9 @@ DistributorStripe::scanNextBucket() updateInternalMetricsForCompletedScan(); leaveRecoveryMode(); send_updated_host_info_if_required(); - _scanner->reset(); } else { const auto &distribution(_bucketSpaceRepo->get(scanResult.getBucketSpace()).getDistribution()); - _bucketDBMetricUpdater.visit( - scanResult.getEntry(), - distribution.getRedundancy()); + _bucketDBMetricUpdater.visit(scanResult.getEntry(), distribution.getRedundancy()); } return scanResult; } @@ -823,12 +808,6 @@ DistributorStripe::getActiveIdealStateOperations() const return _maintenanceOperationOwner.toString(); } -std::string -DistributorStripe::getActiveOperations() const -{ - return _operationOwner.toString(); -} - StripeAccessGuard::PendingOperationStats DistributorStripe::pending_operation_stats() const { @@ -881,7 +860,7 @@ DistributorStripe::merge_entries_into_db(document::BucketSpace bucket_space, const lib::Distribution& distribution, const lib::ClusterState& new_state, const char* storage_up_states, - const std::unordered_set<uint16_t>& outdated_nodes, + const OutdatedNodes& outdated_nodes, const std::vector<dbtransition::Entry>& entries) { bucket_db_updater().merge_entries_into_db(bucket_space, gathered_at_timestamp, distribution, diff --git a/storage/src/vespa/storage/distributor/distributor_stripe.h b/storage/src/vespa/storage/distributor/distributor_stripe.h index 801efa0ff73..338a6c72125 100644 --- a/storage/src/vespa/storage/distributor/distributor_stripe.h +++ b/storage/src/vespa/storage/distributor/distributor_stripe.h @@ -122,7 +122,6 @@ public: StripeAccessGuard::PendingOperationStats pending_operation_stats() const override; std::string getActiveIdealStateOperations() const; - std::string getActiveOperations() const; framework::ThreadWaitInfo doNonCriticalTick(framework::ThreadIndex); @@ -219,7 +218,7 @@ private: /** * Return a copy of the latest min replica data, see MinReplicaProvider. */ - std::unordered_map<uint16_t, uint32_t> getMinReplica() const override; + MinReplicaMap getMinReplica() const override; PerNodeBucketSpacesStats getBucketSpacesStats() const override; @@ -286,7 +285,7 @@ private: const lib::Distribution& distribution, const lib::ClusterState& new_state, const char* storage_up_states, - const std::unordered_set<uint16_t>& outdated_nodes, + const OutdatedNodes & outdated_nodes, const std::vector<dbtransition::Entry>& entries) override; void update_read_snapshot_before_db_pruning() override; void update_read_snapshot_after_db_pruning(const lib::ClusterStateBundle& new_state) override; diff --git a/storage/src/vespa/storage/distributor/distributor_stripe_component.cpp b/storage/src/vespa/storage/distributor/distributor_stripe_component.cpp index 9a5fd595b1d..16cc887096f 100644 --- a/storage/src/vespa/storage/distributor/distributor_stripe_component.cpp +++ b/storage/src/vespa/storage/distributor/distributor_stripe_component.cpp @@ -17,12 +17,11 @@ using document::BucketSpace; namespace storage::distributor { -DistributorStripeComponent::DistributorStripeComponent( - DistributorStripeInterface& distributor, - DistributorBucketSpaceRepo& bucketSpaceRepo, - DistributorBucketSpaceRepo& readOnlyBucketSpaceRepo, - DistributorComponentRegister& compReg, - const std::string& name) +DistributorStripeComponent::DistributorStripeComponent(DistributorStripeInterface& distributor, + DistributorBucketSpaceRepo& bucketSpaceRepo, + DistributorBucketSpaceRepo& readOnlyBucketSpaceRepo, + DistributorComponentRegister& compReg, + const std::string& name) : storage::DistributorComponent(compReg, name), _distributor(distributor), _bucketSpaceRepo(bucketSpaceRepo), @@ -44,30 +43,6 @@ DistributorStripeComponent::sendUp(const api::StorageMessage::SP& msg) _distributor.getMessageSender().sendUp(msg); } -void -DistributorStripeComponent::enumerateUnavailableNodes( - std::vector<uint16_t>& unavailableNodes, - const lib::ClusterState& s, - const document::Bucket& bucket, - const std::vector<BucketCopy>& candidates) const -{ - const auto* up_states = storage_node_up_states(); - for (uint32_t i = 0; i < candidates.size(); ++i) { - const BucketCopy& copy(candidates[i]); - const lib::NodeState& ns( - s.getNodeState(lib::Node(lib::NodeType::STORAGE, copy.getNode()))); - if (!ns.getState().oneOf(up_states)) { - LOG(debug, - "Trying to add a bucket copy to %s whose node is marked as " - "down in the cluster state: %s. Ignoring it since no zombies " - "are allowed!", - bucket.toString().c_str(), - copy.toString().c_str()); - unavailableNodes.emplace_back(copy.getNode()); - } - } -} - namespace { /** @@ -97,8 +72,7 @@ UpdateBucketDatabaseProcessor::UpdateBucketDatabaseProcessor(const framework::Cl UpdateBucketDatabaseProcessor::~UpdateBucketDatabaseProcessor() = default; BucketDatabase::Entry -UpdateBucketDatabaseProcessor::create_entry(const document::BucketId &bucket) const -{ +UpdateBucketDatabaseProcessor::create_entry(const document::BucketId &bucket) const { return BucketDatabase::Entry(bucket, BucketInfo()); } @@ -125,21 +99,16 @@ UpdateBucketDatabaseProcessor::process_entry(BucketDatabase::Entry &entry) const } void -DistributorStripeComponent::update_bucket_database( - const document::Bucket& bucket, - const std::vector<BucketCopy>& changed_nodes, - uint32_t update_flags) +DistributorStripeComponent::update_bucket_database(const document::Bucket& bucket, + const std::vector<BucketCopy>& changed_nodes, uint32_t update_flags) { auto &bucketSpace(_bucketSpaceRepo.get(bucket.getBucketSpace())); assert(!(bucket.getBucketId() == document::BucketId())); BucketOwnership ownership(bucketSpace.check_ownership_in_pending_and_current_state(bucket.getBucketId())); if (!ownership.isOwned()) { - LOG(debug, - "Trying to add %s to database that we do not own according to " - "cluster state '%s' - ignoring!", - bucket.toString().c_str(), - ownership.getNonOwnedState().toString().c_str()); + LOG(debug, "Trying to add %s to database that we do not own according to cluster state '%s' - ignoring!", + bucket.toString().c_str(), ownership.getNonOwnedState().toString().c_str()); return; } @@ -168,7 +137,7 @@ DistributorStripeComponent::update_bucket_database( UpdateBucketDatabaseProcessor processor(getClock(), found_down_node ? up_nodes : changed_nodes, - bucketSpace.get_ideal_service_layer_nodes_bundle(bucket.getBucketId()).get_available_nodes(), + bucketSpace.get_ideal_service_layer_nodes_bundle(bucket.getBucketId()).available_nodes(), (update_flags & DatabaseUpdate::RESET_TRUSTED) != 0); bucketSpace.getBucketDatabase().process_update(bucket.getBucketId(), processor, (update_flags & DatabaseUpdate::CREATE_IF_NONEXISTING) != 0); @@ -184,8 +153,7 @@ DistributorStripeComponent::node_address(uint16_t node_index) const noexcept // Implements DistributorStripeOperationContext void -DistributorStripeComponent::remove_nodes_from_bucket_database(const document::Bucket& bucket, - const std::vector<uint16_t>& nodes) +DistributorStripeComponent::remove_nodes_from_bucket_database(const document::Bucket& bucket, const std::vector<uint16_t>& nodes) { auto &bucketSpace(_bucketSpaceRepo.get(bucket.getBucketSpace())); BucketDatabase::Entry dbentry = bucketSpace.getBucketDatabase().get(bucket.getBucketId()); @@ -193,21 +161,15 @@ DistributorStripeComponent::remove_nodes_from_bucket_database(const document::Bu if (dbentry.valid()) { for (uint32_t i = 0; i < nodes.size(); ++i) { if (dbentry->removeNode(nodes[i])) { - LOG(debug, - "Removed node %d from bucket %s. %u copies remaining", - nodes[i], - bucket.toString().c_str(), - dbentry->getNodeCount()); + LOG(debug, "Removed node %d from bucket %s. %u copies remaining", + nodes[i], bucket.toString().c_str(), dbentry->getNodeCount()); } } if (dbentry->getNodeCount() != 0) { bucketSpace.getBucketDatabase().update(dbentry); } else { - LOG(debug, - "After update, bucket %s now has no copies. " - "Removing from database.", - bucket.toString().c_str()); + LOG(debug, "After update, bucket %s now has no copies. Removing from database.", bucket.toString().c_str()); bucketSpace.getBucketDatabase().remove(bucket.getBucketId()); } @@ -218,7 +180,6 @@ document::BucketId DistributorStripeComponent::make_split_bit_constrained_bucket_id(const document::DocumentId& doc_id) const { document::BucketId id(getBucketIdFactory().getBucketId(doc_id)); - id.setUsedBits(_distributor.getConfig().getMinimalBucketSplit()); return id.stripUnused(); } @@ -239,28 +200,18 @@ DistributorStripeComponent::get_sibling(const document::BucketId& bid) const zeroBucket = document::BucketId(1, 0); oneBucket = document::BucketId(1, 1); } else { - document::BucketId joinedBucket = document::BucketId( - bid.getUsedBits() - 1, - bid.getId()); - - zeroBucket = document::BucketId( - bid.getUsedBits(), - joinedBucket.getId()); - + document::BucketId joinedBucket = document::BucketId(bid.getUsedBits() - 1,bid.getId()); + zeroBucket = document::BucketId(bid.getUsedBits(), joinedBucket.getId()); uint64_t hiBit = 1; hiBit <<= (bid.getUsedBits() - 1); - oneBucket = document::BucketId( - bid.getUsedBits(), - joinedBucket.getId() | hiBit); + oneBucket = document::BucketId(bid.getUsedBits(), joinedBucket.getId() | hiBit); } return (zeroBucket == bid) ? oneBucket : zeroBucket; } bool -DistributorStripeComponent::has_pending_message(uint16_t node_index, - const document::Bucket& bucket, - uint32_t message_type) const +DistributorStripeComponent::has_pending_message(uint16_t node_index, const document::Bucket& bucket, uint32_t message_type) const { const auto& sender = static_cast<const DistributorStripeMessageSender&>(getDistributor()); return sender.getPendingMessageTracker().hasPendingMessage(node_index, bucket, message_type); @@ -275,8 +226,7 @@ DistributorStripeComponent::cluster_state_bundle() const bool DistributorStripeComponent::storage_node_is_up(document::BucketSpace bucket_space, uint32_t node_index) const { - const lib::NodeState& ns = cluster_state_bundle().getDerivedClusterState(bucket_space)->getNodeState( - lib::Node(lib::NodeType::STORAGE, node_index)); + const auto & ns = cluster_state_bundle().getDerivedClusterState(bucket_space)->getNodeState(lib::Node(lib::NodeType::STORAGE, node_index)); return ns.getState().oneOf(storage_node_up_states()); } diff --git a/storage/src/vespa/storage/distributor/distributor_stripe_component.h b/storage/src/vespa/storage/distributor/distributor_stripe_component.h index 5bcf9eec76d..8bf507f3fac 100644 --- a/storage/src/vespa/storage/distributor/distributor_stripe_component.h +++ b/storage/src/vespa/storage/distributor/distributor_stripe_component.h @@ -68,20 +68,14 @@ public: /** * Simple API for the common case of modifying a single node. */ - void update_bucket_database(const document::Bucket& bucket, - const BucketCopy& changed_node, - uint32_t update_flags) override { - update_bucket_database(bucket, - toVector<BucketCopy>(changed_node), - update_flags); + void update_bucket_database(const document::Bucket& bucket, const BucketCopy& changed_node, uint32_t update_flags) override { + update_bucket_database(bucket, toVector<BucketCopy>(changed_node),update_flags); } /** * Adds the given copies to the bucket database. */ - void update_bucket_database(const document::Bucket& bucket, - const std::vector<BucketCopy>& changed_nodes, - uint32_t update_flags) override; + void update_bucket_database(const document::Bucket& bucket, const std::vector<BucketCopy>& changed_nodes, uint32_t update_flags) override; /** * Removes a copy from the given bucket from the bucket database. @@ -97,8 +91,7 @@ public: * If the resulting bucket is empty afterwards, removes the entire * bucket entry from the bucket database. */ - void remove_nodes_from_bucket_database(const document::Bucket& bucket, - const std::vector<uint16_t>& nodes) override; + void remove_nodes_from_bucket_database(const document::Bucket& bucket, const std::vector<uint16_t>& nodes) override; const DistributorBucketSpaceRepo& bucket_space_repo() const noexcept override { return _bucketSpaceRepo; @@ -129,9 +122,7 @@ public: const DistributorConfiguration& distributor_config() const noexcept override { return getDistributor().getConfig(); } - void send_inline_split_if_bucket_too_large(document::BucketSpace bucket_space, - const BucketDatabase::Entry& entry, - uint8_t pri) override { + void send_inline_split_if_bucket_too_large(document::BucketSpace bucket_space, const BucketDatabase::Entry& entry, uint8_t pri) override { getDistributor().checkBucketForSplit(bucket_space, entry, pri); } OperationRoutingSnapshot read_snapshot_for_bucket(const document::Bucket& bucket) const override { @@ -143,9 +134,7 @@ public: const PendingMessageTracker& pending_message_tracker() const noexcept override { return getDistributor().getPendingMessageTracker(); } - bool has_pending_message(uint16_t node_index, - const document::Bucket& bucket, - uint32_t message_type) const override; + bool has_pending_message(uint16_t node_index, const document::Bucket& bucket, uint32_t message_type) const override; const lib::ClusterState* pending_cluster_state_or_null(const document::BucketSpace& bucket_space) const override { return getDistributor().pendingClusterStateOrNull(bucket_space); } @@ -171,15 +160,7 @@ public: std::unique_ptr<document::select::Node> parse_selection(const vespalib::string& selection) const override; private: - void enumerateUnavailableNodes( - std::vector<uint16_t>& unavailableNodes, - const lib::ClusterState& s, - const document::Bucket& bucket, - const std::vector<BucketCopy>& candidates) const; DistributorStripeInterface& _distributor; - -protected: - DistributorBucketSpaceRepo& _bucketSpaceRepo; DistributorBucketSpaceRepo& _readOnlyBucketSpaceRepo; }; diff --git a/storage/src/vespa/storage/distributor/ideal_service_layer_nodes_bundle.cpp b/storage/src/vespa/storage/distributor/ideal_service_layer_nodes_bundle.cpp index 0d37219356e..cc4eedd2a35 100644 --- a/storage/src/vespa/storage/distributor/ideal_service_layer_nodes_bundle.cpp +++ b/storage/src/vespa/storage/distributor/ideal_service_layer_nodes_bundle.cpp @@ -2,16 +2,27 @@ #include "ideal_service_layer_nodes_bundle.h" #include <vespa/vdslib/distribution/idealnodecalculator.h> +#include <vespa/vespalib/stllike/hash_set_insert.hpp> + namespace storage::distributor { IdealServiceLayerNodesBundle::IdealServiceLayerNodesBundle() noexcept : _available_nodes(), _available_nonretired_nodes(), - _available_nonretired_or_maintenance_nodes() + _available_nonretired_or_maintenance_nodes(), + _unordered_nonretired_or_maintenance_nodes() { } +void +IdealServiceLayerNodesBundle::set_available_nonretired_or_maintenance_nodes(std::vector<uint16_t> available_nonretired_or_maintenance_nodes) { + _available_nonretired_or_maintenance_nodes = std::move(available_nonretired_or_maintenance_nodes); + _unordered_nonretired_or_maintenance_nodes.clear(); + _unordered_nonretired_or_maintenance_nodes.insert(_available_nonretired_or_maintenance_nodes.begin(), + _available_nonretired_or_maintenance_nodes.end()); +} + IdealServiceLayerNodesBundle::IdealServiceLayerNodesBundle(IdealServiceLayerNodesBundle &&) noexcept = default; IdealServiceLayerNodesBundle::~IdealServiceLayerNodesBundle() = default; diff --git a/storage/src/vespa/storage/distributor/ideal_service_layer_nodes_bundle.h b/storage/src/vespa/storage/distributor/ideal_service_layer_nodes_bundle.h index 929ec7aadc1..9577ec09208 100644 --- a/storage/src/vespa/storage/distributor/ideal_service_layer_nodes_bundle.h +++ b/storage/src/vespa/storage/distributor/ideal_service_layer_nodes_bundle.h @@ -1,8 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vector> -#include <cstdint> +#include <vespa/vespalib/stllike/hash_set.h> namespace storage::distributor { @@ -13,6 +12,7 @@ class IdealServiceLayerNodesBundle { std::vector<uint16_t> _available_nodes; std::vector<uint16_t> _available_nonretired_nodes; std::vector<uint16_t> _available_nonretired_or_maintenance_nodes; + vespalib::hash_set<uint16_t> _unordered_nonretired_or_maintenance_nodes; public: IdealServiceLayerNodesBundle() noexcept; IdealServiceLayerNodesBundle(IdealServiceLayerNodesBundle &&) noexcept; @@ -24,14 +24,15 @@ public: void set_available_nonretired_nodes(std::vector<uint16_t> available_nonretired_nodes) { _available_nonretired_nodes = std::move(available_nonretired_nodes); } - void set_available_nonretired_or_maintenance_nodes(std::vector<uint16_t> available_nonretired_or_maintenance_nodes) { - _available_nonretired_or_maintenance_nodes = std::move(available_nonretired_or_maintenance_nodes); - } - std::vector<uint16_t> get_available_nodes() const { return _available_nodes; } - std::vector<uint16_t> get_available_nonretired_nodes() const { return _available_nonretired_nodes; } - std::vector<uint16_t> get_available_nonretired_or_maintenance_nodes() const { + void set_available_nonretired_or_maintenance_nodes(std::vector<uint16_t> available_nonretired_or_maintenance_nodes); + const std::vector<uint16_t> & available_nodes() const noexcept { return _available_nodes; } + const std::vector<uint16_t> & available_nonretired_nodes() const noexcept { return _available_nonretired_nodes; } + const std::vector<uint16_t> & available_nonretired_or_maintenance_nodes() const noexcept { return _available_nonretired_or_maintenance_nodes; } + bool is_nonretired_or_maintenance(uint16_t node) const noexcept { + return _unordered_nonretired_or_maintenance_nodes.contains(node); + } }; } diff --git a/storage/src/vespa/storage/distributor/idealstatemanager.cpp b/storage/src/vespa/storage/distributor/idealstatemanager.cpp index cad141e76ed..bc928ca3d41 100644 --- a/storage/src/vespa/storage/distributor/idealstatemanager.cpp +++ b/storage/src/vespa/storage/distributor/idealstatemanager.cpp @@ -10,10 +10,9 @@ #include <vespa/storageapi/message/persistence.h> #include <vespa/document/bucket/fixed_bucket_spaces.h> #include <vespa/vespalib/util/assert.h> -#include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/log/log.h> -LOG_SETUP(".distributor.operation.queue"); +LOG_SETUP(".distributor.idealstatemanager"); using document::BucketSpace; using storage::lib::Node; @@ -21,10 +20,9 @@ using storage::lib::NodeType; namespace storage::distributor { -IdealStateManager::IdealStateManager( - const DistributorNodeContext& node_ctx, - DistributorStripeOperationContext& op_ctx, - IdealStateMetricSet& metrics) +IdealStateManager::IdealStateManager(const DistributorNodeContext& node_ctx, + DistributorStripeOperationContext& op_ctx, + IdealStateMetricSet& metrics) : _metrics(metrics), _stateCheckers(), _splitBucketStateChecker(nullptr), @@ -56,9 +54,7 @@ IdealStateManager::fillParentAndChildBuckets(StateChecker::Context& c) { c.db.getAll(c.getBucketId(), c.entries); if (c.entries.empty()) { - LOG(spam, - "Did not find bucket %s in bucket database", - c.bucket.toString().c_str()); + LOG(spam, "Did not find bucket %s in bucket database", c.bucket.toString().c_str()); } } void @@ -85,8 +81,7 @@ namespace { * overwriting if already explicitly set. */ bool -canOverwriteResult(const StateChecker::Result& existing, - const StateChecker::Result& candidate) +canOverwriteResult(const StateChecker::Result& existing, const StateChecker::Result& candidate) { return (!existing.getPriority().requiresMaintenance() && candidate.getPriority().requiresMaintenance()); @@ -101,9 +96,7 @@ IdealStateManager::runStateCheckers(StateChecker::Context& c) const // We go through _all_ active state checkers so that statistics can be // collected across all checkers, not just the ones that are highest pri. for (const auto & checker : _stateCheckers) { - if (!operation_context().distributor_config().stateCheckerIsActive( - checker->getName())) - { + if (!operation_context().distributor_config().stateCheckerIsActive(checker->getName())) { LOG(spam, "Skipping state checker %s", checker->getName()); continue; } @@ -116,7 +109,8 @@ IdealStateManager::runStateCheckers(StateChecker::Context& c) const return highestPri; } -void IdealStateManager::verify_only_live_nodes_in_context(const StateChecker::Context& c) const { +void +IdealStateManager::verify_only_live_nodes_in_context(const StateChecker::Context& c) const { if (_has_logged_phantom_replica_warning) { return; } @@ -125,11 +119,8 @@ void IdealStateManager::verify_only_live_nodes_in_context(const StateChecker::Co const auto& state = c.systemState.getNodeState(lib::Node(lib::NodeType::STORAGE, index)); // Only nodes in Up, Initializing or Retired should ever be present in the DB. if (!state.getState().oneOf("uir")) { - LOG(error, "%s in bucket DB is on node %u, which is in unavailable state %s. " - "Current cluster state is '%s'", - c.entry.getBucketId().toString().c_str(), - index, - state.getState().toString().c_str(), + LOG(error, "%s in bucket DB is on node %u, which is in unavailable state %s. Current cluster state is '%s'", + c.entry.getBucketId().toString().c_str(), index, state.getState().toString().c_str(), c.systemState.toString().c_str()); ASSERT_ONCE_OR_LOG(false, "Bucket DB contains replicas on unavailable node", 10000); _has_logged_phantom_replica_warning = true; @@ -138,9 +129,7 @@ void IdealStateManager::verify_only_live_nodes_in_context(const StateChecker::Co } StateChecker::Result -IdealStateManager::generateHighestPriority( - const document::Bucket& bucket, - NodeMaintenanceStatsTracker& statsTracker) const +IdealStateManager::generateHighestPriority(const document::Bucket& bucket, NodeMaintenanceStatsTracker& statsTracker) const { auto& distributorBucketSpace = _op_ctx.bucket_space_repo().get(bucket.getBucketSpace()); StateChecker::Context c(node_context(), operation_context(), distributorBucketSpace, statsTracker, bucket); @@ -159,9 +148,7 @@ IdealStateManager::generateHighestPriority( } MaintenancePriorityAndType -IdealStateManager::prioritize( - const document::Bucket& bucket, - NodeMaintenanceStatsTracker& statsTracker) const +IdealStateManager::prioritize(const document::Bucket& bucket, NodeMaintenanceStatsTracker& statsTracker) const { StateChecker::Result generated(generateHighestPriority(bucket, statsTracker)); MaintenancePriority priority(generated.getPriority()); @@ -172,8 +159,7 @@ IdealStateManager::prioritize( } IdealStateOperation::SP -IdealStateManager::generateInterceptingSplit(BucketSpace bucketSpace, - const BucketDatabase::Entry& e, +IdealStateManager::generateInterceptingSplit(BucketSpace bucketSpace, const BucketDatabase::Entry& e, api::StorageMessage::Priority pri) { NodeMaintenanceStatsTracker statsTracker; @@ -199,18 +185,15 @@ MaintenanceOperation::SP IdealStateManager::generate(const document::Bucket& bucket) const { NodeMaintenanceStatsTracker statsTracker; - IdealStateOperation::SP op( - generateHighestPriority(bucket, statsTracker).createOperation()); + IdealStateOperation::SP op(generateHighestPriority(bucket, statsTracker).createOperation()); if (op.get()) { - op->setIdealStateManager( - const_cast<IdealStateManager*>(this)); + op->setIdealStateManager(const_cast<IdealStateManager*>(this)); } return op; } std::vector<MaintenanceOperation::SP> -IdealStateManager::generateAll(const document::Bucket &bucket, - NodeMaintenanceStatsTracker& statsTracker) const +IdealStateManager::generateAll(const document::Bucket &bucket, NodeMaintenanceStatsTracker& statsTracker) const { auto& distributorBucketSpace = _op_ctx.bucket_space_repo().get(bucket.getBucketSpace()); StateChecker::Context c(node_context(), operation_context(), distributorBucketSpace, statsTracker, bucket); @@ -234,15 +217,11 @@ IdealStateManager::generateAll(const document::Bucket &bucket, } void -IdealStateManager::getBucketStatus( - BucketSpace bucketSpace, - const BucketDatabase::ConstEntryRef& entry, - NodeMaintenanceStatsTracker& statsTracker, - std::ostream& out) const +IdealStateManager::getBucketStatus(BucketSpace bucketSpace, const BucketDatabase::ConstEntryRef& entry, + NodeMaintenanceStatsTracker& statsTracker, std::ostream& out) const { document::Bucket bucket(bucketSpace, entry.getBucketId()); - std::vector<MaintenanceOperation::SP> operations( - generateAll(bucket, statsTracker)); + std::vector<MaintenanceOperation::SP> operations(generateAll(bucket, statsTracker)); if (operations.empty()) { out << entry.getBucketId() << " : "; } else { @@ -261,13 +240,15 @@ IdealStateManager::getBucketStatus( out << "[" << entry->toString() << "]<br>\n"; } -void IdealStateManager::dump_bucket_space_db_status(document::BucketSpace bucket_space, std::ostream& out) const { +void +IdealStateManager::dump_bucket_space_db_status(document::BucketSpace bucket_space, std::ostream& out) const { StatusBucketVisitor proc(*this, bucket_space, out); auto& distributorBucketSpace = _op_ctx.bucket_space_repo().get(bucket_space); distributorBucketSpace.getBucketDatabase().for_each_upper_bound(proc); } -void IdealStateManager::getBucketStatus(std::ostream& out) const { +void +IdealStateManager::getBucketStatus(std::ostream& out) const { LOG(debug, "Dumping bucket database valid at cluster state version %u", operation_context().cluster_state_bundle().getVersion()); diff --git a/storage/src/vespa/storage/distributor/idealstatemanager.h b/storage/src/vespa/storage/distributor/idealstatemanager.h index 0c9e3ffa1c6..39a662e4a81 100644 --- a/storage/src/vespa/storage/distributor/idealstatemanager.h +++ b/storage/src/vespa/storage/distributor/idealstatemanager.h @@ -49,18 +49,14 @@ public: MaintenanceOperation::SP generate(const document::Bucket& bucket) const override; // MaintenanceOperationGenerator - std::vector<MaintenanceOperation::SP> generateAll( - const document::Bucket& bucket, - NodeMaintenanceStatsTracker& statsTracker) const override; + std::vector<MaintenanceOperation::SP> generateAll(const document::Bucket& bucket, NodeMaintenanceStatsTracker& statsTracker) const override; /** * If the given bucket is too large, generate a split operation for it, * with higher priority than the given one. */ - IdealStateOperation::SP generateInterceptingSplit( - document::BucketSpace bucketSpace, - const BucketDatabase::Entry& e, - api::StorageMessage::Priority pri); + IdealStateOperation::SP generateInterceptingSplit(document::BucketSpace bucketSpace, const BucketDatabase::Entry& e, + api::StorageMessage::Priority pri); IdealStateMetricSet& getMetrics() noexcept { return _metrics; } @@ -78,9 +74,7 @@ private: void verify_only_live_nodes_in_context(const StateChecker::Context& c) const; static void fillParentAndChildBuckets(StateChecker::Context& c); static void fillSiblingBucket(StateChecker::Context& c); - StateChecker::Result generateHighestPriority( - const document::Bucket& bucket, - NodeMaintenanceStatsTracker& statsTracker) const; + StateChecker::Result generateHighestPriority(const document::Bucket& bucket, NodeMaintenanceStatsTracker& statsTracker) const; StateChecker::Result runStateCheckers(StateChecker::Context& c) const; static BucketDatabase::Entry* getEntryForPrimaryBucket(StateChecker::Context& c); diff --git a/storage/src/vespa/storage/distributor/idealstatemetricsset.cpp b/storage/src/vespa/storage/distributor/idealstatemetricsset.cpp index d79bf2c4810..d50b2004bf2 100644 --- a/storage/src/vespa/storage/distributor/idealstatemetricsset.cpp +++ b/storage/src/vespa/storage/distributor/idealstatemetricsset.cpp @@ -134,7 +134,7 @@ IdealStateMetricSet::IdealStateMetricSet() IdealStateMetricSet::~IdealStateMetricSet() = default; -void IdealStateMetricSet::setPendingOperations(const std::vector<uint64_t>& newMetrics) { +void IdealStateMetricSet::setPendingOperations(vespalib::ConstArrayRef<uint64_t> newMetrics) { for (uint32_t i = 0; i < IdealStateOperation::OPERATION_COUNT; i++) { operations[i]->pending.set(newMetrics[i]); } diff --git a/storage/src/vespa/storage/distributor/idealstatemetricsset.h b/storage/src/vespa/storage/distributor/idealstatemetricsset.h index 6528ad7dc72..0bbc13d061a 100644 --- a/storage/src/vespa/storage/distributor/idealstatemetricsset.h +++ b/storage/src/vespa/storage/distributor/idealstatemetricsset.h @@ -5,6 +5,7 @@ #include <vespa/metrics/valuemetric.h> #include <vespa/metrics/countmetric.h> #include <vespa/storage/distributor/operations/idealstate/idealstateoperation.h> +#include <vespa/vespalib/util/arrayref.h> namespace storage::distributor { @@ -61,7 +62,7 @@ public: IdealStateMetricSet(); ~IdealStateMetricSet() override; - void setPendingOperations(const std::vector<uint64_t>& newMetrics); + void setPendingOperations(vespalib::ConstArrayRef<uint64_t> newMetrics); }; } // storage::distributor diff --git a/storage/src/vespa/storage/distributor/maintenance/maintenancescanner.h b/storage/src/vespa/storage/distributor/maintenance/maintenancescanner.h index 4fec2e57cbc..e4ccb6d88ad 100644 --- a/storage/src/vespa/storage/distributor/maintenance/maintenancescanner.h +++ b/storage/src/vespa/storage/distributor/maintenance/maintenancescanner.h @@ -32,8 +32,6 @@ public: }; virtual ScanResult scanNext() = 0; - - virtual void reset() = 0; }; } diff --git a/storage/src/vespa/storage/distributor/maintenance/node_maintenance_stats_tracker.cpp b/storage/src/vespa/storage/distributor/maintenance/node_maintenance_stats_tracker.cpp index 47b8aec4aa6..592d92940d6 100644 --- a/storage/src/vespa/storage/distributor/maintenance/node_maintenance_stats_tracker.cpp +++ b/storage/src/vespa/storage/distributor/maintenance/node_maintenance_stats_tracker.cpp @@ -1,22 +1,14 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "node_maintenance_stats_tracker.h" +#include <vespa/vespalib/stllike/hash_map.hpp> +#include <vespa/vespalib/stllike/hash_map_equal.hpp> #include <ostream> namespace storage::distributor { const NodeMaintenanceStats NodeMaintenanceStatsTracker::_emptyNodeMaintenanceStats; -void -NodeMaintenanceStats::merge(const NodeMaintenanceStats& rhs) -{ - movingOut += rhs.movingOut; - syncing += rhs.syncing; - copyingIn += rhs.copyingIn; - copyingOut += rhs.copyingOut; - total += rhs.total; -} - namespace { void @@ -32,6 +24,54 @@ merge_bucket_spaces_stats(NodeMaintenanceStatsTracker::BucketSpacesStats& dest, } void +NodeMaintenanceStatsTracker::incMovingOut(uint16_t node, document::BucketSpace bucketSpace) { + ++_node_stats[node][bucketSpace].movingOut; + ++_total_stats.movingOut; +} + +void +NodeMaintenanceStatsTracker::incSyncing(uint16_t node, document::BucketSpace bucketSpace) { + ++_node_stats[node][bucketSpace].syncing; + ++_total_stats.syncing; +} + +void +NodeMaintenanceStatsTracker::incCopyingIn(uint16_t node, document::BucketSpace bucketSpace) { + ++_node_stats[node][bucketSpace].copyingIn; + ++_total_stats.copyingIn; +} + +void +NodeMaintenanceStatsTracker::incCopyingOut(uint16_t node, document::BucketSpace bucketSpace) { + ++_node_stats[node][bucketSpace].copyingOut; + ++_total_stats.copyingOut; +} + +void +NodeMaintenanceStatsTracker::NodeMaintenanceStatsTracker::incTotal(uint16_t node, document::BucketSpace bucketSpace) { + ++_node_stats[node][bucketSpace].total; + ++_total_stats.total; +} + +const NodeMaintenanceStats& +NodeMaintenanceStatsTracker::forNode(uint16_t node, document::BucketSpace bucketSpace) const { + auto nodeItr = _node_stats.find(node); + if (nodeItr != _node_stats.end()) { + auto bucketSpaceItr = nodeItr->second.find(bucketSpace); + if (bucketSpaceItr != nodeItr->second.end()) { + return bucketSpaceItr->second; + } + } + return _emptyNodeMaintenanceStats; +} + +bool +NodeMaintenanceStatsTracker::operator==(const NodeMaintenanceStatsTracker& rhs) const noexcept { + return ((_node_stats == rhs._node_stats) && + (_max_observed_time_since_last_gc == rhs._max_observed_time_since_last_gc)); +} + +void NodeMaintenanceStatsTracker::merge(const NodeMaintenanceStatsTracker& rhs) { for (const auto& entry : rhs._node_stats) { @@ -55,13 +95,24 @@ operator<<(std::ostream& os, const NodeMaintenanceStats& stats) return os; } -NodeMaintenanceStatsTracker::NodeMaintenanceStatsTracker() +NodeMaintenanceStatsTracker::NodeMaintenanceStatsTracker() noexcept : _node_stats(), _total_stats(), _max_observed_time_since_last_gc(0) {} +NodeMaintenanceStatsTracker::NodeMaintenanceStatsTracker(NodeMaintenanceStatsTracker &&) noexcept = default; +NodeMaintenanceStatsTracker & NodeMaintenanceStatsTracker::operator =(NodeMaintenanceStatsTracker &&) noexcept = default; +NodeMaintenanceStatsTracker::NodeMaintenanceStatsTracker(const NodeMaintenanceStatsTracker &) = default; NodeMaintenanceStatsTracker::~NodeMaintenanceStatsTracker() = default; +void +NodeMaintenanceStatsTracker::reset(size_t nodes) { + _node_stats.clear(); + _node_stats.resize(nodes); + _total_stats = NodeMaintenanceStats(); + _max_observed_time_since_last_gc = vespalib::duration::zero(); +} + } diff --git a/storage/src/vespa/storage/distributor/maintenance/node_maintenance_stats_tracker.h b/storage/src/vespa/storage/distributor/maintenance/node_maintenance_stats_tracker.h index deeef118685..84705fbca9d 100644 --- a/storage/src/vespa/storage/distributor/maintenance/node_maintenance_stats_tracker.h +++ b/storage/src/vespa/storage/distributor/maintenance/node_maintenance_stats_tracker.h @@ -1,9 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <unordered_map> #include <vespa/document/bucket/bucketspace.h> #include <vespa/vespalib/util/time.h> +#include <vespa/vespalib/stllike/hash_map.h> namespace storage::distributor { @@ -37,7 +37,13 @@ struct NodeMaintenanceStats return !(*this == other); } - void merge(const NodeMaintenanceStats& rhs); + void merge(const NodeMaintenanceStats& rhs) noexcept { + movingOut += rhs.movingOut; + syncing += rhs.syncing; + copyingIn += rhs.copyingIn; + copyingOut += rhs.copyingOut; + total += rhs.total; + } }; std::ostream& operator<<(std::ostream&, const NodeMaintenanceStats&); @@ -45,8 +51,8 @@ std::ostream& operator<<(std::ostream&, const NodeMaintenanceStats&); class NodeMaintenanceStatsTracker { public: - using BucketSpacesStats = std::unordered_map<document::BucketSpace, NodeMaintenanceStats, document::BucketSpace::hash>; - using PerNodeStats = std::unordered_map<uint16_t, BucketSpacesStats>; + using BucketSpacesStats = vespalib::hash_map<document::BucketSpace, NodeMaintenanceStats, document::BucketSpace::hash>; + using PerNodeStats = vespalib::hash_map<uint16_t, BucketSpacesStats>; private: PerNodeStats _node_stats; @@ -56,33 +62,23 @@ private: static const NodeMaintenanceStats _emptyNodeMaintenanceStats; public: - NodeMaintenanceStatsTracker(); + NodeMaintenanceStatsTracker() noexcept; + NodeMaintenanceStatsTracker(NodeMaintenanceStatsTracker &&) noexcept; + NodeMaintenanceStatsTracker & operator =(NodeMaintenanceStatsTracker &&) noexcept; + NodeMaintenanceStatsTracker(const NodeMaintenanceStatsTracker &); ~NodeMaintenanceStatsTracker(); + void reset(size_t nodes); + size_t numNodes() const { return _node_stats.size(); } - void incMovingOut(uint16_t node, document::BucketSpace bucketSpace) { - ++_node_stats[node][bucketSpace].movingOut; - ++_total_stats.movingOut; - } + void incMovingOut(uint16_t node, document::BucketSpace bucketSpace); - void incSyncing(uint16_t node, document::BucketSpace bucketSpace) { - ++_node_stats[node][bucketSpace].syncing; - ++_total_stats.syncing; - } + void incSyncing(uint16_t node, document::BucketSpace bucketSpace); - void incCopyingIn(uint16_t node, document::BucketSpace bucketSpace) { - ++_node_stats[node][bucketSpace].copyingIn; - ++_total_stats.copyingIn; - } + void incCopyingIn(uint16_t node, document::BucketSpace bucketSpace); - void incCopyingOut(uint16_t node, document::BucketSpace bucketSpace) { - ++_node_stats[node][bucketSpace].copyingOut; - ++_total_stats.copyingOut; - } + void incCopyingOut(uint16_t node, document::BucketSpace bucketSpace); - void incTotal(uint16_t node, document::BucketSpace bucketSpace) { - ++_node_stats[node][bucketSpace].total; - ++_total_stats.total; - } + void incTotal(uint16_t node, document::BucketSpace bucketSpace); void update_observed_time_since_last_gc(vespalib::duration time_since_gc) noexcept { _max_observed_time_since_last_gc = std::max(time_since_gc, _max_observed_time_since_last_gc); @@ -92,18 +88,9 @@ public: * Returned statistics for a given node index and bucket space, or all zero statistics * if none have been recorded yet */ - const NodeMaintenanceStats& forNode(uint16_t node, document::BucketSpace bucketSpace) const { - auto nodeItr = _node_stats.find(node); - if (nodeItr != _node_stats.end()) { - auto bucketSpaceItr = nodeItr->second.find(bucketSpace); - if (bucketSpaceItr != nodeItr->second.end()) { - return bucketSpaceItr->second; - } - } - return _emptyNodeMaintenanceStats; - } + const NodeMaintenanceStats& forNode(uint16_t node, document::BucketSpace bucketSpace) const; - const PerNodeStats& perNodeStats() const { + const PerNodeStats& perNodeStats() const noexcept { return _node_stats; } @@ -118,10 +105,7 @@ public: return _max_observed_time_since_last_gc; } - bool operator==(const NodeMaintenanceStatsTracker& rhs) const { - return ((_node_stats == rhs._node_stats) && - (_max_observed_time_since_last_gc == rhs._max_observed_time_since_last_gc)); - } + bool operator==(const NodeMaintenanceStatsTracker& rhs) const noexcept; void merge(const NodeMaintenanceStatsTracker& rhs); }; diff --git a/storage/src/vespa/storage/distributor/maintenance/simplemaintenancescanner.cpp b/storage/src/vespa/storage/distributor/maintenance/simplemaintenancescanner.cpp index cb60e3eb0fc..e0c1abaaffa 100644 --- a/storage/src/vespa/storage/distributor/maintenance/simplemaintenancescanner.cpp +++ b/storage/src/vespa/storage/distributor/maintenance/simplemaintenancescanner.cpp @@ -13,22 +13,22 @@ SimpleMaintenanceScanner::SimpleMaintenanceScanner(BucketPriorityDatabase& bucke _priorityGenerator(priorityGenerator), _bucketSpaceRepo(bucketSpaceRepo), _bucketSpaceItr(_bucketSpaceRepo.begin()), - _bucketCursor() + _bucketCursor(), + _pendingMaintenance() { } SimpleMaintenanceScanner::~SimpleMaintenanceScanner() = default; bool -SimpleMaintenanceScanner::GlobalMaintenanceStats::operator==(const GlobalMaintenanceStats& rhs) const +SimpleMaintenanceScanner::GlobalMaintenanceStats::operator==(const GlobalMaintenanceStats& rhs) const noexcept { return pending == rhs.pending; } void -SimpleMaintenanceScanner::GlobalMaintenanceStats::merge(const GlobalMaintenanceStats& rhs) +SimpleMaintenanceScanner::GlobalMaintenanceStats::merge(const GlobalMaintenanceStats& rhs) noexcept { - assert(pending.size() == rhs.pending.size()); for (size_t i = 0; i < pending.size(); ++i) { pending[i] += rhs.pending[i]; } @@ -41,11 +41,20 @@ SimpleMaintenanceScanner::PendingMaintenanceStats::merge(const PendingMaintenanc perNodeStats.merge(rhs.perNodeStats); } -SimpleMaintenanceScanner::PendingMaintenanceStats::PendingMaintenanceStats() = default; +SimpleMaintenanceScanner::PendingMaintenanceStats::PendingMaintenanceStats() noexcept = default; SimpleMaintenanceScanner::PendingMaintenanceStats::~PendingMaintenanceStats() = default; SimpleMaintenanceScanner::PendingMaintenanceStats::PendingMaintenanceStats(const PendingMaintenanceStats &) = default; +SimpleMaintenanceScanner::PendingMaintenanceStats::PendingMaintenanceStats(PendingMaintenanceStats &&) noexcept = default; SimpleMaintenanceScanner::PendingMaintenanceStats & -SimpleMaintenanceScanner::PendingMaintenanceStats::operator = (const PendingMaintenanceStats &) = default; +SimpleMaintenanceScanner::PendingMaintenanceStats::operator = (PendingMaintenanceStats &&) noexcept = default; + +SimpleMaintenanceScanner::PendingMaintenanceStats +SimpleMaintenanceScanner::PendingMaintenanceStats::reset() { + PendingMaintenanceStats prev = std::move(*this); + global = GlobalMaintenanceStats(); + perNodeStats.reset(prev.perNodeStats.numNodes()); + return prev; +} MaintenanceScanner::ScanResult SimpleMaintenanceScanner::scanNext() @@ -68,12 +77,12 @@ SimpleMaintenanceScanner::scanNext() } } -void +SimpleMaintenanceScanner::PendingMaintenanceStats SimpleMaintenanceScanner::reset() { _bucketCursor = document::BucketId(); _bucketSpaceItr = _bucketSpaceRepo.begin(); - _pendingMaintenance = PendingMaintenanceStats(); + return _pendingMaintenance.reset(); } void @@ -99,8 +108,7 @@ SimpleMaintenanceScanner::prioritizeBucket(const document::Bucket &bucket) } std::ostream& -operator<<(std::ostream& os, - const SimpleMaintenanceScanner::GlobalMaintenanceStats& stats) +operator<<(std::ostream& os, const SimpleMaintenanceScanner::GlobalMaintenanceStats& stats) { using MO = MaintenanceOperation; os << "delete bucket: " << stats.pending[MO::DELETE_BUCKET] diff --git a/storage/src/vespa/storage/distributor/maintenance/simplemaintenancescanner.h b/storage/src/vespa/storage/distributor/maintenance/simplemaintenancescanner.h index a867d4a5267..35b022c7af7 100644 --- a/storage/src/vespa/storage/distributor/maintenance/simplemaintenancescanner.h +++ b/storage/src/vespa/storage/distributor/maintenance/simplemaintenancescanner.h @@ -13,21 +13,24 @@ class SimpleMaintenanceScanner : public MaintenanceScanner { public: struct GlobalMaintenanceStats { - std::vector<uint64_t> pending; + std::array<uint64_t, MaintenanceOperation::OPERATION_COUNT> pending; - GlobalMaintenanceStats() - : pending(MaintenanceOperation::OPERATION_COUNT) + GlobalMaintenanceStats() noexcept + : pending() { } - bool operator==(const GlobalMaintenanceStats& rhs) const; - void merge(const GlobalMaintenanceStats& rhs); + bool operator==(const GlobalMaintenanceStats& rhs) const noexcept; + void merge(const GlobalMaintenanceStats& rhs) noexcept; }; struct PendingMaintenanceStats { - PendingMaintenanceStats(); + PendingMaintenanceStats() noexcept; PendingMaintenanceStats(const PendingMaintenanceStats &); - PendingMaintenanceStats &operator = (const PendingMaintenanceStats &); + PendingMaintenanceStats &operator = (const PendingMaintenanceStats &) = delete; + PendingMaintenanceStats(PendingMaintenanceStats &&) noexcept; + PendingMaintenanceStats &operator = (PendingMaintenanceStats &&) noexcept; ~PendingMaintenanceStats(); - GlobalMaintenanceStats global; + PendingMaintenanceStats reset(); + GlobalMaintenanceStats global; NodeMaintenanceStatsTracker perNodeStats; void merge(const PendingMaintenanceStats& rhs); @@ -47,14 +50,15 @@ public: const DistributorBucketSpaceRepo& bucketSpaceRepo); SimpleMaintenanceScanner(const SimpleMaintenanceScanner&) = delete; SimpleMaintenanceScanner& operator=(const SimpleMaintenanceScanner&) = delete; - ~SimpleMaintenanceScanner(); + ~SimpleMaintenanceScanner() override; ScanResult scanNext() override; - void reset() override; + PendingMaintenanceStats reset(); // TODO: move out into own interface! void prioritizeBucket(const document::Bucket &id); + // TODO Only for testing const PendingMaintenanceStats& getPendingMaintenanceStats() const noexcept { return _pendingMaintenance; } diff --git a/storage/src/vespa/storage/distributor/messagetracker.cpp b/storage/src/vespa/storage/distributor/messagetracker.cpp index 8830e5ecabc..28fbaad4619 100644 --- a/storage/src/vespa/storage/distributor/messagetracker.cpp +++ b/storage/src/vespa/storage/distributor/messagetracker.cpp @@ -3,6 +3,7 @@ #include "messagetracker.h" #include <vespa/storageapi/messageapi/bucketcommand.h> #include <vespa/storageapi/messageapi/bucketreply.h> +#include <vespa/vespalib/stllike/hash_map.hpp> #include <cinttypes> #include <vespa/log/log.h> @@ -19,10 +20,11 @@ MessageTracker::~MessageTracker() = default; void MessageTracker::flushQueue(MessageSender& sender) { - for (uint32_t i = 0; i < _commandQueue.size(); i++) { - _commandQueue[i]._msg->setAddress(api::StorageMessageAddress::create(_cluster_ctx.cluster_name_ptr(), lib::NodeType::STORAGE, _commandQueue[i]._target)); - _sentMessages[_commandQueue[i]._msg->getMsgId()] = _commandQueue[i]._target; - sender.sendCommand(_commandQueue[i]._msg); + _sentMessages.resize(_commandQueue.size()); + for (const auto & toSend : _commandQueue) { + toSend._msg->setAddress(api::StorageMessageAddress::create(_cluster_ctx.cluster_name_ptr(), lib::NodeType::STORAGE, toSend._target)); + _sentMessages[toSend._msg->getMsgId()] = toSend._target; + sender.sendCommand(toSend._msg); } _commandQueue.clear(); @@ -31,21 +33,14 @@ MessageTracker::flushQueue(MessageSender& sender) uint16_t MessageTracker::handleReply(api::BucketReply& reply) { - std::map<uint64_t, uint16_t>::iterator found = _sentMessages.find(reply.getMsgId()); - if (found == _sentMessages.end()) { + const auto found = _sentMessages.find(reply.getMsgId()); + if (found == _sentMessages.end()) [[unlikely]] { LOG(warning, "Received reply %" PRIu64 " for callback which we have no recollection of", reply.getMsgId()); return (uint16_t)-1; - } else { - uint16_t node = found->second; - _sentMessages.erase(found); - return node; } -} - -bool -MessageTracker::finished() -{ - return _sentMessages.empty(); + uint16_t node = found->second; + _sentMessages.erase(found); + return node; } } diff --git a/storage/src/vespa/storage/distributor/messagetracker.h b/storage/src/vespa/storage/distributor/messagetracker.h index 73e2461eb7a..a0234f425a0 100644 --- a/storage/src/vespa/storage/distributor/messagetracker.h +++ b/storage/src/vespa/storage/distributor/messagetracker.h @@ -4,8 +4,7 @@ #include <vespa/storage/common/cluster_context.h> #include <vespa/storage/common/messagesender.h> #include <vespa/vespalib/stllike/string.h> -#include <vector> -#include <map> +#include <vespa/vespalib/stllike/hash_map.h> namespace storage::api { class BucketCommand; @@ -18,16 +17,17 @@ class MessageTracker { public: class ToSend { public: - ToSend(std::shared_ptr<api::BucketCommand> msg, uint16_t target) noexcept : - _msg(std::move(msg)), _target(target) {}; + ToSend(std::shared_ptr<api::BucketCommand> msg, uint16_t target) noexcept + : _msg(std::move(msg)), _target(target) + {} std::shared_ptr<api::BucketCommand> _msg; uint16_t _target; }; MessageTracker(const ClusterContext& cluster_context); - MessageTracker(MessageTracker&&) = default; - MessageTracker& operator=(MessageTracker&&) = delete; + MessageTracker(MessageTracker&&) noexcept = default; + MessageTracker& operator=(MessageTracker&&) noexcept = delete; MessageTracker(const MessageTracker &) = delete; MessageTracker& operator=(const MessageTracker&) = delete; ~MessageTracker(); @@ -35,6 +35,9 @@ public: void queueCommand(std::shared_ptr<api::BucketCommand> msg, uint16_t target) { _commandQueue.emplace_back(std::move(msg), target); } + void reserve_more_commands(size_t sz) { + _commandQueue.reserve(_commandQueue.size() + sz); + } void flushQueue(MessageSender& sender); @@ -46,13 +49,15 @@ public: /** Returns true if all messages sent have been received. */ - bool finished(); + bool finished() const noexcept { + return _sentMessages.empty(); + } protected: - std::vector<ToSend> _commandQueue; + std::vector<ToSend> _commandQueue; // Keeps track of which node a message was sent to. - std::map<uint64_t, uint16_t> _sentMessages; - const ClusterContext& _cluster_ctx; + vespalib::hash_map<uint64_t, uint16_t> _sentMessages; + const ClusterContext& _cluster_ctx; }; } diff --git a/storage/src/vespa/storage/distributor/min_replica_provider.cpp b/storage/src/vespa/storage/distributor/min_replica_provider.cpp index f503672af39..52780b99948 100644 --- a/storage/src/vespa/storage/distributor/min_replica_provider.cpp +++ b/storage/src/vespa/storage/distributor/min_replica_provider.cpp @@ -5,8 +5,7 @@ namespace storage::distributor { void -merge_min_replica_stats(std::unordered_map<uint16_t, uint32_t>& dest, - const std::unordered_map<uint16_t, uint32_t>& src) +merge_min_replica_stats(MinReplicaMap & dest, const MinReplicaMap & src) { for (const auto& entry : src) { auto node_index = entry.first; diff --git a/storage/src/vespa/storage/distributor/min_replica_provider.h b/storage/src/vespa/storage/distributor/min_replica_provider.h index a4374b906fe..75d3a150d21 100644 --- a/storage/src/vespa/storage/distributor/min_replica_provider.h +++ b/storage/src/vespa/storage/distributor/min_replica_provider.h @@ -1,11 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <stdint.h> -#include <unordered_map> +#include <vespa/vespalib/stllike/hash_map.h> namespace storage::distributor { +using MinReplicaMap = vespalib::hash_map<uint16_t, uint32_t>; + class MinReplicaProvider { public: @@ -17,11 +18,10 @@ public: * Can be called at any time after registration from another thread context * and the call must thus be thread safe and data race free. */ - virtual std::unordered_map<uint16_t, uint32_t> getMinReplica() const = 0; + virtual MinReplicaMap getMinReplica() const = 0; }; -void merge_min_replica_stats(std::unordered_map<uint16_t, uint32_t>& dest, - const std::unordered_map<uint16_t, uint32_t>& src); +void merge_min_replica_stats(MinReplicaMap & dest, const MinReplicaMap & src); } diff --git a/storage/src/vespa/storage/distributor/multi_threaded_stripe_access_guard.cpp b/storage/src/vespa/storage/distributor/multi_threaded_stripe_access_guard.cpp index b00e4ce3cba..01c2875671b 100644 --- a/storage/src/vespa/storage/distributor/multi_threaded_stripe_access_guard.cpp +++ b/storage/src/vespa/storage/distributor/multi_threaded_stripe_access_guard.cpp @@ -83,7 +83,7 @@ MultiThreadedStripeAccessGuard::merge_entries_into_db(document::BucketSpace buck const lib::Distribution& distribution, const lib::ClusterState& new_state, const char* storage_up_states, - const std::unordered_set<uint16_t>& outdated_nodes, + const OutdatedNodes & outdated_nodes, const std::vector<dbtransition::Entry>& entries) { if (entries.empty()) { diff --git a/storage/src/vespa/storage/distributor/multi_threaded_stripe_access_guard.h b/storage/src/vespa/storage/distributor/multi_threaded_stripe_access_guard.h index c52a01fdded..7a58a784eda 100644 --- a/storage/src/vespa/storage/distributor/multi_threaded_stripe_access_guard.h +++ b/storage/src/vespa/storage/distributor/multi_threaded_stripe_access_guard.h @@ -46,7 +46,7 @@ public: const lib::Distribution& distribution, const lib::ClusterState& new_state, const char* storage_up_states, - const std::unordered_set<uint16_t>& outdated_nodes, + const OutdatedNodes & outdated_nodes, const std::vector<dbtransition::Entry>& entries) override; void update_read_snapshot_before_db_pruning() override; diff --git a/storage/src/vespa/storage/distributor/nodeinfo.cpp b/storage/src/vespa/storage/distributor/nodeinfo.cpp index 6bb1949d606..3e645f57393 100644 --- a/storage/src/vespa/storage/distributor/nodeinfo.cpp +++ b/storage/src/vespa/storage/distributor/nodeinfo.cpp @@ -5,14 +5,16 @@ namespace storage::distributor { -NodeInfo::NodeInfo(const framework::Clock& clock) +NodeInfo::NodeInfo(const framework::Clock& clock) noexcept : _clock(clock) {} -uint32_t NodeInfo::getPendingCount(uint16_t idx) const { +uint32_t +NodeInfo::getPendingCount(uint16_t idx) const { return getNode(idx)._pending; } -bool NodeInfo::isBusy(uint16_t idx) const { +bool +NodeInfo::isBusy(uint16_t idx) const { const SingleNodeInfo& info = getNode(idx); if (info._busyUntilTime.time_since_epoch().count() != 0) { if (_clock.getMonotonicTime() > info._busyUntilTime) { @@ -25,15 +27,18 @@ bool NodeInfo::isBusy(uint16_t idx) const { return false; } -void NodeInfo::setBusy(uint16_t idx, vespalib::duration for_duration) { +void +NodeInfo::setBusy(uint16_t idx, vespalib::duration for_duration) { getNode(idx)._busyUntilTime = _clock.getMonotonicTime() + for_duration; } -void NodeInfo::incPending(uint16_t idx) { +void +NodeInfo::incPending(uint16_t idx) { getNode(idx)._pending++; } -void NodeInfo::decPending(uint16_t idx) { +void +NodeInfo::decPending(uint16_t idx) { SingleNodeInfo& info = getNode(idx); if (info._pending > 0) { @@ -41,12 +46,14 @@ void NodeInfo::decPending(uint16_t idx) { } } -void NodeInfo::clearPending(uint16_t idx) { +void +NodeInfo::clearPending(uint16_t idx) { SingleNodeInfo& info = getNode(idx); info._pending = 0; } -NodeInfo::SingleNodeInfo& NodeInfo::getNode(uint16_t idx) { +NodeInfo::SingleNodeInfo& +NodeInfo::getNode(uint16_t idx) { const auto index_lbound = static_cast<size_t>(idx) + 1; while (_nodes.size() < index_lbound) { _nodes.emplace_back(); @@ -55,7 +62,8 @@ NodeInfo::SingleNodeInfo& NodeInfo::getNode(uint16_t idx) { return _nodes[idx]; } -const NodeInfo::SingleNodeInfo& NodeInfo::getNode(uint16_t idx) const { +const NodeInfo::SingleNodeInfo& +NodeInfo::getNode(uint16_t idx) const { const auto index_lbound = static_cast<size_t>(idx) + 1; while (_nodes.size() < index_lbound) { _nodes.emplace_back(); diff --git a/storage/src/vespa/storage/distributor/nodeinfo.h b/storage/src/vespa/storage/distributor/nodeinfo.h index 7f0716d7804..446739ca7e9 100644 --- a/storage/src/vespa/storage/distributor/nodeinfo.h +++ b/storage/src/vespa/storage/distributor/nodeinfo.h @@ -17,30 +17,24 @@ namespace storage::distributor { class NodeInfo { public: - explicit NodeInfo(const framework::Clock& clock); - + explicit NodeInfo(const framework::Clock& clock) noexcept; uint32_t getPendingCount(uint16_t idx) const; - bool isBusy(uint16_t idx) const; - void setBusy(uint16_t idx, vespalib::duration for_duration); - void incPending(uint16_t idx); - void decPending(uint16_t idx); - void clearPending(uint16_t idx); private: struct SingleNodeInfo { - SingleNodeInfo() : _pending(0), _busyUntilTime() {} + SingleNodeInfo() noexcept : _pending(0), _busyUntilTime() {} - uint32_t _pending; + uint32_t _pending; mutable vespalib::steady_time _busyUntilTime; }; mutable std::vector<SingleNodeInfo> _nodes; - const framework::Clock& _clock; + const framework::Clock& _clock; const SingleNodeInfo& getNode(uint16_t idx) const; SingleNodeInfo& getNode(uint16_t idx); diff --git a/storage/src/vespa/storage/distributor/operations/external/putoperation.cpp b/storage/src/vespa/storage/distributor/operations/external/putoperation.cpp index 8c6fdb314f3..86ea9a559f5 100644 --- a/storage/src/vespa/storage/distributor/operations/external/putoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/external/putoperation.cpp @@ -67,8 +67,8 @@ PutOperation::insertDatabaseEntryAndScheduleCreateBucket(const OperationTargetLi assert(!multipleBuckets); (void) multipleBuckets; BucketDatabase::Entry entry(_bucket_space.getBucketDatabase().get(lastBucket)); - std::vector<uint16_t> idealState( - _bucket_space.get_ideal_service_layer_nodes_bundle(lastBucket).get_available_nodes()); + const std::vector<uint16_t> & idealState = _bucket_space.get_ideal_service_layer_nodes_bundle( + lastBucket).available_nodes(); active = ActiveCopy::calculate(idealState, _bucket_space.getDistribution(), entry, _op_ctx.distributor_config().max_activation_inhibited_out_of_sync_groups()); LOG(debug, "Active copies for bucket %s: %s", entry.getBucketId().toString().c_str(), active.toString().c_str()); @@ -211,11 +211,11 @@ void PutOperation::start_direct_put_dispatch(DistributorStripeMessageSender& sen } if (!createBucketBatch.empty()) { - _tracker.queueMessageBatch(createBucketBatch); + _tracker.queueMessageBatch(std::move(createBucketBatch)); } std::vector<PersistenceMessageTracker::ToSend> putBatch; - + putBatch.reserve(targets.size()); // Now send PUTs for (const auto& target : targets) { sendPutToBucketOnNode(_msg->getBucket().getBucketSpace(), target.getBucketId(), @@ -223,7 +223,7 @@ void PutOperation::start_direct_put_dispatch(DistributorStripeMessageSender& sen } if (!putBatch.empty()) { - _tracker.queueMessageBatch(putBatch); + _tracker.queueMessageBatch(std::move(putBatch)); } else { const char* error = "Can't store document: No storage nodes available"; LOG(debug, "%s", error); diff --git a/storage/src/vespa/storage/distributor/operations/external/removelocationoperation.cpp b/storage/src/vespa/storage/distributor/operations/external/removelocationoperation.cpp index dd6e1e93791..5f52a8208fc 100644 --- a/storage/src/vespa/storage/distributor/operations/external/removelocationoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/external/removelocationoperation.cpp @@ -79,13 +79,11 @@ RemoveLocationOperation::onStart(DistributorStripeMessageSender& sender) std::vector<uint16_t> nodes = e->getNodes(); for (uint32_t i = 0; i < nodes.size(); i++) { - std::shared_ptr<api::RemoveLocationCommand> command( - new api::RemoveLocationCommand( - _msg->getDocumentSelection(), - document::Bucket(_msg->getBucket().getBucketSpace(), e.getBucketId()))); + auto command = std::make_shared<api::RemoveLocationCommand>(_msg->getDocumentSelection(), + document::Bucket(_msg->getBucket().getBucketSpace(), e.getBucketId())); copyMessageSettings(*_msg, *command); - _tracker.queueCommand(command, nodes[i]); + _tracker.queueCommand(std::move(command), nodes[i]); sent = true; } } diff --git a/storage/src/vespa/storage/distributor/operations/external/removeoperation.cpp b/storage/src/vespa/storage/distributor/operations/external/removeoperation.cpp index 96182b0744f..42d8e318f47 100644 --- a/storage/src/vespa/storage/distributor/operations/external/removeoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/external/removeoperation.cpp @@ -65,9 +65,7 @@ void RemoveOperation::start_conditional_remove(DistributorStripeMessageSender& s void RemoveOperation::start_direct_remove_dispatch(DistributorStripeMessageSender& sender) { LOG(spam, "Started remove on document %s", _msg->getDocumentId().toString().c_str()); - document::BucketId bucketId( - _node_ctx.bucket_id_factory().getBucketId( - _msg->getDocumentId())); + document::BucketId bucketId(_node_ctx.bucket_id_factory().getBucketId(_msg->getDocumentId())); std::vector<BucketDatabase::Entry> entries; _bucket_space.getBucketDatabase().getParents(bucketId, entries); @@ -79,8 +77,7 @@ void RemoveOperation::start_direct_remove_dispatch(DistributorStripeMessageSende messages.reserve(e->getNodeCount()); for (uint32_t i = 0; i < e->getNodeCount(); i++) { auto command = std::make_shared<api::RemoveCommand>(document::Bucket(_msg->getBucket().getBucketSpace(), e.getBucketId()), - _msg->getDocumentId(), - _msg->getTimestamp()); + _msg->getDocumentId(), _msg->getTimestamp()); copyMessageSettings(*_msg, *command); command->getTrace().setLevel(_msg->getTrace().getLevel()); @@ -90,7 +87,7 @@ void RemoveOperation::start_direct_remove_dispatch(DistributorStripeMessageSende sent = true; } - _tracker.queueMessageBatch(messages); + _tracker.queueMessageBatch(std::move(messages)); } if (!sent) { diff --git a/storage/src/vespa/storage/distributor/operations/external/updateoperation.cpp b/storage/src/vespa/storage/distributor/operations/external/updateoperation.cpp index 8988f2589ce..f43a6092372 100644 --- a/storage/src/vespa/storage/distributor/operations/external/updateoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/external/updateoperation.cpp @@ -106,19 +106,18 @@ UpdateOperation::onStart(DistributorStripeMessageSender& sender) const std::vector<uint16_t>& nodes = entry->getNodes(); std::vector<MessageTracker::ToSend> messages; + messages.reserve(nodes.size()); for (uint16_t node : nodes) { - auto command = std::make_shared<api::UpdateCommand>( - document::Bucket(_msg->getBucket().getBucketSpace(), entry.getBucketId()), - _msg->getUpdate(), - _msg->getTimestamp()); + auto command = std::make_shared<api::UpdateCommand>(document::Bucket(_msg->getBucket().getBucketSpace(), entry.getBucketId()), + _msg->getUpdate(), _msg->getTimestamp()); copyMessageSettings(*_msg, *command); command->setOldTimestamp(_msg->getOldTimestamp()); command->setCondition(_msg->getCondition()); messages.emplace_back(std::move(command), node); } - _tracker.queueMessageBatch(messages); + _tracker.queueMessageBatch(std::move(messages)); } _tracker.flushQueue(sender); diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.cpp b/storage/src/vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.cpp index 5599f9fb51e..2e6d0e95ec9 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.cpp @@ -87,7 +87,7 @@ void GarbageCollectionOperation::send_current_phase_remove_locations(Distributor command->setPriority((_phase != Phase::WriteRemovesPhase) ? _priority : _manager->operation_context().distributor_config().default_external_feed_priority()); - _tracker.queueCommand(command, nodes[i]); + _tracker.queueCommand(std::move(command), nodes[i]); } _tracker.flushQueue(sender); } diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/joinoperation.cpp b/storage/src/vespa/storage/distributor/operations/idealstate/joinoperation.cpp index 0e9873f3434..616c4962dca 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/joinoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/idealstate/joinoperation.cpp @@ -81,12 +81,11 @@ JoinOperation::enqueueJoinMessagePerTargetNode( return false; } for (const auto& node : nodeToBuckets) { - std::shared_ptr<api::JoinBucketsCommand> msg( - new api::JoinBucketsCommand(getBucket())); + auto msg = std::make_shared<api::JoinBucketsCommand>(getBucket()); msg->getSourceBuckets() = node.second; msg->setTimeout(MAX_TIMEOUT); setCommandMeta(*msg); - _tracker.queueCommand(msg, node.first); + _tracker.queueCommand(std::move(msg), node.first); } return true; } diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/removebucketoperation.cpp b/storage/src/vespa/storage/distributor/operations/idealstate/removebucketoperation.cpp index e46ccebffba..7bec6bbe53a 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/removebucketoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/idealstate/removebucketoperation.cpp @@ -24,16 +24,11 @@ RemoveBucketOperation::onStartInternal(DistributorStripeMessageSender& sender) uint16_t node = getNodes()[i]; const BucketCopy* copy(entry->getNode(node)); if (!copy) { - LOG(debug, "Node %u was removed between scheduling remove " - "operation and starting it; not sending DeleteBucket to it", - node); + LOG(debug, "Node %u was removed between scheduling remove operation and starting it; not sending DeleteBucket to it", node); continue; } - LOG(debug, "Sending DeleteBucket for %s to node %u", - getBucketId().toString().c_str(), - node); - std::shared_ptr<api::DeleteBucketCommand> msg( - new api::DeleteBucketCommand(getBucket())); + LOG(debug, "Sending DeleteBucket for %s to node %u", getBucketId().toString().c_str(), node); + auto msg = std::make_shared<api::DeleteBucketCommand>(getBucket()); setCommandMeta(*msg); msg->setBucketInfo(copy->getBucketInfo()); msgs.push_back(std::make_pair(node, msg)); @@ -42,8 +37,8 @@ RemoveBucketOperation::onStartInternal(DistributorStripeMessageSender& sender) _ok = true; if (!getNodes().empty()) { _manager->operation_context().remove_nodes_from_bucket_database(getBucket(), getNodes()); - for (uint32_t i = 0; i < msgs.size(); ++i) { - _tracker.queueCommand(msgs[i].second, msgs[i].first); + for (auto & msg : msgs) { + _tracker.queueCommand(std::move(msg.second), msg.first); } _tracker.flushQueue(sender); } diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/setbucketstateoperation.cpp b/storage/src/vespa/storage/distributor/operations/idealstate/setbucketstateoperation.cpp index 00906d22ea4..9547bee6583 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/setbucketstateoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/idealstate/setbucketstateoperation.cpp @@ -26,11 +26,9 @@ SetBucketStateOperation::enqueueSetBucketStateCommand(uint16_t node, bool active active ? api::SetBucketStateCommand::ACTIVE : api::SetBucketStateCommand::INACTIVE); - LOG(debug, "Enqueuing %s for %s to node %u", - active ? "Activate" : "Deactivate", - getBucketId().toString().c_str(), node); + LOG(debug, "Enqueuing %s for %s to node %u", active ? "Activate" : "Deactivate", getBucketId().toString().c_str(), node); setCommandMeta(*msg); - _tracker.queueCommand(msg, node); + _tracker.queueCommand(std::move(msg), node); } bool diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/splitoperation.cpp b/storage/src/vespa/storage/distributor/operations/idealstate/splitoperation.cpp index 8e64fb227a7..d704a42e96b 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/splitoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/idealstate/splitoperation.cpp @@ -35,7 +35,7 @@ SplitOperation::onStart(DistributorStripeMessageSender& sender) msg->setMinByteSize(_splitSize); msg->setTimeout(MAX_TIMEOUT); setCommandMeta(*msg); - _tracker.queueCommand(msg, entry->getNodeRef(i).getNode()); + _tracker.queueCommand(std::move(msg), entry->getNodeRef(i).getNode()); _ok = true; } diff --git a/storage/src/vespa/storage/distributor/operationtargetresolverimpl.cpp b/storage/src/vespa/storage/distributor/operationtargetresolverimpl.cpp index 6a9d7e0e6da..736d8c692e3 100644 --- a/storage/src/vespa/storage/distributor/operationtargetresolverimpl.cpp +++ b/storage/src/vespa/storage/distributor/operationtargetresolverimpl.cpp @@ -23,9 +23,8 @@ make_node_list(const std::vector<uint16_t>& nodes) } -BucketInstance::BucketInstance( - const document::BucketId& id, const api::BucketInfo& info, - lib::Node node, uint16_t idealLocationPriority, bool trusted, bool exist) +BucketInstance::BucketInstance(const document::BucketId& id, const api::BucketInfo& info, lib::Node node, + uint16_t idealLocationPriority, bool trusted, bool exist) : _bucket(id), _info(info), _node(node), _idealLocationPriority(idealLocationPriority), _trusted(trusted), _exist(exist) { @@ -39,12 +38,8 @@ BucketInstance::print(vespalib::asciistream& out, const PrintProperties&) const std::ostringstream ost; ost << std::hex << _bucket.getId(); - out << "(" << ost.str() << ", " - << infoString << ", node " << _node.getIndex() - << ", ideal " << _idealLocationPriority - << (_trusted ? ", trusted" : "") - << (_exist ? "" : ", new copy") - << ")"; + out << "(" << ost.str() << ", " << infoString << ", node " << _node.getIndex() << ", ideal " << _idealLocationPriority + << (_trusted ? ", trusted" : "") << (_exist ? "" : ", new copy") << ")"; } bool @@ -56,15 +51,12 @@ BucketInstanceList::contains(lib::Node node) const { } void -BucketInstanceList::add(BucketDatabase::Entry& e, - const lib::IdealNodeList& idealState) +BucketInstanceList::add(const BucketDatabase::Entry& e, const lib::IdealNodeList& idealState) { for (uint32_t i = 0; i < e.getBucketInfo().getNodeCount(); ++i) { const BucketCopy& copy(e.getBucketInfo().getNodeRef(i)); lib::Node node(lib::NodeType::STORAGE, copy.getNode()); - _instances.push_back(BucketInstance( - e.getBucketId(), copy.getBucketInfo(), node, - idealState.indexOf(node), copy.trusted())); + _instances.emplace_back(e.getBucketId(), copy.getBucketInfo(), node, idealState.indexOf(node), copy.trusted()); } } @@ -73,9 +65,9 @@ BucketInstanceList::populate(const document::BucketId& specificId, const Distrib { std::vector<BucketDatabase::Entry> entries; db.getParents(specificId, entries); - for (uint32_t i=0; i<entries.size(); ++i) { - lib::IdealNodeList idealNodes(make_node_list(distributor_bucket_space.get_ideal_service_layer_nodes_bundle(entries[i].getBucketId()).get_available_nonretired_or_maintenance_nodes())); - add(entries[i], idealNodes); + for (const auto & entry : entries) { + lib::IdealNodeList idealNodes(make_node_list(distributor_bucket_space.get_ideal_service_layer_nodes_bundle(entry.getBucketId()).available_nonretired_or_maintenance_nodes())); + add(entry, idealNodes); } } @@ -102,40 +94,34 @@ BucketInstanceList::limitToRedundancyCopies(uint16_t redundancy) } document::BucketId -BucketInstanceList::leastSpecificLeafBucketInSubtree( - const document::BucketId& candidateId, - const document::BucketId& mostSpecificId, - const BucketDatabase& db) const +BucketInstanceList::leastSpecificLeafBucketInSubtree(const document::BucketId& candidateId, + const document::BucketId& mostSpecificId, + const BucketDatabase& db) const { assert(candidateId.contains(mostSpecificId)); document::BucketId treeNode = candidateId; // treeNode may reach at most 58 bits since buckets at 58 bits by definition // cannot have any children. while (db.childCount(treeNode) != 0) { - treeNode = document::BucketId(treeNode.getUsedBits() + 1, - mostSpecificId.getRawId()); + treeNode = document::BucketId(treeNode.getUsedBits() + 1, mostSpecificId.getRawId()); } assert(treeNode.contains(mostSpecificId)); return treeNode; } void -BucketInstanceList::extendToEnoughCopies( - const DistributorBucketSpace& distributor_bucket_space, - const BucketDatabase& db, - const document::BucketId& targetIfNonPreExisting, - const document::BucketId& mostSpecificId) +BucketInstanceList::extendToEnoughCopies(const DistributorBucketSpace& distributor_bucket_space, + const BucketDatabase& db, + const document::BucketId& targetIfNonPreExisting, + const document::BucketId& mostSpecificId) { - document::BucketId newTarget(_instances.empty() ? targetIfNonPreExisting - : _instances[0]._bucket); + document::BucketId newTarget(_instances.empty() ? targetIfNonPreExisting : _instances[0]._bucket); newTarget = leastSpecificLeafBucketInSubtree(newTarget, mostSpecificId, db); - lib::IdealNodeList idealNodes(make_node_list(distributor_bucket_space.get_ideal_service_layer_nodes_bundle(newTarget).get_available_nonretired_nodes())); + lib::IdealNodeList idealNodes(make_node_list(distributor_bucket_space.get_ideal_service_layer_nodes_bundle(newTarget).available_nonretired_nodes())); for (uint32_t i=0; i<idealNodes.size(); ++i) { if (!contains(idealNodes[i])) { - _instances.push_back(BucketInstance( - newTarget, api::BucketInfo(), idealNodes[i], - i, false, false)); + _instances.emplace_back(newTarget, api::BucketInfo(), idealNodes[i], i, false, false); } } } @@ -188,22 +174,17 @@ struct InstanceOrder { } // anonymous BucketInstanceList -OperationTargetResolverImpl::getAllInstances(OperationType type, - const document::BucketId& id) +OperationTargetResolverImpl::getAllInstances(OperationType type, const document::BucketId& id) { BucketInstanceList instances; if (type == PUT) { instances.populate(id, _distributor_bucket_space, _bucketDatabase); instances.sort(InstanceOrder()); instances.removeNodeDuplicates(); - instances.extendToEnoughCopies( - _distributor_bucket_space, - _bucketDatabase, - _bucketDatabase.getAppropriateBucket(_minUsedBucketBits, id), - id); + instances.extendToEnoughCopies(_distributor_bucket_space, _bucketDatabase, + _bucketDatabase.getAppropriateBucket(_minUsedBucketBits, id), id); } else { - throw vespalib::IllegalArgumentException( - "Unsupported operation type given", VESPA_STRLOC); + throw vespalib::IllegalArgumentException("Unsupported operation type given", VESPA_STRLOC); } return instances; } diff --git a/storage/src/vespa/storage/distributor/operationtargetresolverimpl.h b/storage/src/vespa/storage/distributor/operationtargetresolverimpl.h index 9ff65475fa4..0caeee466e0 100644 --- a/storage/src/vespa/storage/distributor/operationtargetresolverimpl.h +++ b/storage/src/vespa/storage/distributor/operationtargetresolverimpl.h @@ -65,7 +65,7 @@ public: const document::BucketId& mostSpecificId); void populate(const document::BucketId&, const DistributorBucketSpace&, BucketDatabase&); - void add(BucketDatabase::Entry& e, const lib::IdealNodeList& idealState); + void add(const BucketDatabase::Entry& e, const lib::IdealNodeList& idealState); template <typename Order> void sort(const Order& order) { diff --git a/storage/src/vespa/storage/distributor/outdated_nodes.h b/storage/src/vespa/storage/distributor/outdated_nodes.h index cef799ee4aa..d014a3074a4 100644 --- a/storage/src/vespa/storage/distributor/outdated_nodes.h +++ b/storage/src/vespa/storage/distributor/outdated_nodes.h @@ -2,10 +2,10 @@ #pragma once -#include <unordered_set> +#include <vespa/vespalib/stllike/hash_set.h> namespace storage::distributor::dbtransition { -using OutdatedNodes = std::unordered_set<uint16_t>; +using OutdatedNodes = vespalib::hash_set<uint16_t>; } diff --git a/storage/src/vespa/storage/distributor/pending_bucket_space_db_transition.cpp b/storage/src/vespa/storage/distributor/pending_bucket_space_db_transition.cpp index 62de3b50b51..19cc7bc522f 100644 --- a/storage/src/vespa/storage/distributor/pending_bucket_space_db_transition.cpp +++ b/storage/src/vespa/storage/distributor/pending_bucket_space_db_transition.cpp @@ -113,10 +113,7 @@ PendingBucketSpaceDbTransition::DbMerger::removeCopiesFromNodesThatWereRequested && (info.getTimestamp() < _creation_timestamp) && e->removeNode(entryNode, TrustedUpdate::DEFER)) { - LOG(spam, - "Removed bucket %s from node %d", - bucketId.toString().c_str(), - entryNode); + LOG(spam, "Removed bucket %s from node %d", bucketId.toString().c_str(), entryNode); updated = true; // After removing current node, getNodeRef(i) will point to the _next_ node, so don't increment `i`. } else { @@ -391,8 +388,7 @@ PendingBucketSpaceDbTransition::markAllAvailableNodesAsRequiringRequest() } void -PendingBucketSpaceDbTransition::addAdditionalNodesToOutdatedSet( - const std::unordered_set<uint16_t>& nodes) +PendingBucketSpaceDbTransition::addAdditionalNodesToOutdatedSet(const OutdatedNodes & nodes) { const uint16_t nodeCount(newStateStorageNodeCount()); for (uint16_t node : nodes) { diff --git a/storage/src/vespa/storage/distributor/pending_bucket_space_db_transition.h b/storage/src/vespa/storage/distributor/pending_bucket_space_db_transition.h index bce0c9bdc93..9fb6e4ed315 100644 --- a/storage/src/vespa/storage/distributor/pending_bucket_space_db_transition.h +++ b/storage/src/vespa/storage/distributor/pending_bucket_space_db_transition.h @@ -76,7 +76,7 @@ public: const lib::Distribution& _distribution; const lib::ClusterState& _new_state; const char* _storage_up_states; - const std::unordered_set<uint16_t>& _outdated_nodes; // TODO hash_set + const OutdatedNodes & _outdated_nodes; // TODO hash_set const std::vector<dbtransition::Entry>& _entries; uint32_t _iter; public: @@ -84,7 +84,7 @@ public: const lib::Distribution& distribution, const lib::ClusterState& new_state, const char* storage_up_states, - const std::unordered_set<uint16_t>& outdated_nodes, + const OutdatedNodes & outdated_nodes, const std::vector<dbtransition::Entry>& entries) : _creation_timestamp(creation_timestamp), _distribution(distribution), diff --git a/storage/src/vespa/storage/distributor/pendingmessagetracker.cpp b/storage/src/vespa/storage/distributor/pendingmessagetracker.cpp index 5b8fa6b69e3..7b3cdacf702 100644 --- a/storage/src/vespa/storage/distributor/pendingmessagetracker.cpp +++ b/storage/src/vespa/storage/distributor/pendingmessagetracker.cpp @@ -17,6 +17,7 @@ PendingMessageTracker::PendingMessageTracker(framework::ComponentRegister& cr, u _nodeInfo(_component.getClock()), _nodeBusyDuration(60s), _deferred_read_tasks(), + _trackTime(false), _lock() { _component.registerStatusPage(*this); @@ -69,6 +70,13 @@ pairAsRange(Pair pair) return PairAsRange<Pair>(std::move(pair)); } +document::Bucket +getBucket(const api::StorageMessage & msg) { + return (msg.getType() != api::MessageType::REQUESTBUCKETINFO) + ? msg.getBucket() + : document::Bucket(msg.getBucket().getBucketSpace(), dynamic_cast<const api::RequestBucketInfoCommand&>(msg).super_bucket_id()); +} + } std::vector<uint64_t> @@ -91,17 +99,19 @@ PendingMessageTracker::clearMessagesForNode(uint16_t node) void PendingMessageTracker::insert(const std::shared_ptr<api::StorageMessage>& msg) { - std::lock_guard guard(_lock); if (msg->getAddress()) { // TODO STRIPE reevaluate if getBucket() on RequestBucketInfo msgs should transparently return superbucket..! - document::Bucket bucket = (msg->getType() != api::MessageType::REQUESTBUCKETINFO) - ? msg->getBucket() - : document::Bucket(msg->getBucket().getBucketSpace(), - dynamic_cast<api::RequestBucketInfoCommand&>(*msg).super_bucket_id()); - _messages.emplace(currentTime(), msg->getType().getId(), msg->getPriority(), msg->getMsgId(), - bucket, msg->getAddress()->getIndex()); - - _nodeInfo.incPending(msg->getAddress()->getIndex()); + document::Bucket bucket = getBucket(*msg); + { + // We will not start tracking time until we have been asked for html at least once. + // Time tracking is only used for presenting pending messages for debugging. + TimePoint now = (_trackTime.load(std::memory_order_relaxed)) ? currentTime() : TimePoint(); + std::lock_guard guard(_lock); + _messages.emplace(now, msg->getType().getId(), msg->getPriority(), msg->getMsgId(), + bucket, msg->getAddress()->getIndex()); + + _nodeInfo.incPending(msg->getAddress()->getIndex()); + } LOG(debug, "Sending message %s with id %" PRIu64 " to %s", msg->toString().c_str(), msg->getMsgId(), msg->getAddress()->toString().c_str()); @@ -111,15 +121,13 @@ PendingMessageTracker::insert(const std::shared_ptr<api::StorageMessage>& msg) document::Bucket PendingMessageTracker::reply(const api::StorageReply& r) { - std::unique_lock guard(_lock); document::Bucket bucket; - LOG(debug, "Got reply: %s", r.toString().c_str()); uint64_t msgId = r.getMsgId(); + std::unique_lock guard(_lock); MessagesByMsgId& msgs = boost::multi_index::get<0>(_messages); MessagesByMsgId::iterator iter = msgs.find(msgId); - if (iter != msgs.end()) { bucket = iter->bucket; _nodeInfo.decPending(r.getAddress()->getIndex()); @@ -127,7 +135,6 @@ PendingMessageTracker::reply(const api::StorageReply& r) if (code == api::ReturnCode::BUSY || code == api::ReturnCode::TIMEOUT) { _nodeInfo.setBusy(r.getAddress()->getIndex(), _nodeBusyDuration); } - LOG(debug, "Erased message with id %" PRIu64 " for bucket %s", msgId, bucket.toString().c_str()); msgs.erase(msgId); auto deferred_tasks = get_deferred_ops_if_bucket_writes_drained(bucket); // Deferred tasks may try to send messages, which in turn will invoke the PendingMessageTracker. @@ -139,6 +146,7 @@ PendingMessageTracker::reply(const api::StorageReply& r) for (auto& task : deferred_tasks) { task->run(TaskRunState::OK); } + LOG(debug, "Erased message with id %" PRIu64 " for bucket %s", msgId, bucket.toString().c_str()); } return bucket; @@ -328,6 +336,7 @@ PendingMessageTracker::getStatusPerNode(std::ostream& out) const void PendingMessageTracker::reportHtmlStatus(std::ostream& out, const framework::HttpUrlPath& path) const { + _trackTime.store(true, std::memory_order_relaxed); if (!path.hasAttribute("order")) { getStatusStartPage(out); } else if (path.getAttribute("order") == "bucket") { diff --git a/storage/src/vespa/storage/distributor/pendingmessagetracker.h b/storage/src/vespa/storage/distributor/pendingmessagetracker.h index fb672d5ee31..4b5655d3f3c 100644 --- a/storage/src/vespa/storage/distributor/pendingmessagetracker.h +++ b/storage/src/vespa/storage/distributor/pendingmessagetracker.h @@ -178,11 +178,12 @@ private: document::Bucket::hash >; - Messages _messages; - framework::Component _component; - NodeInfo _nodeInfo; - vespalib::duration _nodeBusyDuration; - DeferredBucketTaskMap _deferred_read_tasks; + Messages _messages; + framework::Component _component; + NodeInfo _nodeInfo; + vespalib::duration _nodeBusyDuration; + DeferredBucketTaskMap _deferred_read_tasks; + mutable std::atomic<bool> _trackTime; // Since distributor is currently single-threaded, this will only // contend when status page is being accessed. It is, however, required diff --git a/storage/src/vespa/storage/distributor/persistencemessagetracker.cpp b/storage/src/vespa/storage/distributor/persistencemessagetracker.cpp index a30663bde2f..a4295613fd2 100644 --- a/storage/src/vespa/storage/distributor/persistencemessagetracker.cpp +++ b/storage/src/vespa/storage/distributor/persistencemessagetracker.cpp @@ -65,9 +65,7 @@ PersistenceMessageTrackerImpl::fail(MessageSender& sender, const api::ReturnCode } uint16_t -PersistenceMessageTrackerImpl::receiveReply( - MessageSender& sender, - api::BucketInfoReply& reply) +PersistenceMessageTrackerImpl::receiveReply(MessageSender& sender, api::BucketInfoReply& reply) { uint16_t node = handleReply(reply); @@ -79,9 +77,7 @@ PersistenceMessageTrackerImpl::receiveReply( } void -PersistenceMessageTrackerImpl::revert( - MessageSender& sender, - const std::vector<BucketNodePair>& revertNodes) +PersistenceMessageTrackerImpl::revert(MessageSender& sender, const std::vector<BucketNodePair>& revertNodes) { if (_revertTimestamp != 0) { // Since we're reverting, all received bucket info is voided. @@ -101,15 +97,18 @@ PersistenceMessageTrackerImpl::revert( } void -PersistenceMessageTrackerImpl::queueMessageBatch(const std::vector<MessageTracker::ToSend>& messages) { +PersistenceMessageTrackerImpl::queueMessageBatch(std::vector<MessageTracker::ToSend> messages) { _messageBatches.emplace_back(); - for (const auto & message : messages) { + auto & batch = _messageBatches.back(); + batch.reserve(messages.size()); + reserve_more_commands(messages.size()); + for (auto & message : messages) { if (_reply) { message._msg->getTrace().setLevel(_reply->getTrace().getLevel()); } - _messageBatches.back().push_back(message._msg->getMsgId()); - queueCommand(message._msg, message._target); + batch.push_back(message._msg->getMsgId()); + queueCommand(std::move(message._msg), message._target); } } @@ -153,24 +152,18 @@ PersistenceMessageTrackerImpl::canSendReplyEarly() const } void -PersistenceMessageTrackerImpl::addBucketInfoFromReply( - uint16_t node, - const api::BucketInfoReply& reply) +PersistenceMessageTrackerImpl::addBucketInfoFromReply(uint16_t node, const api::BucketInfoReply& reply) { document::Bucket bucket(reply.getBucket()); const api::BucketInfo& bucketInfo(reply.getBucketInfo()); if (reply.hasBeenRemapped()) { LOG(debug, "Bucket %s: Received remapped bucket info %s from node %d", - bucket.toString().c_str(), - bucketInfo.toString().c_str(), - node); + bucket.toString().c_str(), bucketInfo.toString().c_str(), node); _remapBucketInfo[bucket].emplace_back(_op_ctx.generate_unique_timestamp(), node, bucketInfo); } else { LOG(debug, "Bucket %s: Received bucket info %s from node %d", - bucket.toString().c_str(), - bucketInfo.toString().c_str(), - node); + bucket.toString().c_str(), bucketInfo.toString().c_str(), node); _bucketInfo[bucket].emplace_back(_op_ctx.generate_unique_timestamp(), node, bucketInfo); } } @@ -179,17 +172,12 @@ void PersistenceMessageTrackerImpl::logSuccessfulReply(uint16_t node, const api::BucketInfoReply& reply) const { LOG(spam, "Bucket %s: Received successful reply %s", - reply.getBucketId().toString().c_str(), - reply.toString().c_str()); + reply.getBucketId().toString().c_str(), reply.toString().c_str()); if (!reply.getBucketInfo().valid()) { - LOG(error, - "Reply %s from node %d contained invalid bucket " - "information %s. This is a bug! Please report " - "this to the Vespa team", - reply.toString().c_str(), - node, - reply.getBucketInfo().toString().c_str()); + LOG(error, "Reply %s from node %d contained invalid bucket information %s. This is a bug! " + "Please report this to the Vespa team", + reply.toString().c_str(), node, reply.getBucketInfo().toString().c_str()); } } @@ -233,12 +221,8 @@ void PersistenceMessageTrackerImpl::updateFailureResult(const api::BucketInfoReply& reply) { LOG(debug, "Bucket %s: Received failed reply %s with result %s", - reply.getBucketId().toString().c_str(), - reply.toString().c_str(), - reply.getResult().toString().c_str()); - if (reply.getResult().getResult() > - _reply->getResult().getResult()) - { + reply.getBucketId().toString().c_str(), reply.toString().c_str(), reply.getResult().toString().c_str()); + if (reply.getResult().getResult() > _reply->getResult().getResult()) { _reply->setResult(reply.getResult()); } @@ -246,12 +230,9 @@ PersistenceMessageTrackerImpl::updateFailureResult(const api::BucketInfoReply& r } void -PersistenceMessageTrackerImpl::handleCreateBucketReply( - api::BucketInfoReply& reply, - uint16_t node) +PersistenceMessageTrackerImpl::handleCreateBucketReply(api::BucketInfoReply& reply, uint16_t node) { - LOG(spam, "Received CreateBucket reply for %s from node %u", - reply.getBucketId().toString().c_str(), node); + LOG(spam, "Received CreateBucket reply for %s from node %u", reply.getBucketId().toString().c_str(), node); if (!reply.getResult().success() && reply.getResult().getResult() != api::ReturnCode::EXISTS) { @@ -268,9 +249,7 @@ PersistenceMessageTrackerImpl::handleCreateBucketReply( } void -PersistenceMessageTrackerImpl::handlePersistenceReply( - api::BucketInfoReply& reply, - uint16_t node) +PersistenceMessageTrackerImpl::handlePersistenceReply(api::BucketInfoReply& reply, uint16_t node) { ++_n_persistence_replies_total; if (reply.getBucketInfo().valid()) { @@ -295,10 +274,7 @@ PersistenceMessageTrackerImpl::transfer_trace_state_to_reply() } void -PersistenceMessageTrackerImpl::updateFromReply( - MessageSender& sender, - api::BucketInfoReply& reply, - uint16_t node) +PersistenceMessageTrackerImpl::updateFromReply(MessageSender& sender, api::BucketInfoReply& reply, uint16_t node) { _trace.addChild(reply.steal_trace()); diff --git a/storage/src/vespa/storage/distributor/persistencemessagetracker.h b/storage/src/vespa/storage/distributor/persistencemessagetracker.h index 923ecf45649..9b06547dd98 100644 --- a/storage/src/vespa/storage/distributor/persistencemessagetracker.h +++ b/storage/src/vespa/storage/distributor/persistencemessagetracker.h @@ -8,7 +8,6 @@ #include <vespa/storageapi/messageapi/bucketinfocommand.h> #include <vespa/storageapi/messageapi/bucketinforeply.h> - namespace storage::distributor { struct PersistenceMessageTracker { @@ -16,7 +15,7 @@ struct PersistenceMessageTracker { using ToSend = MessageTracker::ToSend; virtual void fail(MessageSender&, const api::ReturnCode&) = 0; - virtual void queueMessageBatch(const std::vector<ToSend>&) = 0; + virtual void queueMessageBatch(std::vector<ToSend> messages) = 0; virtual uint16_t receiveReply(MessageSender&, api::BucketInfoReply&) = 0; virtual std::shared_ptr<api::BucketInfoReply>& getReply() = 0; virtual void updateFromReply(MessageSender&, api::BucketInfoReply&, uint16_t node) = 0; @@ -65,7 +64,7 @@ public: have at most (messages.size() - initial redundancy) messages left in the queue and have it's first message be done. */ - void queueMessageBatch(const std::vector<MessageTracker::ToSend>& messages) override; + void queueMessageBatch(std::vector<MessageTracker::ToSend> messages) override; private: using MessageBatch = std::vector<uint64_t>; diff --git a/storage/src/vespa/storage/distributor/statechecker.cpp b/storage/src/vespa/storage/distributor/statechecker.cpp index eaff1f0b780..cd8b6e934d4 100644 --- a/storage/src/vespa/storage/distributor/statechecker.cpp +++ b/storage/src/vespa/storage/distributor/statechecker.cpp @@ -4,6 +4,7 @@ #include "distributor_stripe_component.h" #include <vespa/vdslib/distribution/distribution.h> #include <vespa/vdslib/state/clusterstate.h> +#include <vespa/vespalib/stllike/hash_set_insert.hpp> #include <vespa/log/log.h> LOG_SETUP(".distributor.statechecker"); @@ -50,13 +51,11 @@ public: StateChecker::Result StateChecker::Result::noMaintenanceNeeded() { - return Result(std::unique_ptr<ResultImpl>()); + return Result({}); } StateChecker::Result -StateChecker::Result::createStoredResult( - IdealStateOperation::UP operation, - MaintenancePriority::Priority priority) +StateChecker::Result::createStoredResult(IdealStateOperation::UP operation, MaintenancePriority::Priority priority) { return Result(std::make_unique<StoredResultImpl>(std::move(operation), MaintenancePriority(priority))); } @@ -73,15 +72,13 @@ StateChecker::Context::Context(const DistributorNodeContext& node_ctx_in, distributorConfig(op_ctx_in.distributor_config()), distribution(distributorBucketSpace.getDistribution()), gcTimeCalculator(op_ctx_in.bucket_id_hasher(), distributorConfig.getGarbageCollectionInterval()), + idealStateBundle(distributorBucketSpace.get_ideal_service_layer_nodes_bundle(bucket.getBucketId())), node_ctx(node_ctx_in), op_ctx(op_ctx_in), db(distributorBucketSpace.getBucketDatabase()), stats(statsTracker), merges_inhibited_in_bucket_space(distributorBucketSpace.merges_inhibited()) -{ - idealState = distributorBucketSpace.get_ideal_service_layer_nodes_bundle(bucket.getBucketId()).get_available_nonretired_or_maintenance_nodes(); - unorderedIdealState.insert(idealState.begin(), idealState.end()); -} +{ } StateChecker::Context::~Context() = default; diff --git a/storage/src/vespa/storage/distributor/statechecker.h b/storage/src/vespa/storage/distributor/statechecker.h index 348d90bc712..25918e7a047 100644 --- a/storage/src/vespa/storage/distributor/statechecker.h +++ b/storage/src/vespa/storage/distributor/statechecker.h @@ -2,11 +2,12 @@ #pragma once #include "bucketgctimecalculator.h" +#include "ideal_service_layer_nodes_bundle.h" #include <vespa/storage/distributor/maintenance/maintenancepriority.h> #include <vespa/storage/distributor/operations/idealstate/idealstateoperation.h> #include <vespa/storage/common/storagecomponent.h> #include <vespa/storage/bucketdb/bucketdatabase.h> -#include <unordered_set> +#include <vespa/vespalib/stllike/hash_set.h> #include <map> #include <set> @@ -63,28 +64,20 @@ public: std::vector<BucketDatabase::Entry> entries; // Common - const lib::ClusterState& systemState; - const lib::ClusterState* pending_cluster_state; // nullptr if no state is pending. - const DistributorConfiguration& distributorConfig; - const lib::Distribution& distribution; - BucketGcTimeCalculator gcTimeCalculator; - - // Separate ideal state into ordered sequence and unordered set, as we - // need to both know the actual order (activation prioritization etc) as - // well as have the ability to quickly check if a node is in an ideal - // location. - std::vector<uint16_t> idealState; - std::unordered_set<uint16_t> unorderedIdealState; - - const DistributorNodeContext& node_ctx; - const DistributorStripeOperationContext& op_ctx; - const BucketDatabase& db; - NodeMaintenanceStatsTracker& stats; - const bool merges_inhibited_in_bucket_space; - - const BucketDatabase::Entry& getSiblingEntry() const noexcept { - return siblingEntry; - } + const lib::ClusterState & systemState; + const lib::ClusterState * pending_cluster_state; // nullptr if no state is pending. + const DistributorConfiguration & distributorConfig; + const lib::Distribution & distribution; + BucketGcTimeCalculator gcTimeCalculator; + const IdealServiceLayerNodesBundle & idealStateBundle; + const DistributorNodeContext & node_ctx; + const DistributorStripeOperationContext & op_ctx; + const BucketDatabase & db; + NodeMaintenanceStatsTracker & stats; + const bool merges_inhibited_in_bucket_space; + + const BucketDatabase::Entry& getSiblingEntry() const noexcept { return siblingEntry; } + const std::vector<uint16_t> & idealState() const noexcept { return idealStateBundle.available_nonretired_or_maintenance_nodes(); } document::Bucket getBucket() const noexcept { return bucket; } document::BucketId getBucketId() const noexcept { return bucket.getBucketId(); } @@ -107,28 +100,19 @@ public: std::unique_ptr<ResultImpl> _impl; public: IdealStateOperation::UP createOperation() { - return (_impl - ? _impl->createOperation() - : IdealStateOperation::UP()); + return (_impl ? _impl->createOperation() : IdealStateOperation::UP()); } MaintenancePriority getPriority() const { - return (_impl - ? _impl->getPriority() - : MaintenancePriority()); + return (_impl ? _impl->getPriority() : MaintenancePriority()); } MaintenanceOperation::Type getType() const { - return (_impl - ? _impl->getType() - : MaintenanceOperation::OPERATION_COUNT); - + return (_impl ? _impl->getType() : MaintenanceOperation::OPERATION_COUNT); } static Result noMaintenanceNeeded(); - static Result createStoredResult( - IdealStateOperation::UP operation, - MaintenancePriority::Priority priority); + static Result createStoredResult(IdealStateOperation::UP operation, MaintenancePriority::Priority priority); private: explicit Result(std::unique_ptr<ResultImpl> impl) : _impl(std::move(impl)) diff --git a/storage/src/vespa/storage/distributor/statecheckers.cpp b/storage/src/vespa/storage/distributor/statecheckers.cpp index f9c26bf113e..43766225155 100644 --- a/storage/src/vespa/storage/distributor/statecheckers.cpp +++ b/storage/src/vespa/storage/distributor/statecheckers.cpp @@ -27,9 +27,7 @@ SplitBucketStateChecker::validForSplit(Context& c) { // Can't split if we have no nodes. if (c.entry->getNodeCount() == 0) { - LOG(spam, - "Can't split bucket %s, since it has no copies", - c.bucket.toString().c_str()); + LOG(spam, "Can't split bucket %s, since it has no copies", c.bucket.toString().c_str()); return false; } @@ -83,33 +81,21 @@ SplitBucketStateChecker::getBucketSizeRelativeToMax(Context& c) } StateChecker::Result -SplitBucketStateChecker::generateMinimumBucketSplitOperation( - Context& c) +SplitBucketStateChecker::generateMinimumBucketSplitOperation(Context& c) { - auto so = std::make_unique<SplitOperation>( - c.node_ctx, - BucketAndNodes(c.getBucket(), c.entry->getNodes()), - c.distributorConfig.getMinimalBucketSplit(), - 0, - 0); + auto so = std::make_unique<SplitOperation>(c.node_ctx, BucketAndNodes(c.getBucket(), c.entry->getNodes()), + c.distributorConfig.getMinimalBucketSplit(), 0, 0); so->setPriority(c.distributorConfig.getMaintenancePriorities().splitDistributionBits); - so->setDetailedReason( - "[Splitting bucket because the current system size requires " - "a higher minimum split bit]"); + so->setDetailedReason("[Splitting bucket because the current system size requires a higher minimum split bit]"); return Result::createStoredResult(std::move(so), MaintenancePriority::MEDIUM); } StateChecker::Result -SplitBucketStateChecker::generateMaxSizeExceededSplitOperation( - Context& c) +SplitBucketStateChecker::generateMaxSizeExceededSplitOperation(Context& c) { - auto so = std::make_unique<SplitOperation>( - c.node_ctx, - BucketAndNodes(c.getBucket(), c.entry->getNodes()), - 58, - c.distributorConfig.getSplitCount(), - c.distributorConfig.getSplitSize()); + auto so = std::make_unique<SplitOperation>(c.node_ctx, BucketAndNodes(c.getBucket(), c.entry->getNodes()), 58, + c.distributorConfig.getSplitCount(), c.distributorConfig.getSplitSize()); so->setPriority(c.distributorConfig.getMaintenancePriorities().splitLargeBucket); @@ -160,8 +146,7 @@ JoinBucketsStateChecker::isFirstSibling(const document::BucketId& bucketId) namespace { bool -equalNodeSet(const std::vector<uint16_t>& idealState, - const BucketDatabase::Entry& dbEntry) +equalNodeSet(const std::vector<uint16_t>& idealState, const BucketDatabase::Entry& dbEntry) { if (idealState.size() != dbEntry->getNodeCount()) { return false; @@ -179,12 +164,10 @@ equalNodeSet(const std::vector<uint16_t>& idealState, bool bucketAndSiblingReplicaLocationsEqualIdealState(const StateChecker::Context& context) { - if (!equalNodeSet(context.idealState, context.entry)) { + if (!equalNodeSet(context.idealState(), context.entry)) { return false; } - std::vector<uint16_t> siblingIdealState( - context.distribution.getIdealStorageNodes( - context.systemState, context.siblingBucket)); + std::vector<uint16_t> siblingIdealState = context.distribution.getIdealStorageNodes(context.systemState, context.siblingBucket); if (!equalNodeSet(siblingIdealState, context.siblingEntry)) { return false; } @@ -213,41 +196,29 @@ JoinBucketsStateChecker::siblingsAreInSync(const Context& context) const auto& siblingEntry(context.siblingEntry); if (entry->getNodeCount() != siblingEntry->getNodeCount()) { - LOG(spam, - "Not joining bucket %s because sibling bucket %s had different " - "node count", - context.bucket.toString().c_str(), - context.siblingBucket.toString().c_str()); + LOG(spam, "Not joining bucket %s because sibling bucket %s had different node count", + context.bucket.toString().c_str(), context.siblingBucket.toString().c_str()); return false; } bool siblingsCoLocated = true; for (uint32_t i = 0; i < entry->getNodeCount(); ++i) { - if (entry->getNodeRef(i).getNode() - != siblingEntry->getNodeRef(i).getNode()) - { + if (entry->getNodeRef(i).getNode() != siblingEntry->getNodeRef(i).getNode()) { siblingsCoLocated = false; break; } } if (!siblingsCoLocated && !inconsistentJoinIsAllowed(context)) { - LOG(spam, - "Not joining bucket %s because sibling bucket %s " - "does not have the same node set, or inconsistent joins cannot be " - "performed either due to config or because replicas were not in " - "their ideal location", - context.bucket.toString().c_str(), - context.siblingBucket.toString().c_str()); + LOG(spam, "Not joining bucket %s because sibling bucket %s does not have the same node set, or inconsistent " + "joins cannot be performed either due to config or because replicas were not in their ideal location", + context.bucket.toString().c_str(), context.siblingBucket.toString().c_str()); return false; } if (!entry->validAndConsistent() || !siblingEntry->validAndConsistent()) { - LOG(spam, - "Not joining bucket %s because it or %s is out of sync " - "and syncing it may cause it to become too large", - context.bucket.toString().c_str(), - context.siblingBucket.toString().c_str()); + LOG(spam, "Not joining bucket %s because it or %s is out of sync and syncing it may cause it to become too large", + context.bucket.toString().c_str(), context.siblingBucket.toString().c_str()); return false; } @@ -291,9 +262,8 @@ contextBucketHasTooManyReplicas(const StateChecker::Context& c) bool bucketAtDistributionBitLimit(const document::BucketId& bucket, const StateChecker::Context& c) { - return (bucket.getUsedBits() <= std::max( - uint32_t(c.systemState.getDistributionBitCount()), - c.distributorConfig.getMinimalBucketSplit())); + return (bucket.getUsedBits() <= std::max(uint32_t(c.systemState.getDistributionBitCount()), + c.distributorConfig.getMinimalBucketSplit())); } } @@ -302,31 +272,23 @@ bool JoinBucketsStateChecker::shouldJoin(const Context& c) { if (c.entry->getNodeCount() == 0) { - LOG(spam, "Not joining bucket %s because it has no nodes", - c.bucket.toString().c_str()); + LOG(spam, "Not joining bucket %s because it has no nodes", c.bucket.toString().c_str()); return false; } if (contextBucketHasTooManyReplicas(c)) { - LOG(spam, "Not joining %s because it has too high replication level", - c.bucket.toString().c_str()); + LOG(spam, "Not joining %s because it has too high replication level", c.bucket.toString().c_str()); return false; } if (c.distributorConfig.getJoinSize() == 0 && c.distributorConfig.getJoinCount() == 0) { - LOG(spam, "Not joining bucket %s because join is disabled", - c.bucket.toString().c_str()); + LOG(spam, "Not joining bucket %s because join is disabled", c.bucket.toString().c_str()); return false; } if (bucketAtDistributionBitLimit(c.getBucketId(), c)) { - LOG(spam, - "Not joining bucket %s because it is below the min split " - "count (config: %u, cluster state: %u, bucket has: %u)", - c.bucket.toString().c_str(), - c.distributorConfig.getMinimalBucketSplit(), - c.systemState.getDistributionBitCount(), - c.getBucketId().getUsedBits()); + LOG(spam, "Not joining bucket %s because it is below the min split count (config: %u, cluster state: %u, bucket has: %u)", + c.bucket.toString().c_str(), c.distributorConfig.getMinimalBucketSplit(), c.systemState.getDistributionBitCount(), c.getBucketId().getUsedBits()); return false; } @@ -336,11 +298,8 @@ JoinBucketsStateChecker::shouldJoin(const Context& c) if (c.getSiblingEntry().valid()) { if (!isFirstSibling(c.getBucketId())) { - LOG(spam, - "Not joining bucket %s because it is the second sibling of " - "%s and not the first", - c.bucket.toString().c_str(), - c.siblingBucket.toString().c_str()); + LOG(spam, "Not joining bucket %s because it is the second sibling of %s and not the first", + c.bucket.toString().c_str(), c.siblingBucket.toString().c_str()); return false; } if (!siblingsAreInSync(c)) { @@ -463,24 +422,13 @@ JoinBucketsStateChecker::check(Context& c) const sourceBuckets.push_back(c.getBucketId()); } sourceBuckets.push_back(c.getBucketId()); - auto op = std::make_unique<JoinOperation>( - c.node_ctx, - BucketAndNodes(joinedBucket, c.entry->getNodes()), - sourceBuckets); + auto op = std::make_unique<JoinOperation>(c.node_ctx, BucketAndNodes(joinedBucket, c.entry->getNodes()), sourceBuckets); op->setPriority(c.distributorConfig.getMaintenancePriorities().joinBuckets); vespalib::asciistream ost; - ost << "[Joining buckets " - << sourceBuckets[1].toString() - << " and " << sourceBuckets[0].toString() - << " because their size (" - << getTotalUsedFileSize(c) - << " bytes, " - << getTotalMetaCount(c) - << " docs) is less than the configured limit of (" - << c.distributorConfig.getJoinSize() - << ", " - << c.distributorConfig.getJoinCount() - << ")"; + ost << "[Joining buckets " << sourceBuckets[1].toString() << " and " << sourceBuckets[0].toString() + << " because their size (" << getTotalUsedFileSize(c) << " bytes, " + << getTotalMetaCount(c) << " docs) is less than the configured limit of (" + << c.distributorConfig.getJoinSize() << ", " << c.distributorConfig.getJoinCount() << ")"; op->setDetailedReason(ost.str()); @@ -516,8 +464,7 @@ vespalib::string SplitInconsistentStateChecker::getReason(const document::BucketId& bucketId, const std::vector<BucketDatabase::Entry>& entries) { vespalib::asciistream reason; - reason << "[Bucket is inconsistently split (list includes " - << vespalib::hex << "0x" << bucketId.getId(); + reason << "[Bucket is inconsistently split (list includes " << vespalib::hex << "0x" << bucketId.getId(); for (uint32_t i = 0, found = 0; i < entries.size() && found < 3; i++) { if (!(entries[i].getBucketId() == bucketId)) { @@ -530,10 +477,7 @@ SplitInconsistentStateChecker::getReason(const document::BucketId& bucketId, con reason << " and " << vespalib::dec << entries.size() - 4 << " others"; } - reason << ") Splitting it to improve the problem (max used bits " - << vespalib::dec - << getHighestUsedBits(entries) - << ")]"; + reason << ") Splitting it to improve the problem (max used bits " << vespalib::dec << getHighestUsedBits(entries) << ")]"; return reason.str(); } @@ -559,12 +503,8 @@ SplitInconsistentStateChecker::check(Context& c) const return Result::noMaintenanceNeeded(); } - auto op = std::make_unique<SplitOperation>( - c.node_ctx, - BucketAndNodes(c.getBucket(), c.entry->getNodes()), - getHighestUsedBits(c.entries), - 0, - 0); + auto op = std::make_unique<SplitOperation>(c.node_ctx, BucketAndNodes(c.getBucket(), c.entry->getNodes()), + getHighestUsedBits(c.entries), 0, 0); op->setPriority(c.distributorConfig.getMaintenancePriorities().splitInconsistentBucket); op->setDetailedReason(getReason(c.getBucketId(), c.entries)); @@ -576,8 +516,7 @@ namespace { bool containsMaintenanceNode(const std::vector<uint16_t>& ideal, const StateChecker::Context& c) { for (uint16_t n : ideal) { - lib::Node node(lib::NodeType::STORAGE, n); - if (c.systemState.getNodeState(node).getState() == lib::State::MAINTENANCE) { + if (c.systemState.getNodeState(lib::Node(lib::NodeType::STORAGE, n)).getState() == lib::State::MAINTENANCE) { return true; } } @@ -588,9 +527,8 @@ bool ideal_node_is_unavailable_in_pending_state(const StateChecker::Context& c) if (!c.pending_cluster_state) { return false; } - for (uint16_t n : c.idealState) { - lib::Node node(lib::NodeType::STORAGE, n); - if (!c.pending_cluster_state->getNodeState(node).getState().oneOf("uir")){ + for (uint16_t n : c.idealState()) { + if (!c.pending_cluster_state->getNodeState(lib::Node(lib::NodeType::STORAGE, n)).getState().oneOf("uir")){ return true; } } @@ -598,9 +536,7 @@ bool ideal_node_is_unavailable_in_pending_state(const StateChecker::Context& c) } bool -consistentApartFromEmptyBucketsInNonIdealLocationAndInvalidEntries( - const std::vector<uint16_t>& idealNodes, - const BucketInfo& entry) +consistentApartFromEmptyBucketsInNonIdealLocationAndInvalidEntries(const std::vector<uint16_t>& idealNodes, const BucketInfo& entry) { api::BucketInfo info; for (uint32_t i=0, n=entry.getNodeCount(); i<n; ++i) { @@ -720,9 +656,9 @@ private: MergeNodes::~MergeNodes() = default; bool -presentInIdealState(const StateChecker::Context& c, uint16_t node) +presentInIdealState(const StateChecker::Context& c, uint16_t node) noexcept { - return c.unorderedIdealState.find(node) != c.unorderedIdealState.end(); + return c.idealStateBundle.is_nonretired_or_maintenance(node); } void @@ -730,7 +666,7 @@ addStatisticsForNonIdealNodes(const StateChecker::Context& c, bool missingReplic { // Common case is that ideal state == actual state with no missing replicas. // If so, do nothing. - if (!missingReplica && (c.idealState.size() == c.entry->getNodeCount())) { + if (!missingReplica && (c.idealState().size() == c.entry->getNodeCount())) { return; } for (uint32_t j = 0; j < c.entry->getNodeCount(); ++j) { @@ -753,26 +689,23 @@ checkForNodesMissingFromIdealState(StateChecker::Context& c) // Check if we need to add copies to get to ideal state. if (!c.entry->emptyAndConsistent()) { bool hasMissingReplica = false; - for (uint32_t i = 0; i < c.idealState.size(); i++) { + for (uint16_t node : c.idealState()) { bool found = false; for (uint32_t j = 0; j < c.entry->getNodeCount(); j++) { - if (c.entry->getNodeRef(j).getNode() == c.idealState[i]) { + if (c.entry->getNodeRef(j).getNode() == node) { found = true; break; } } if (!found) { - const DistributorConfiguration::MaintenancePriorities& mp( - c.distributorConfig.getMaintenancePriorities()); - if (c.idealState.size() > c.entry->getNodeCount()) { - ret.markMissingReplica(c.idealState[i], - mp.mergeTooFewCopies); + const auto & mp = c.distributorConfig.getMaintenancePriorities(); + if (c.idealState().size() > c.entry->getNodeCount()) { + ret.markMissingReplica(node, mp.mergeTooFewCopies); } else { - ret.markMoveToIdealLocation(c.idealState[i], - mp.mergeMoveToIdealNode); + ret.markMoveToIdealLocation(node,mp.mergeMoveToIdealNode); } - c.stats.incCopyingIn(c.idealState[i], c.getBucketSpace()); + c.stats.incCopyingIn(node, c.getBucketSpace()); hasMissingReplica = true; } } @@ -795,12 +728,8 @@ MergeNodes checkIfBucketsAreOutOfSyncAndNeedMerging(StateChecker::Context& c) { MergeNodes ret; - if (!consistentApartFromEmptyBucketsInNonIdealLocationAndInvalidEntries( - c.idealState, - c.entry.getBucketInfo())) - { - auto pri(c.distributorConfig.getMaintenancePriorities() - .mergeOutOfSyncCopies); + if (!consistentApartFromEmptyBucketsInNonIdealLocationAndInvalidEntries(c.idealState(),c.entry.getBucketInfo())) { + auto pri(c.distributorConfig.getMaintenancePriorities().mergeOutOfSyncCopies); ret.markOutOfSync(c, pri); addStatisticsForOutOfSyncCopies(c); } @@ -839,7 +768,7 @@ SynchronizeAndMoveStateChecker::check(Context& c) const if (isInconsistentlySplit(c)) { return Result::noMaintenanceNeeded(); } - if (containsMaintenanceNode(c.idealState, c)) { + if (containsMaintenanceNode(c.idealState(), c)) { return Result::noMaintenanceNeeded(); } if (ideal_node_is_unavailable_in_pending_state(c)) { @@ -856,16 +785,14 @@ SynchronizeAndMoveStateChecker::check(Context& c) const result += checkIfBucketsAreOutOfSyncAndNeedMerging(c); if (result.shouldMerge()) { - IdealStateOperation::UP op( - new MergeOperation(BucketAndNodes(c.getBucket(), result.nodes()), - c.distributorConfig.getMaxNodesPerMerge())); + auto op = std::make_unique<MergeOperation>(BucketAndNodes(c.getBucket(), result.nodes()), + c.distributorConfig.getMaxNodesPerMerge()); op->setDetailedReason(result.reason()); MaintenancePriority::Priority schedPri; if ((c.getBucketSpace() == document::FixedBucketSpaces::default_space()) || !c.distributorConfig.prioritize_global_bucket_merges()) { - schedPri = (result.needsMoveOnly() ? MaintenancePriority::LOW - : MaintenancePriority::MEDIUM); + schedPri = (result.needsMoveOnly() ? MaintenancePriority::LOW : MaintenancePriority::MEDIUM); op->setPriority(result.priority()); } else { // Since the default bucket space has a dependency on the global bucket space, @@ -877,10 +804,8 @@ SynchronizeAndMoveStateChecker::check(Context& c) const return Result::createStoredResult(std::move(op), schedPri); } else { - LOG(spam, "Bucket %s: No need for merge, as bucket is in consistent state " - "(or inconsistent buckets are empty) %s", - c.bucket.toString().c_str(), - c.entry->toString().c_str()); + LOG(spam, "Bucket %s: No need for merge, as bucket is in consistent state (or inconsistent buckets are empty) %s", + c.bucket.toString().c_str(), c.entry->toString().c_str()); return Result::noMaintenanceNeeded(); } } @@ -895,7 +820,7 @@ DeleteExtraCopiesStateChecker::bucketHasNoData(const Context& c) bool DeleteExtraCopiesStateChecker::copyIsInIdealState(const BucketCopy& cp, const Context& c) { - return hasItem(c.idealState, cp.getNode()); + return hasItem(c.idealState(), cp.getNode()); } bool @@ -911,9 +836,7 @@ DeleteExtraCopiesStateChecker::addToRemoveSet( std::vector<uint16_t>& removedCopies, vespalib::asciistream& reasons) { - reasons << "[Removing " << reasonForRemoval - << " from node " << copyToRemove.getNode() - << ']'; + reasons << "[Removing " << reasonForRemoval << " from node " << copyToRemove.getNode() << ']'; removedCopies.push_back(copyToRemove.getNode()); } @@ -981,7 +904,7 @@ DeleteExtraCopiesStateChecker::check(Context& c) const } // Maintain symmetry with merge; don't try to mess with nodes that have an // ideal copy on a node set in maintenance mode. - if (containsMaintenanceNode(c.idealState, c)) { + if (containsMaintenanceNode(c.idealState(), c)) { return Result::noMaintenanceNeeded(); } @@ -989,8 +912,7 @@ DeleteExtraCopiesStateChecker::check(Context& c) const std::vector<uint16_t> removedCopies; if (bucketHasNoData(c)) { - reasons << "[Removing all copies since bucket is empty:" - << c.entry->toString() << "]"; + reasons << "[Removing all copies since bucket is empty:" << c.entry->toString() << "]"; for (uint32_t j = 0, cnt = c.entry->getNodeCount(); j < cnt; ++j) { removedCopies.push_back(c.entry->getNodeRef(j).getNode()); @@ -1004,9 +926,7 @@ DeleteExtraCopiesStateChecker::check(Context& c) const } if (!removedCopies.empty()) { - auto ro = std::make_unique<RemoveBucketOperation>( - c.node_ctx, - BucketAndNodes(c.getBucket(), removedCopies)); + auto ro = std::make_unique<RemoveBucketOperation>(c.node_ctx, BucketAndNodes(c.getBucket(), removedCopies)); ro->setPriority(c.distributorConfig.getMaintenancePriorities().deleteBucketCopy); ro->setDetailedReason(reasons.str()); @@ -1030,7 +950,7 @@ BucketStateStateChecker::shouldSkipActivationDueToMaintenance(const ActiveList& // If copy is not ready, we don't want to activate it if a node // is set in maintenance. Doing so would imply that we want proton // to start background indexing. - return containsMaintenanceNode(c.idealState, c); + return containsMaintenanceNode(c.idealState(), c); } // else: activation does not imply indexing, so we can safely do it at any time. } } @@ -1058,9 +978,8 @@ BucketStateStateChecker::check(Context& c) const return Result::noMaintenanceNeeded(); } - ActiveList activeNodes( - ActiveCopy::calculate(c.idealState, c.distribution, c.entry, - c.distributorConfig.max_activation_inhibited_out_of_sync_groups())); + ActiveList activeNodes = ActiveCopy::calculate(c.idealState(), c.distribution, c.entry, + c.distributorConfig.max_activation_inhibited_out_of_sync_groups()); if (activeNodes.empty()) { return Result::noMaintenanceNeeded(); } @@ -1076,8 +995,7 @@ BucketStateStateChecker::check(Context& c) const continue; } operationNodes.push_back(activeNodes[i]._nodeIndex); - reason << "[Setting node " << activeNodes[i]._nodeIndex << " as active: " - << activeNodes[i].getReason() << "]"; + reason << "[Setting node " << activeNodes[i]._nodeIndex << " as active: " << activeNodes[i].getReason() << "]"; } // Deactivate all copies that are currently marked as active. @@ -1106,10 +1024,7 @@ BucketStateStateChecker::check(Context& c) const for (uint32_t i=0; i<activeNodes.size(); ++i) { activeNodeIndexes.push_back(activeNodes[i]._nodeIndex); } - auto op = std::make_unique<SetBucketStateOperation>( - c.node_ctx, - BucketAndNodes(c.getBucket(), operationNodes), - activeNodeIndexes); + auto op = std::make_unique<SetBucketStateOperation>(c.node_ctx, BucketAndNodes(c.getBucket(), operationNodes), activeNodeIndexes); // If activeNodes > 1, we're dealing with a active-per-leaf group case and // we currently always send high pri activations. @@ -1135,7 +1050,7 @@ GarbageCollectionStateChecker::needs_garbage_collection(const Context& c, vespal if (c.entry->getNodeCount() == 0) { return false; } - if (containsMaintenanceNode(c.idealState, c)) { + if (containsMaintenanceNode(c.idealState(), c)) { return false; } std::chrono::seconds lastRunAt(c.entry->getLastGarbageCollectionTime()); diff --git a/storage/src/vespa/storage/distributor/stripe_access_guard.h b/storage/src/vespa/storage/distributor/stripe_access_guard.h index 2ed40cfcf2e..d2d3615b776 100644 --- a/storage/src/vespa/storage/distributor/stripe_access_guard.h +++ b/storage/src/vespa/storage/distributor/stripe_access_guard.h @@ -4,6 +4,7 @@ #include "bucket_space_distribution_configs.h" #include "pending_bucket_space_db_transition_entry.h" #include "potential_data_loss_report.h" +#include "outdated_nodes.h" #include <vespa/document/bucket/bucketspace.h> #include <vespa/storageapi/defs.h> #include <unordered_set> // TODO use hash_set instead @@ -30,6 +31,7 @@ class NodeSupportedFeaturesRepo; */ class StripeAccessGuard { public: + using OutdatedNodes = dbtransition::OutdatedNodes; virtual ~StripeAccessGuard() = default; virtual void flush_and_close() = 0; @@ -51,7 +53,7 @@ public: const lib::Distribution& distribution, const lib::ClusterState& new_state, const char* storage_up_states, - const std::unordered_set<uint16_t>& outdated_nodes, + const OutdatedNodes& outdated_nodes, const std::vector<dbtransition::Entry>& entries) = 0; virtual void update_read_snapshot_before_db_pruning() = 0; diff --git a/storage/src/vespa/storage/distributor/stripe_bucket_db_updater.cpp b/storage/src/vespa/storage/distributor/stripe_bucket_db_updater.cpp index 8fce8c3137a..09e7d370a98 100644 --- a/storage/src/vespa/storage/distributor/stripe_bucket_db_updater.cpp +++ b/storage/src/vespa/storage/distributor/stripe_bucket_db_updater.cpp @@ -257,7 +257,7 @@ StripeBucketDBUpdater::merge_entries_into_db(document::BucketSpace bucket_space, const lib::Distribution& distribution, const lib::ClusterState& new_state, const char* storage_up_states, - const std::unordered_set<uint16_t>& outdated_nodes, + const OutdatedNodes & outdated_nodes, const std::vector<dbtransition::Entry>& entries) { auto& s = _op_ctx.bucket_space_repo().get(bucket_space); diff --git a/storage/src/vespa/storage/distributor/stripe_bucket_db_updater.h b/storage/src/vespa/storage/distributor/stripe_bucket_db_updater.h index 04efe91e9e7..b8b729edbeb 100644 --- a/storage/src/vespa/storage/distributor/stripe_bucket_db_updater.h +++ b/storage/src/vespa/storage/distributor/stripe_bucket_db_updater.h @@ -33,6 +33,7 @@ class StripeBucketDBUpdater final public api::MessageHandler { public: + using OutdatedNodes = dbtransition::OutdatedNodes; StripeBucketDBUpdater(const DistributorNodeContext& node_ctx, DistributorStripeOperationContext& op_ctx, DistributorStripeInterface& owner, @@ -178,7 +179,7 @@ private: const lib::Distribution& distribution, const lib::ClusterState& new_state, const char* storage_up_states, - const std::unordered_set<uint16_t>& outdated_nodes, + const OutdatedNodes & outdated_nodes, const std::vector<dbtransition::Entry>& entries); void enqueueRecheckUntilPendingStateEnabled(uint16_t node, const document::Bucket&); diff --git a/storage/src/vespa/storage/distributor/tickable_stripe.h b/storage/src/vespa/storage/distributor/tickable_stripe.h index e458043ac64..499cb41ee34 100644 --- a/storage/src/vespa/storage/distributor/tickable_stripe.h +++ b/storage/src/vespa/storage/distributor/tickable_stripe.h @@ -24,6 +24,7 @@ class NodeSupportedFeaturesRepo; */ class TickableStripe { public: + using OutdatedNodes = dbtransition::OutdatedNodes; virtual ~TickableStripe() = default; // Perform a single operation tick of the stripe logic. @@ -53,7 +54,7 @@ public: const lib::Distribution& distribution, const lib::ClusterState& new_state, const char* storage_up_states, - const std::unordered_set<uint16_t>& outdated_nodes, + const OutdatedNodes & outdated_nodes, const std::vector<dbtransition::Entry>& entries) = 0; virtual void update_read_snapshot_before_db_pruning() = 0; diff --git a/storage/src/vespa/storage/distributor/top_level_distributor.cpp b/storage/src/vespa/storage/distributor/top_level_distributor.cpp index 80c096135fa..f957af5362e 100644 --- a/storage/src/vespa/storage/distributor/top_level_distributor.cpp +++ b/storage/src/vespa/storage/distributor/top_level_distributor.cpp @@ -340,10 +340,10 @@ TopLevelDistributor::propagate_default_distribution_thread_unsafe( } } -std::unordered_map<uint16_t, uint32_t> +MinReplicaMap TopLevelDistributor::getMinReplica() const { - std::unordered_map<uint16_t, uint32_t> result; + MinReplicaMap result; for (const auto& stripe : _stripes) { merge_min_replica_stats(result, stripe->getMinReplica()); } @@ -360,15 +360,6 @@ TopLevelDistributor::getBucketSpacesStats() const return result; } -SimpleMaintenanceScanner::PendingMaintenanceStats -TopLevelDistributor::pending_maintenance_stats() const { - SimpleMaintenanceScanner::PendingMaintenanceStats result; - for (const auto& stripe : _stripes) { - result.merge(stripe->pending_maintenance_stats()); - } - return result; -} - void TopLevelDistributor::propagateInternalScanMetricsToExternal() { diff --git a/storage/src/vespa/storage/distributor/top_level_distributor.h b/storage/src/vespa/storage/distributor/top_level_distributor.h index aa3a7b3655d..278a68f72c6 100644 --- a/storage/src/vespa/storage/distributor/top_level_distributor.h +++ b/storage/src/vespa/storage/distributor/top_level_distributor.h @@ -142,10 +142,9 @@ private: /** * Return a copy of the latest min replica data, see MinReplicaProvider. */ - std::unordered_map<uint16_t, uint32_t> getMinReplica() const override; + MinReplicaMap getMinReplica() const override; PerNodeBucketSpacesStats getBucketSpacesStats() const override; - SimpleMaintenanceScanner::PendingMaintenanceStats pending_maintenance_stats() const; /** * Atomically publish internal metrics to external ideal state metrics. diff --git a/storage/src/vespa/storage/storageserver/rpc/slime_cluster_state_bundle_codec.cpp b/storage/src/vespa/storage/storageserver/rpc/slime_cluster_state_bundle_codec.cpp index 8c994991b9b..ea049493348 100644 --- a/storage/src/vespa/storage/storageserver/rpc/slime_cluster_state_bundle_codec.cpp +++ b/storage/src/vespa/storage/storageserver/rpc/slime_cluster_state_bundle_codec.cpp @@ -44,7 +44,7 @@ OutputBuf::~OutputBuf() = default; vespalib::string serialize_state(const lib::ClusterState& state) { vespalib::asciistream as; - state.serialize(as, false); + state.serialize(as); return as.str(); } diff --git a/storage/src/vespa/storage/storageutil/distributorstatecache.h b/storage/src/vespa/storage/storageutil/distributorstatecache.h index 8c4d07e39bf..0652a980e3a 100644 --- a/storage/src/vespa/storage/storageutil/distributorstatecache.h +++ b/storage/src/vespa/storage/storageutil/distributorstatecache.h @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once + #include <vespa/vdslib/state/clusterstate.h> #include <vespa/vdslib/distribution/distribution.h> @@ -9,9 +10,7 @@ namespace storage { class DistributorStateCache { public: - DistributorStateCache( - const lib::Distribution& distr, - const lib::ClusterState& state) + DistributorStateCache(const lib::Distribution& distr, const lib::ClusterState& state) : _distribution(distr), _state(state), _distrBitMask(0xffffffffffffffffull), @@ -22,8 +21,7 @@ public: _distrBitMask >>= (64 - state.getDistributionBitCount()); } - uint16_t getOwner(const document::BucketId& bid, - const char* upStates = "ui") + uint16_t getOwner(const document::BucketId& bid, const char* upStates = "ui") { uint64_t distributionBits = bid.getRawId() & _distrBitMask; diff --git a/storage/src/vespa/storage/tools/getidealstate.cpp b/storage/src/vespa/storage/tools/getidealstate.cpp index 8b120924aaa..9e80517f4f7 100644 --- a/storage/src/vespa/storage/tools/getidealstate.cpp +++ b/storage/src/vespa/storage/tools/getidealstate.cpp @@ -64,18 +64,13 @@ Options::Options(int argc, const char* const* argv) Options::~Options() {} -void processBucket(const lib::Distribution& distribution, - const lib::ClusterState& clusterState, - const std::string& upStates, - const document::BucketId& bucket) +void processBucket(const lib::Distribution& distribution, const lib::ClusterState& clusterState, + const std::string& upStates, const document::BucketId& bucket) { std::ostringstream ost; - std::vector<uint16_t> storageNodes(distribution.getIdealStorageNodes( - clusterState, bucket, upStates.c_str())); - uint16_t distributorNode(distribution.getIdealDistributorNode( - clusterState, bucket, upStates.c_str())); - ost << bucket << " distributor: " << distributorNode - << ", storage:"; + std::vector<uint16_t> storageNodes(distribution.getIdealStorageNodes(clusterState, bucket, upStates.c_str())); + uint16_t distributorNode(distribution.getIdealDistributorNode(clusterState, bucket, upStates.c_str())); + ost << bucket << " distributor: " << distributorNode << ", storage:"; for (uint32_t i=0; i<storageNodes.size(); ++i) { ost << " " << storageNodes[i]; } diff --git a/storage/src/vespa/storageframework/generic/status/htmlstatusreporter.cpp b/storage/src/vespa/storageframework/generic/status/htmlstatusreporter.cpp index 9b7c4919403..4cc32a2fc3d 100644 --- a/storage/src/vespa/storageframework/generic/status/htmlstatusreporter.cpp +++ b/storage/src/vespa/storageframework/generic/status/htmlstatusreporter.cpp @@ -4,17 +4,14 @@ namespace storage::framework { -HtmlStatusReporter::HtmlStatusReporter(vespalib::stringref id, - vespalib::stringref name) +HtmlStatusReporter::HtmlStatusReporter(vespalib::stringref id, vespalib::stringref name) : StatusReporter(id, name) -{ -} +{ } HtmlStatusReporter::~HtmlStatusReporter() = default; void -HtmlStatusReporter::reportHtmlHeader(std::ostream& out, - const HttpUrlPath& path) const +HtmlStatusReporter::reportHtmlHeader(std::ostream& out, const HttpUrlPath& path) const { out << "<html>\n" << "<head>\n" @@ -26,8 +23,7 @@ HtmlStatusReporter::reportHtmlHeader(std::ostream& out, } void -HtmlStatusReporter::reportHtmlFooter(std::ostream& out, - const HttpUrlPath&) const +HtmlStatusReporter::reportHtmlFooter(std::ostream& out, const HttpUrlPath&) const { out << "</body>\n</html>\n"; } @@ -39,8 +35,7 @@ HtmlStatusReporter::getReportContentType(const HttpUrlPath&) const } bool -HtmlStatusReporter::reportStatus(std::ostream& out, - const HttpUrlPath& path) const +HtmlStatusReporter::reportStatus(std::ostream& out, const HttpUrlPath& path) const { if (!isValidStatusRequest()) return false; reportHtmlHeader(out, path); diff --git a/storage/src/vespa/storageframework/generic/status/htmlstatusreporter.h b/storage/src/vespa/storageframework/generic/status/htmlstatusreporter.h index 4ffba20a3fa..ee3d65b0de3 100644 --- a/storage/src/vespa/storageframework/generic/status/htmlstatusreporter.h +++ b/storage/src/vespa/storageframework/generic/status/htmlstatusreporter.h @@ -29,8 +29,7 @@ struct HtmlStatusReporter : public StatusReporter { * some code in the <head></head> part of the HTML, such as javascript * functions. */ - virtual void reportHtmlHeaderAdditions(std::ostream&, - const HttpUrlPath&) const {} + virtual void reportHtmlHeaderAdditions(std::ostream&, const HttpUrlPath&) const {} /** * Write a default HTML header. It writes the start of an HTML diff --git a/vdslib/src/tests/distribution/distributiontest.cpp b/vdslib/src/tests/distribution/distributiontest.cpp index b5c756aece9..ec7c05fa7a2 100644 --- a/vdslib/src/tests/distribution/distributiontest.cpp +++ b/vdslib/src/tests/distribution/distributiontest.cpp @@ -53,9 +53,7 @@ TEST(DistributionTest, test_verify_java_distributions) long maxBucket = 1; long mask = 0; - for (uint32_t distributionBits = 0; distributionBits <= 32; - ++distributionBits) - { + for (uint32_t distributionBits = 0; distributionBits <= 32; ++distributionBits) { state.setDistributionBitCount(distributionBits); RandomGen randomizer(distributionBits); for (uint32_t bucketIndex = 0; bucketIndex < 64; ++bucketIndex) { @@ -66,11 +64,8 @@ TEST(DistributionTest, test_verify_java_distributions) bucketId = randomizer.nextUint64(); } document::BucketId bucket(distributionBits, bucketId); - for (uint32_t redundancy = 1; - redundancy <= distr.getRedundancy(); ++redundancy) - { - int distributorIndex = distr.getIdealDistributorNode( - state, bucket, "uim"); + for (uint32_t redundancy = 1; redundancy <= distr.getRedundancy(); ++redundancy) { + int distributorIndex = distr.getIdealDistributorNode(state, bucket, "uim"); of << distributionBits << " " << (bucketId & mask) << " " << redundancy << " " << distributorIndex << "\n"; } @@ -102,22 +97,16 @@ struct ExpectedResult { }; void -verifyJavaDistribution(const vespalib::string& name, - const ClusterState& state, - const Distribution& distribution, - const NodeType& nodeType, - uint16_t redundancy, - uint16_t nodeCount, - vespalib::stringref upStates, - const std::vector<ExpectedResult> results) +verifyJavaDistribution(const vespalib::string& name, const ClusterState& state, const Distribution& distribution, + const NodeType& nodeType, uint16_t redundancy, uint16_t nodeCount, + vespalib::stringref upStates, const std::vector<ExpectedResult> results) { (void) nodeCount; for (uint32_t i=0, n=results.size(); i<n; ++i) { std::string testId = name + " " + results[i].bucket.toString(); try { std::vector<uint16_t> nvect; - distribution.getIdealNodes(nodeType, state, results[i].bucket, - nvect, upStates.data(), redundancy); + distribution.getIdealNodes(nodeType, state, results[i].bucket, nvect, upStates.data(), redundancy); IdealNodeList nodes; for (uint32_t j=0, m=nvect.size(); j<m; ++j) { nodes.push_back(Node(nodeType, nvect[j])); @@ -155,8 +144,7 @@ auto readFile(const std::string & filename) { TEST(DistributionTest, test_verify_java_distributions_2) { - vespalib::DirectoryList files( - vespalib::listDirectory("distribution/testdata")); + vespalib::DirectoryList files(vespalib::listDirectory("distribution/testdata")); for (uint32_t i=0, n=files.size(); i<n; ++i) { size_t pos = files[i].find(".java.results"); if (pos == vespalib::string::npos || pos + 13 != files[i].size()) { @@ -189,8 +177,7 @@ TEST(DistributionTest, test_verify_java_distributions_2) ClusterState cs(c["cluster-state"].asString().make_string()); std::string distConfig(c["distribution"].asString().make_string()); Distribution d(distConfig); - const NodeType& nt( - NodeType::get(c["node-type"].asString().make_string())); + const NodeType& nt(NodeType::get(c["node-type"].asString().make_string())); uint32_t redundancy(c["redundancy"].asLong()); uint32_t nodeCount(c["node-count"].asLong()); vespalib::string upStates(c["up-states"].asString().make_string()); @@ -209,8 +196,7 @@ TEST(DistributionTest, test_verify_java_distributions_2) } results.push_back(result); } - verifyJavaDistribution(name, cs, d, nt, redundancy, nodeCount, - upStates, results); + verifyJavaDistribution(name, cs, d, nt, redundancy, nodeCount, upStates, results); //std::cerr << name << ": Verified " << results.size() << " tests.\n"; } } @@ -223,8 +209,7 @@ TEST(DistributionTest, test_unchanged_distribution) std::ifstream in("distribution/testdata/41-distributordistribution"); for (unsigned i = 0; i < 64_Ki; i++) { - uint16_t node = distr.getIdealDistributorNode( - state, document::BucketId(16, i), "u"); + uint16_t node = distr.getIdealDistributorNode(state, document::BucketId(16, i), "u"); char buf[100]; in.getline(buf, 100); @@ -272,9 +257,7 @@ struct MyTest { document::BucketId bucket(16, i); std::vector<uint16_t> nodes; ClusterState clusterState(_state); - _distribution->getIdealNodes( - *_nodeType, clusterState, bucket, nodes, - _upStates, _redundancy); + _distribution->getIdealNodes(*_nodeType, clusterState, bucket, nodes, _upStates, _redundancy); for (uint32_t j=0; j<nodes.size(); ++j) { ++result[nodes[j]]; } @@ -293,8 +276,7 @@ MyTest::MyTest() { } MyTest::~MyTest() = default; -std::vector<uint16_t> createNodeCountList(const std::string& source, - std::vector<uint16_t>& vals) { +std::vector<uint16_t> createNodeCountList(const std::string& source, std::vector<uint16_t>& vals) { std::vector<uint16_t> result(vals.size(), 0); vespalib::StringTokenizer st(source, " "); for (uint32_t i=0; i<st.size(); ++i) { @@ -375,15 +357,9 @@ TEST(DistributionTest, testHighSplitBit) document::BucketId bid1 = document::BucketId(bits, base); document::BucketId bid2 = document::BucketId(bits, base); - std::vector<uint16_t> nodes1 = - distr.getIdealStorageNodes(state, - bid1, - "u"); + std::vector<uint16_t> nodes1 = distr.getIdealStorageNodes(state, bid1, "u"); - std::vector<uint16_t> nodes2 = - distr.getIdealStorageNodes(state, - bid2, - "u"); + std::vector<uint16_t> nodes2 = distr.getIdealStorageNodes(state, bid2, "u"); ost1 << bid1 << " vs. " << bid2 << ": "; ost2 << bid1 << " vs. " << bid2 << ": "; @@ -424,16 +400,14 @@ TEST(DistributionTest, test_distribution) s1 << "storage:" << n << std::endl; ClusterState systemState(s1.str()); - Distribution distr( - Distribution::getDefaultDistributionConfig(3, n)); + Distribution distr(Distribution::getDefaultDistributionConfig(3, n)); std::vector<std::pair<uint64_t, std::vector<uint16_t> > > _distribution(b); std::vector<int> _nodeCount(n, 0); for (int i = 0; i < b; i++) { _distribution[i].first = i; - _distribution[i].second = distr.getIdealStorageNodes( - systemState, document::BucketId(26, i)); + _distribution[i].second = distr.getIdealStorageNodes(systemState, document::BucketId(26, i)); sort(_distribution[i].second.begin(), _distribution[i].second.end()); auto unique_nodes = std::distance(_distribution[i].second.begin(), unique(_distribution[i].second.begin(), _distribution[i].second.end())); _distribution[i].second.resize(unique_nodes); @@ -469,9 +443,7 @@ TEST(DistributionTest, test_move) { ClusterState systemState("storage:3"); document::BucketId bucket(16, 0x8b4f67ae); - Distribution distr(Distribution::getDefaultDistributionConfig(2, 3)); - res = distr.getIdealStorageNodes(systemState, bucket); EXPECT_EQ(size_t(2), res.size()); } @@ -479,11 +451,8 @@ TEST(DistributionTest, test_move) std::vector<uint16_t> res2; { ClusterState systemState("storage:4"); - Distribution distr(Distribution::getDefaultDistributionConfig(2, 4)); - document::BucketId bucket(16, 0x8b4f67ae); - res2 = distr.getIdealStorageNodes(systemState, bucket); EXPECT_EQ(size_t(2), res2.size()); } @@ -506,8 +475,7 @@ TEST(DistributionTest, test_move_constraints) std::vector<std::vector<uint16_t> > initBuckets(10000); for (unsigned i = 0; i < initBuckets.size(); i++) { - initBuckets[i] = distr.getIdealStorageNodes( - clusterState, document::BucketId(16, i)); + initBuckets[i] = distr.getIdealStorageNodes(clusterState, document::BucketId(16, i)); sort(initBuckets[i].begin(), initBuckets[i].end()); } @@ -517,8 +485,7 @@ TEST(DistributionTest, test_move_constraints) ClusterState systemState("storage:11 .10.s:d"); for (unsigned i = 0; i < addedDownBuckets.size(); i++) { - addedDownBuckets[i] = distr.getIdealStorageNodes( - systemState, document::BucketId(16, i)); + addedDownBuckets[i] = distr.getIdealStorageNodes(systemState, document::BucketId(16, i)); sort(addedDownBuckets[i].begin(), addedDownBuckets[i].end()); } for (unsigned i = 0; i < initBuckets.size(); i++) { @@ -541,15 +508,14 @@ TEST(DistributionTest, test_move_constraints) ClusterState systemState("storage:10 .0.s:d"); for (unsigned i = 0; i < removed0Buckets.size(); i++) { - removed0Buckets[i] = distr.getIdealStorageNodes( - systemState, document::BucketId(16, i)); + removed0Buckets[i] = distr.getIdealStorageNodes(systemState, document::BucketId(16, i)); sort(removed0Buckets[i].begin(), removed0Buckets[i].end()); } for (unsigned i = 0; i < initBuckets.size(); i++) { std::vector<uint16_t> movedAway; set_difference(initBuckets[i].begin(), initBuckets[i].end(), - removed0Buckets[i].begin(), removed0Buckets[i].end(), - back_inserter(movedAway)); + removed0Buckets[i].begin(), removed0Buckets[i].end(), + back_inserter(movedAway)); if (movedAway.size() > 0) { if (movedAway[0] != 0) { std::cerr << i << ": "; @@ -572,15 +538,14 @@ TEST(DistributionTest, test_move_constraints) ClusterState systemState("storage:11"); for (unsigned i = 0; i < added10Buckets.size(); i++) { - added10Buckets[i] = distr.getIdealStorageNodes( - systemState, document::BucketId(16, i)); + added10Buckets[i] = distr.getIdealStorageNodes(systemState, document::BucketId(16, i)); sort(added10Buckets[i].begin(), added10Buckets[i].end()); } for (unsigned i = 0; i < initBuckets.size(); i++) { std::vector<uint16_t> movedInto; std::set_difference(added10Buckets[i].begin(), added10Buckets[i].end(), - initBuckets[i].begin(), initBuckets[i].end(), - std::inserter(movedInto, movedInto.begin())); + initBuckets[i].begin(), initBuckets[i].end(), + std::inserter(movedInto, movedInto.begin())); if (movedInto.size() > 0) { ASSERT_EQ((size_t)1, movedInto.size()); EXPECT_EQ((uint16_t)10, movedInto[0]); @@ -601,11 +566,9 @@ TEST(DistributionTest, test_distribution_bits) for (unsigned i = 0; i < 100; i++) { int val = rand(); - uint32_t index = distr.getIdealDistributorNode( - state1, document::BucketId(19, val), "u"); + uint32_t index = distr.getIdealDistributorNode(state1, document::BucketId(19, val), "u"); ost1 << index << " "; - index = distr.getIdealDistributorNode( - state2, document::BucketId(19, val), "u"); + index = distr.getIdealDistributorNode(state2, document::BucketId(19, val), "u"); ost2 << index << " "; } @@ -620,10 +583,8 @@ TEST(DistributionTest, test_redundancy_hierarchical_distribution) Distribution distr2(Distribution::getDefaultDistributionConfig(2, 10)); for (unsigned i = 0; i < 100; i++) { - uint16_t d1 = distr1.getIdealDistributorNode( - state, document::BucketId(16, i), "u"); - uint16_t d2 = distr2.getIdealDistributorNode( - state, document::BucketId(16, i), "u"); + uint16_t d1 = distr1.getIdealDistributorNode(state, document::BucketId(16, i), "u"); + uint16_t d2 = distr2.getIdealDistributorNode(state, document::BucketId(16, i), "u"); EXPECT_EQ(d1, d2); } } @@ -653,20 +614,17 @@ TEST(DistributionTest, test_hierarchical_distribution) ClusterState state("distributor:6 storage:6"); for (uint32_t i = 0; i < 3; ++i) { - EXPECT_EQ( - vespalib::string("rack0"), - distr.getNodeGraph().getGroupForNode(i)->getName()); + EXPECT_EQ(vespalib::string("rack0"), + distr.getNodeGraph().getGroupForNode(i)->getName()); } for (uint32_t i = 3; i < 6; ++i) { - EXPECT_EQ( - vespalib::string("rack1"), - distr.getNodeGraph().getGroupForNode(i)->getName()); + EXPECT_EQ(vespalib::string("rack1"), + distr.getNodeGraph().getGroupForNode(i)->getName()); } std::vector<int> mainNode(6); for (uint32_t i=0; i<100; ++i) { - std::vector<uint16_t> nodes = distr.getIdealStorageNodes( - state, document::BucketId(16, i), "u"); + std::vector<uint16_t> nodes = distr.getIdealStorageNodes(state, document::BucketId(16, i), "u"); ASSERT_EQ((size_t) 4, nodes.size()); EXPECT_LT(nodes[0], mainNode.size()); ++mainNode[nodes[0]]; @@ -710,8 +668,7 @@ TEST(DistributionTest, test_group_capacity) int group0count = 0; int group1count = 0; for (uint32_t i = 0; i < 1000; i++) { - std::vector<uint16_t> nodes = distr.getIdealStorageNodes( - state, document::BucketId(16, i), "u"); + std::vector<uint16_t> nodes = distr.getIdealStorageNodes(state, document::BucketId(16, i), "u"); if (nodes[0] == 0 || nodes[0] == 1 || nodes[0] == 2) { group0count++; } @@ -794,14 +751,12 @@ TEST(DistributionTest, test_hierarchical_no_redistribution) EXPECT_EQ(numBuckets, v.size()); v.clear(); - state.setNodeState(Node(NodeType::STORAGE, 0), - NodeState(NodeType::STORAGE, State::DOWN)); + state.setNodeState(Node(NodeType::STORAGE, 0),NodeState(NodeType::STORAGE, State::DOWN)); std::vector< std::vector<uint16_t> > distr2(4); for (size_t i = 0; i < numBuckets; i++) { - nodes = distribution.getIdealStorageNodes( - state, document::BucketId(16, i), "u"); + nodes = distribution.getIdealStorageNodes(state, document::BucketId(16, i), "u"); for (uint16_t j=0; j<nodes.size(); ++j) { ASSERT_TRUE(0 != nodes[j]); distr2[nodes[j]].push_back(i); @@ -1010,7 +965,7 @@ group[2].nodes[1].retired false auto nodes_of = [&](uint32_t bucket){ std::vector<uint16_t> actual; - distr.getIdealNodes(NodeType::STORAGE, state, document::BucketId(16, bucket), actual); + distr.getIdealNodes(NodeType::STORAGE, state, document::BucketId(16, bucket), actual, "uim"); return actual; }; @@ -1071,9 +1026,13 @@ TEST(DistributionTest, DISABLED_benchmark_ideal_state_for_many_groups) { std::vector<uint16_t> actual; uint32_t bucket = 0; auto min_time = vespalib::BenchmarkTimer::benchmark([&]{ - distr.getIdealNodes(NodeType::STORAGE, state, document::BucketId(16, (bucket++ & 0xffffU)), actual); + distr.getIdealNodes(NodeType::STORAGE, state, document::BucketId(16, (bucket++ & 0xffffU)), actual, "uim"); }, 5.0); fprintf(stderr, "%.10f seconds\n", min_time); } +TEST(DistributionTest, control_size_of_IndexList) { + EXPECT_EQ(24u, sizeof(Distribution::IndexList)); +} + } diff --git a/vdslib/src/tests/state/clusterstatetest.cpp b/vdslib/src/tests/state/clusterstatetest.cpp index a08ec007f55..0b278177453 100644 --- a/vdslib/src/tests/state/clusterstatetest.cpp +++ b/vdslib/src/tests/state/clusterstatetest.cpp @@ -13,10 +13,10 @@ using ::testing::ContainsRegex; namespace storage::lib { -#define VERIFY3(source, result, type, typestr) { \ +#define VERIFY3(source, result, typestr) { \ vespalib::asciistream ost; \ try { \ - state->serialize(ost, type); \ + state->serialize(ost); \ } catch (std::exception& e) { \ FAIL() << ("Failed to serialize system state " \ + state->toString(true) + " in " + std::string(typestr) \ @@ -26,24 +26,18 @@ namespace storage::lib { vespalib::string(typestr) + " \"" + ost.str() + "\"") << state->toString(true); \ } -#define VERIFY2(serialized, result, testOld, testNew) { \ +#define VERIFY2(serialized, result) { \ std::unique_ptr<ClusterState> state; \ try { \ state.reset(new ClusterState(serialized)); \ } catch (std::exception& e) { \ - FAIL() << ("Failed to parse '" + std::string(serialized) \ - + "': " + e.what()); \ + FAIL() << ("Failed to parse '" + std::string(serialized) + "': " + e.what()); \ } \ - if (testOld) VERIFY3(serialized, result, true, "Old") \ - if (testNew) VERIFY3(serialized, result, false, "New") \ + VERIFY3(serialized, result, "New") \ } -#define VERIFYSAMEOLD(serialized) VERIFY2(serialized, serialized, true, false) -#define VERIFYOLD(serialized, result) VERIFY2(serialized, result, true, false) -#define VERIFYSAMENEW(serialized) VERIFY2(serialized, serialized, false, true) -#define VERIFYNEW(serialized, result) VERIFY2(serialized, result, false, true) -#define VERIFYSAME(serialized) VERIFY2(serialized, serialized, true, true) -#define VERIFY(serialized, result) VERIFY2(serialized, result, true, true) +#define VERIFYSAMENEW(serialized) VERIFY2(serialized, serialized) +#define VERIFYNEW(serialized, result) VERIFY2(serialized, result) #define VERIFY_FAIL(serialized, error) { \ try{ \ diff --git a/vdslib/src/vespa/vdslib/distribution/distribution.cpp b/vdslib/src/vespa/vdslib/distribution/distribution.cpp index e9113d7dd23..87a1f6d3758 100644 --- a/vdslib/src/vespa/vdslib/distribution/distribution.cpp +++ b/vdslib/src/vespa/vdslib/distribution/distribution.cpp @@ -20,16 +20,19 @@ LOG_SETUP(".vdslib.distribution"); namespace storage::lib { namespace { - std::vector<uint32_t> getDistributionBitMasks() { - std::vector<uint32_t> masks; - masks.resize(32 + 1); - uint32_t mask = 0; - for (uint32_t i=0; i<=32; ++i) { - masks[i] = mask; - mask = (mask << 1) | 1; - } - return masks; + +std::vector<uint32_t> +getDistributionBitMasks() { + std::vector<uint32_t> masks; + masks.resize(32 + 1); + uint32_t mask = 0; + for (uint32_t i=0; i<=32; ++i) { + masks[i] = mask; + mask = (mask << 1) | 1; } + return masks; +} + } VESPA_IMPLEMENT_EXCEPTION(NoDistributorsAvailableException, vespalib::Exception); @@ -65,8 +68,8 @@ Distribution::Distribution(const Distribution& d) configure(*reader.read()); } -Distribution::ConfigWrapper::ConfigWrapper(std::unique_ptr<DistributionConfig> cfg) : - _cfg(std::move(cfg)) +Distribution::ConfigWrapper::ConfigWrapper(std::unique_ptr<DistributionConfig> cfg) noexcept + : _cfg(std::move(cfg)) { } Distribution::ConfigWrapper::~ConfigWrapper() = default; @@ -150,8 +153,7 @@ Distribution::configure(const vespa::config::content::StorDistributionConfig& co if ( ! nodeGraph) { throw vespalib::IllegalStateException( "Got config that didn't seem to specify even a root group. Must " - "have a root group at minimum:\n" - + _serialized, VESPA_STRLOC); + "have a root group at minimum:\n" + _serialized, VESPA_STRLOC); } nodeGraph->calculateDistributionHashValues(); _nodeGraph = std::move(nodeGraph); @@ -161,14 +163,11 @@ Distribution::configure(const vespa::config::content::StorDistributionConfig& co _ensurePrimaryPersisted = config.ensurePrimaryPersisted; _readyCopies = config.readyCopies; _activePerGroup = config.activePerLeafGroup; - _distributorAutoOwnershipTransferOnWholeGroupDown - = config.distributorAutoOwnershipTransferOnWholeGroupDown; + _distributorAutoOwnershipTransferOnWholeGroupDown = config.distributorAutoOwnershipTransferOnWholeGroupDown; } uint32_t -Distribution::getGroupSeed( - const document::BucketId& bucket, const ClusterState& clusterState, - const Group& group) const +Distribution::getGroupSeed(const document::BucketId& bucket, const ClusterState& clusterState, const Group& group) const { uint32_t seed(static_cast<uint32_t>(bucket.getRawId()) & _distributionBitMasks[clusterState.getDistributionBitCount()]); @@ -177,8 +176,7 @@ Distribution::getGroupSeed( } uint32_t -Distribution::getDistributorSeed( - const document::BucketId& bucket, const ClusterState& state) const +Distribution::getDistributorSeed(const document::BucketId& bucket, const ClusterState& state) const { uint32_t seed(static_cast<uint32_t>(bucket.getRawId()) & _distributionBitMasks[state.getDistributionBitCount()]); @@ -186,8 +184,7 @@ Distribution::getDistributorSeed( } uint32_t -Distribution::getStorageSeed( - const document::BucketId& bucket, const ClusterState& state) const +Distribution::getStorageSeed(const document::BucketId& bucket, const ClusterState& state) const { uint32_t seed(static_cast<uint32_t>(bucket.getRawId()) & _distributionBitMasks[state.getDistributionBitCount()]); @@ -262,11 +259,8 @@ namespace { } void -Distribution::getIdealGroups(const document::BucketId& bucket, - const ClusterState& clusterState, - const Group& parent, - uint16_t redundancy, - std::vector<ResultGroup>& results) const +Distribution::getIdealGroups(const document::BucketId& bucket, const ClusterState& clusterState, const Group& parent, + uint16_t redundancy, std::vector<ResultGroup>& results) const { if (parent.isLeafGroup()) { results.emplace_back(parent, redundancy); @@ -300,15 +294,12 @@ Distribution::getIdealGroups(const document::BucketId& bucket, // This should never happen. Config should verify that each group // has enough groups beneath them. assert(group._group != nullptr); - getIdealGroups(bucket, clusterState, *group._group, - redundancyArray[i], results); + getIdealGroups(bucket, clusterState, *group._group, redundancyArray[i], results); } } const Group* -Distribution::getIdealDistributorGroup(const document::BucketId& bucket, - const ClusterState& clusterState, - const Group& parent) const +Distribution::getIdealDistributorGroup(const document::BucketId& bucket, const ClusterState& clusterState, const Group& parent) const { if (parent.isLeafGroup()) { return &parent; @@ -357,12 +348,8 @@ Distribution::allDistributorsDown(const Group& g, const ClusterState& cs) } void -Distribution::getIdealNodes(const NodeType& nodeType, - const ClusterState& clusterState, - const document::BucketId& bucket, - std::vector<uint16_t>& resultNodes, - const char* upStates, - uint16_t redundancy) const +Distribution::getIdealNodes(const NodeType& nodeType, const ClusterState& clusterState, const document::BucketId& bucket, + std::vector<uint16_t>& resultNodes, const char* upStates, uint16_t redundancy) const { if (redundancy == DEFAULT_REDUNDANCY) redundancy = _redundancy; resultNodes.clear(); @@ -388,8 +375,7 @@ Distribution::getIdealNodes(const NodeType& nodeType, const Group* group(getIdealDistributorGroup(bucket, clusterState, *_nodeGraph)); if (group == nullptr) { vespalib::asciistream ss; - ss << "There is no legal distributor target in state with version " - << clusterState.getVersion(); + ss << "There is no legal distributor target in state with version " << clusterState.getVersion(); throw NoDistributorsAvailableException(ss.str(), VESPA_STRLOC); } _groupDistribution.push_back(ResultGroup(*group, 1)); @@ -459,9 +445,7 @@ Distribution::getDefaultDistributionConfig(uint16_t redundancy, uint16_t nodeCou } std::vector<uint16_t> -Distribution::getIdealStorageNodes( - const ClusterState& state, const document::BucketId& bucket, - const char* upStates) const +Distribution::getIdealStorageNodes(const ClusterState& state, const document::BucketId& bucket, const char* upStates) const { std::vector<uint16_t> nodes; getIdealNodes(NodeType::STORAGE, state, bucket, nodes, upStates); @@ -469,33 +453,28 @@ Distribution::getIdealStorageNodes( } uint16_t -Distribution::getIdealDistributorNode( - const ClusterState& state, - const document::BucketId& bucket, - const char* upStates) const +Distribution::getIdealDistributorNode(const ClusterState& state, const document::BucketId& bucket, const char* upStates) const { std::vector<uint16_t> nodes; getIdealNodes(NodeType::DISTRIBUTOR, state, bucket, nodes, upStates); assert(nodes.size() <= 1); if (nodes.empty()) { vespalib::asciistream ss; - ss << "There is no legal distributor target in state with version " - << state.getVersion(); + ss << "There is no legal distributor target in state with version " << state.getVersion(); throw NoDistributorsAvailableException(ss.str(), VESPA_STRLOC); } return nodes[0]; } std::vector<Distribution::IndexList> -Distribution::splitNodesIntoLeafGroups(IndexList nodeList) const +Distribution::splitNodesIntoLeafGroups(vespalib::ConstArrayRef<uint16_t> nodeList) const { std::vector<IndexList> result; std::map<uint16_t, IndexList> nodes; for (auto node : nodeList) { const Group* group((node < _node2Group.size()) ? _node2Group[node] : nullptr); if (group == nullptr) { - LOGBP(warning, "Node %u is not assigned to a group. " - "Should not happen?", node); + LOGBP(warning, "Node %u is not assigned to a group. Should not happen?", node); } else { assert(group->isLeafGroup()); nodes[group->getIndex()].push_back(node); diff --git a/vdslib/src/vespa/vdslib/distribution/distribution.h b/vdslib/src/vespa/vdslib/distribution/distribution.h index 355b87884c1..8cf93b01630 100644 --- a/vdslib/src/vespa/vdslib/distribution/distribution.h +++ b/vdslib/src/vespa/vdslib/distribution/distribution.h @@ -12,6 +12,7 @@ #include <vespa/document/bucket/bucketid.h> #include <vespa/vdslib/state/nodetype.h> #include <vespa/vespalib/util/exception.h> +#include <vespa/vespalib/util/small_vector.h> namespace vespa::config::content::internal { class InternalStorDistributionType; @@ -38,9 +39,9 @@ private: uint16_t _redundancy; uint16_t _initialRedundancy; uint16_t _readyCopies; - bool _activePerGroup; - bool _ensurePrimaryPersisted; - bool _distributorAutoOwnershipTransferOnWholeGroupDown; + bool _activePerGroup; + bool _ensurePrimaryPersisted; + bool _distributorAutoOwnershipTransferOnWholeGroupDown; vespalib::string _serialized; struct ResultGroup { @@ -50,7 +51,7 @@ private: ResultGroup(const Group& group, uint16_t redundancy) noexcept : _group(&group), _redundancy(redundancy) {} - bool operator<(const ResultGroup& other) const { + bool operator<(const ResultGroup& other) const noexcept { return _group->getIndex() < other._group->getIndex(); } }; @@ -59,32 +60,23 @@ private: * Get seed to use for ideal state algorithm's random number generator * to decide which hierarchical group we should pick. */ - uint32_t getGroupSeed( - const document::BucketId&, const ClusterState&, - const Group&) const; + uint32_t getGroupSeed(const document::BucketId&, const ClusterState&, const Group&) const; /** * Get seed to use for ideal state algorithm's random number generator * to decide which distributor node this bucket should be mapped to. */ - uint32_t getDistributorSeed( - const document::BucketId&, const ClusterState&) const; + uint32_t getDistributorSeed(const document::BucketId&, const ClusterState&) const; /** * Get seed to use for ideal state algorithm's random number generator * to decide which storage node this bucket should be mapped to. */ - uint32_t getStorageSeed( - const document::BucketId&, const ClusterState&) const; + uint32_t getStorageSeed(const document::BucketId&, const ClusterState&) const; - void getIdealGroups(const document::BucketId& bucket, - const ClusterState& clusterState, - const Group& parent, - uint16_t redundancy, - std::vector<ResultGroup>& results) const; + void getIdealGroups(const document::BucketId& bucket, const ClusterState& clusterState, const Group& parent, + uint16_t redundancy, std::vector<ResultGroup>& results) const; - const Group* getIdealDistributorGroup(const document::BucketId& bucket, - const ClusterState& clusterState, - const Group& parent) const; + const Group* getIdealDistributorGroup(const document::BucketId& bucket, const ClusterState& clusterState, const Group& parent) const; /** * Since distribution object may be used often in ideal state calculations @@ -97,9 +89,9 @@ private: public: class ConfigWrapper { public: - ConfigWrapper(ConfigWrapper && rhs) = default; - ConfigWrapper & operator = (ConfigWrapper && rhs) = default; - ConfigWrapper(std::unique_ptr<DistributionConfig> cfg); + ConfigWrapper(ConfigWrapper && rhs) noexcept = default; + ConfigWrapper & operator = (ConfigWrapper && rhs) noexcept = default; + ConfigWrapper(std::unique_ptr<DistributionConfig> cfg) noexcept; ~ConfigWrapper(); const DistributionConfig & get() const { return *_cfg; } private: @@ -114,33 +106,26 @@ public: Distribution& operator=(const Distribution&) = delete; - const vespalib::string& serialize() const { return _serialized; } + const vespalib::string& serialize() const noexcept { return _serialized; } - const Group& getNodeGraph() const { return *_nodeGraph; } - uint16_t getRedundancy() const { return _redundancy; } - uint16_t getInitialRedundancy() const { return _initialRedundancy; } - uint16_t getReadyCopies() const { return _readyCopies; } - bool ensurePrimaryPersisted() const { return _ensurePrimaryPersisted; } - bool distributorAutoOwnershipTransferOnWholeGroupDown() const - { return _distributorAutoOwnershipTransferOnWholeGroupDown; } - bool activePerGroup() const { return _activePerGroup; } + const Group& getNodeGraph() const noexcept { return *_nodeGraph; } + uint16_t getRedundancy() const noexcept { return _redundancy; } + uint16_t getInitialRedundancy() const noexcept { return _initialRedundancy; } + uint16_t getReadyCopies() const noexcept { return _readyCopies; } + bool ensurePrimaryPersisted() const noexcept { return _ensurePrimaryPersisted; } + bool distributorAutoOwnershipTransferOnWholeGroupDown() const noexcept { return _distributorAutoOwnershipTransferOnWholeGroupDown; } + bool activePerGroup() const noexcept { return _activePerGroup; } - bool operator==(const Distribution& o) const - { return (_serialized == o._serialized); } - bool operator!=(const Distribution& o) const - { return (_serialized != o._serialized); } + bool operator==(const Distribution& o) const noexcept { return (_serialized == o._serialized); } + bool operator!=(const Distribution& o) const noexcept { return (_serialized != o._serialized); } void print(std::ostream& out, bool, const std::string&) const override; /** Simplified wrapper for getIdealNodes() */ - std::vector<uint16_t> getIdealStorageNodes( - const ClusterState&, const document::BucketId&, - const char* upStates = "uim") const; + std::vector<uint16_t> getIdealStorageNodes(const ClusterState&, const document::BucketId&, const char* upStates = "uim") const; /** Simplified wrapper for getIdealNodes() */ - uint16_t getIdealDistributorNode( - const ClusterState&, const document::BucketId&, - const char* upStates = "uim") const; + uint16_t getIdealDistributorNode(const ClusterState&, const document::BucketId&, const char* upStates = "uim") const; /** * @throws TooFewBucketBitsInUseException If distribution bit count is @@ -149,25 +134,22 @@ public: * in any upstate. */ enum { DEFAULT_REDUNDANCY = 0xffff }; - void getIdealNodes(const NodeType&, const ClusterState&, - const document::BucketId&, std::vector<uint16_t>& nodes, - const char* upStates = "uim", - uint16_t redundancy = DEFAULT_REDUNDANCY) const; + void getIdealNodes(const NodeType&, const ClusterState&, const document::BucketId&, std::vector<uint16_t>& nodes, + const char* upStates, uint16_t redundancy = DEFAULT_REDUNDANCY) const; /** * Unit tests can use this function to get raw config for this class to use * with a really simple setup with no hierarchical grouping. This function * should not be used by any production code. */ - static ConfigWrapper getDefaultDistributionConfig( - uint16_t redundancy = 2, uint16_t nodeCount = 10); + static ConfigWrapper getDefaultDistributionConfig(uint16_t redundancy = 2, uint16_t nodeCount = 10); /** * Utility function used by distributor to split copies into groups to * handle active per group feature. */ - using IndexList = std::vector<uint16_t>; - std::vector<IndexList> splitNodesIntoLeafGroups(IndexList nodes) const; + using IndexList = vespalib::SmallVector<uint16_t, 4>; + std::vector<IndexList> splitNodesIntoLeafGroups(vespalib::ConstArrayRef<uint16_t> nodes) const; static bool allDistributorsDown(const Group&, const ClusterState&); }; diff --git a/vdslib/src/vespa/vdslib/distribution/group.cpp b/vdslib/src/vespa/vdslib/distribution/group.cpp index 537b4635e75..254a20e1052 100644 --- a/vdslib/src/vespa/vdslib/distribution/group.cpp +++ b/vdslib/src/vespa/vdslib/distribution/group.cpp @@ -11,7 +11,7 @@ namespace storage::lib { -Group::Group(uint16_t index, vespalib::stringref name) +Group::Group(uint16_t index, vespalib::stringref name) noexcept : _name(name), _index(index), _distributionHash(0), @@ -46,7 +46,7 @@ Group::~Group() } bool -Group::operator==(const Group& other) const +Group::operator==(const Group& other) const noexcept { return (_name == other._name && _index == other._index && diff --git a/vdslib/src/vespa/vdslib/distribution/group.h b/vdslib/src/vespa/vdslib/distribution/group.h index 5767f55d20a..3f468bee995 100644 --- a/vdslib/src/vespa/vdslib/distribution/group.h +++ b/vdslib/src/vespa/vdslib/distribution/group.h @@ -49,28 +49,25 @@ private: public: // Create leaf node - Group(uint16_t index, vespalib::stringref name); + Group(uint16_t index, vespalib::stringref name) noexcept; // Create branch node Group(uint16_t index, vespalib::stringref name, const Distribution&, uint16_t redundancy); virtual ~Group(); - bool isLeafGroup() const { return _nodes.size() > 0; } - bool operator==(const Group& other) const; + bool isLeafGroup() const noexcept { return ! _nodes.empty(); } + bool operator==(const Group& other) const noexcept; void print(std::ostream& out, bool verbose, const std::string& indent) const override; - vespalib::Double getCapacity() const { return _capacity; } - const vespalib::string & getName() const { return _name; } - uint16_t getIndex() const { return _index; } + vespalib::Double getCapacity() const noexcept { return _capacity; } + const vespalib::string & getName() const noexcept { return _name; } + uint16_t getIndex() const noexcept { return _index; } std::map<uint16_t, Group*>& getSubGroups() { return _subGroups; } - const std::map<uint16_t, Group*>& getSubGroups() const - { return _subGroups; } - const std::vector<uint16_t>& getNodes() const { return _nodes; }; - const Distribution& getDistributionSpec() const - { return _distributionSpec; } - const Distribution& getDistribution(uint16_t redundancy) const - { return _preCalculated[redundancy]; } - uint32_t getDistributionHash() const { return _distributionHash; } + const std::map<uint16_t, Group*>& getSubGroups() const noexcept { return _subGroups; } + const std::vector<uint16_t>& getNodes() const noexcept { return _nodes; }; + const Distribution& getDistributionSpec() const noexcept { return _distributionSpec; } + const Distribution& getDistribution(uint16_t redundancy) const noexcept { return _preCalculated[redundancy]; } + uint32_t getDistributionHash() const noexcept { return _distributionHash; } void addSubGroup(Group::UP); void setCapacity(vespalib::Double capacity); diff --git a/vdslib/src/vespa/vdslib/distribution/idealnodecalculator.h b/vdslib/src/vespa/vdslib/distribution/idealnodecalculator.h index bc42df1b49c..4eb8f7e04ae 100644 --- a/vdslib/src/vespa/vdslib/distribution/idealnodecalculator.h +++ b/vdslib/src/vespa/vdslib/distribution/idealnodecalculator.h @@ -22,25 +22,20 @@ class ClusterState; * unneeded details, and make it easily printable. */ class IdealNodeList : public document::Printable { - std::vector<Node> _idealNodes; - public: - IdealNodeList(); + IdealNodeList() noexcept; ~IdealNodeList(); void push_back(const Node& node) { _idealNodes.push_back(node); } - const Node& operator[](uint32_t i) const { return _idealNodes[i]; } - uint32_t size() const { return _idealNodes.size(); } - bool contains(const Node& n) const { - for (uint32_t i=0; i<_idealNodes.size(); ++i) { - if (n == _idealNodes[i]) return true; - } - return false; + const Node& operator[](uint32_t i) const noexcept { return _idealNodes[i]; } + uint32_t size() const noexcept { return _idealNodes.size(); } + bool contains(const Node& n) const noexcept { + return indexOf(n) != 0xffff; } - uint16_t indexOf(const Node& n) const { + uint16_t indexOf(const Node& n) const noexcept { for (uint16_t i=0; i<_idealNodes.size(); ++i) { if (n == _idealNodes[i]) return i; } @@ -48,6 +43,8 @@ public: } void print(std::ostream& out, bool, const std::string &) const override; +private: + std::vector<Node> _idealNodes; }; /** @@ -64,17 +61,15 @@ public: virtual ~IdealNodeCalculator() = default; - virtual IdealNodeList getIdealNodes(const NodeType&, - const document::BucketId&, - UpStates upStates = UpInit) const = 0; + virtual IdealNodeList getIdealNodes(const NodeType&, const document::BucketId&, UpStates upStates = UpInit) const = 0; // Wrapper functions to make prettier call if nodetype is given. - IdealNodeList getIdealDistributorNodes(const document::BucketId& bucket, - UpStates upStates = UpInit) const - { return getIdealNodes(NodeType::DISTRIBUTOR, bucket, upStates); } - IdealNodeList getIdealStorageNodes(const document::BucketId& bucket, - UpStates upStates = UpInit) const - { return getIdealNodes(NodeType::STORAGE, bucket, upStates); } + IdealNodeList getIdealDistributorNodes(const document::BucketId& bucket, UpStates upStates = UpInit) const { + return getIdealNodes(NodeType::DISTRIBUTOR, bucket, upStates); + } + IdealNodeList getIdealStorageNodes(const document::BucketId& bucket, UpStates upStates = UpInit) const { + return getIdealNodes(NodeType::STORAGE, bucket, upStates); + } }; diff --git a/vdslib/src/vespa/vdslib/distribution/idealnodecalculatorimpl.cpp b/vdslib/src/vespa/vdslib/distribution/idealnodecalculatorimpl.cpp index da34ec4526a..86123f47d6f 100644 --- a/vdslib/src/vespa/vdslib/distribution/idealnodecalculatorimpl.cpp +++ b/vdslib/src/vespa/vdslib/distribution/idealnodecalculatorimpl.cpp @@ -8,7 +8,7 @@ namespace storage::lib { -IdealNodeList::IdealNodeList() = default; +IdealNodeList::IdealNodeList() noexcept = default; IdealNodeList::~IdealNodeList() = default; void @@ -63,9 +63,10 @@ IdealNodeCalculatorImpl::initUpStateMapping() { _upStates[UpInit] = "ui"; _upStates[UpInitMaintenance] = "uim"; for (uint32_t i=0; i<_upStates.size(); ++i) { - if (_upStates[i] == 0) throw vespalib::IllegalStateException( - "Failed to initialize up state. Code likely not updated " - "after another upstate was added.", VESPA_STRLOC); + if (_upStates[i] == 0) { + throw vespalib::IllegalStateException("Failed to initialize up state. Code likely not updated " + "after another upstate was added.", VESPA_STRLOC); + } } } diff --git a/vdslib/src/vespa/vdslib/state/clusterstate.cpp b/vdslib/src/vespa/vdslib/state/clusterstate.cpp index e9159eef631..f4314a6624b 100644 --- a/vdslib/src/vespa/vdslib/state/clusterstate.cpp +++ b/vdslib/src/vespa/vdslib/state/clusterstate.cpp @@ -7,7 +7,10 @@ #include <vespa/vdslib/distribution/distribution.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/vespalib/stllike/hash_map.hpp> +#include <vespa/vespalib/stllike/hash_map_equal.hpp> #include <sstream> +#include <cassert> #include <vespa/log/log.h> LOG_SETUP(".vdslib.state.cluster"); @@ -24,9 +27,9 @@ namespace storage::lib { ClusterState::ClusterState() : Printable(), _version(0), + _nodeCount(), _clusterState(&State::DOWN), _nodeStates(), - _nodeCount(2), _description(), _distributionBits(16) { } @@ -41,14 +44,10 @@ struct NodeData { NodeData() : empty(true), node(NodeType::STORAGE, 0), ost() {} - void addTo(std::map<Node, NodeState>& nodeStates, - std::vector<uint16_t>& nodeCount) - { + void addTo(ClusterState::NodeMap & nodeStates, ClusterState::NodeCounts & nodeCount) { if (!empty) { NodeState state(ost.str(), &node.getType()); - if (state != NodeState(node.getType(), State::UP) - || state.getDescription().size() > 0) - { + if ((state != NodeState(node.getType(), State::UP)) || (state.getDescription().size() > 0)) { nodeStates.insert(std::make_pair(node, state)); } if (nodeCount[node.getType()] <= node.getIndex()) { @@ -63,9 +62,9 @@ struct NodeData { ClusterState::ClusterState(const vespalib::string& serialized) : Printable(), _version(0), + _nodeCount(), _clusterState(&State::UP), _nodeStates(), - _nodeCount(2), _description(), _distributionBits(16) { @@ -74,13 +73,13 @@ ClusterState::ClusterState(const vespalib::string& serialized) NodeData nodeData; vespalib::string lastAbsolutePath; - for (vespalib::StringTokenizer::Iterator it = st.begin(); it != st.end(); ++it) { - vespalib::string::size_type index = it->find(':'); + for (const auto & token : st) { + vespalib::string::size_type index = token.find(':'); if (index == vespalib::string::npos) { - throw IllegalArgumentException("Token " + *it + " does not contain ':': " + serialized, VESPA_STRLOC); + throw IllegalArgumentException("Token " + token + " does not contain ':': " + serialized, VESPA_STRLOC); } - vespalib::string key = it->substr(0, index); - vespalib::stringref value = it->substr(index + 1); + vespalib::string key = token.substr(0, index); + vespalib::stringref value = token.substr(index + 1); if (key.size() > 0 && key[0] == '.') { if (lastAbsolutePath == "") { throw IllegalArgumentException("The first path in system state string needs to be absolute", VESPA_STRLOC); @@ -111,7 +110,9 @@ ClusterState::parse(vespalib::stringref key, vespalib::stringref value, NodeData break; case 'b': if (key == "bits") { - _distributionBits = atoi(value.data()); + uint32_t numBits = atoi(value.data()); + assert(numBits <= 64); + _distributionBits = numBits; return true; } break; @@ -138,7 +139,7 @@ ClusterState::parse(vespalib::stringref key, vespalib::stringref value, NodeData bool ClusterState::parseSorD(vespalib::stringref key, vespalib::stringref value, NodeData & nodeData) { - const NodeType* nodeType(0); + const NodeType* nodeType = nullptr; vespalib::string::size_type dot = key.find('.'); vespalib::stringref type(dot == vespalib::string::npos ? key : key.substr(0, dot)); @@ -147,10 +148,9 @@ ClusterState::parseSorD(vespalib::stringref key, vespalib::stringref value, Node } else if (type == "distributor") { nodeType = &NodeType::DISTRIBUTOR; } - if (nodeType == 0) return false; + if (nodeType == nullptr) return false; if (dot == vespalib::string::npos) { // Entry that set node counts - uint16_t nodeCount = 0; - nodeCount = atoi(value.data()); + uint16_t nodeCount = atoi(value.data()); if (nodeCount > _nodeCount[*nodeType] ) { _nodeCount[*nodeType] = nodeCount; @@ -158,12 +158,9 @@ ClusterState::parseSorD(vespalib::stringref key, vespalib::stringref value, Node return true; } vespalib::string::size_type dot2 = key.find('.', dot + 1); - Node node; - if (dot2 == vespalib::string::npos) { - node = Node(*nodeType, atoi(key.substr(dot + 1).data())); - } else { - node = Node(*nodeType, atoi(key.substr(dot + 1, dot2 - dot - 1).data())); - } + Node node(*nodeType, (dot2 == vespalib::string::npos) + ? atoi(key.substr(dot + 1).data()) + : atoi(key.substr(dot + 1, dot2 - dot - 1).data())); if (node.getIndex() >= _nodeCount[*nodeType]) { vespalib::asciistream ost; @@ -183,74 +180,73 @@ ClusterState::parseSorD(vespalib::stringref key, vespalib::stringref value, Node return true; } +struct SeparatorPrinter { + bool first; + SeparatorPrinter() : first(true) {} + const char * toString() { + if (first) { + first = false; + return ""; + } + return " "; + } +}; + namespace { - struct SeparatorPrinter { - bool first; - SeparatorPrinter() : first(true) {} - const char * toString() { - if (first) { - first = false; - return ""; + +void +serialize_node(vespalib::asciistream & out, const Node & node, const NodeState & state) { + vespalib::asciistream prefix; + prefix << "." << node.getIndex() << "."; + vespalib::asciistream ost; + state.serialize(ost, prefix.str(), false); + vespalib::stringref content = ost.str(); + if ( !content.empty()) { + out << " " << content; + } +} + +} + +void +ClusterState::serialize_nodes(vespalib::asciistream & out, SeparatorPrinter & sep, const NodeType & nodeType, + const std::vector<NodeStatePair> & nodeStates) const +{ + uint16_t nodeCount = getNodeCount(nodeType); + if (nodeCount > 0) { + out << sep.toString() << nodeType.serialize() << ":" << nodeCount; + for (const auto & entry : nodeStates) { + if (entry.first.getType() == nodeType) { + serialize_node(out, entry.first, entry.second); } - return " "; } - }; + } } void -ClusterState::serialize(vespalib::asciistream & out, bool ignoreNewFeatures) const +ClusterState::serialize(vespalib::asciistream & out) const { SeparatorPrinter sep; - if (!ignoreNewFeatures && _version != 0) { + if (_version != 0) { out << sep.toString() << "version:" << _version; } - if (!ignoreNewFeatures && *_clusterState != State::UP) { + if (*_clusterState != State::UP) { out << sep.toString() << "cluster:" << _clusterState->serialize(); } - if (!ignoreNewFeatures && _distributionBits != 16) { + if (_distributionBits != 16) { out << sep.toString() << "bits:" << _distributionBits; } - uint16_t distCount = getNodeCount(NodeType::DISTRIBUTOR); - if (ignoreNewFeatures || distCount > 0) { - out << sep.toString() << "distributor:" << distCount; - for (std::map<Node, NodeState>::const_iterator it = - _nodeStates.begin(); - it != _nodeStates.end(); ++it) - { - if (it->first.getType() != NodeType::DISTRIBUTOR) continue; - vespalib::asciistream prefix; - prefix << "." << it->first.getIndex() << "."; - vespalib::asciistream ost; - it->second.serialize(ost, prefix.str(), false); - vespalib::stringref content = ost.str(); - if (content.size() > 0) { - out << " " << content; - } - } - } - uint16_t storCount = getNodeCount(NodeType::STORAGE); - if (ignoreNewFeatures || storCount > 0) { - out << sep.toString() << "storage:" << storCount; - for (std::map<Node, NodeState>::const_iterator it = - _nodeStates.begin(); - it != _nodeStates.end(); ++it) - { - if (it->first.getType() != NodeType::STORAGE) continue; - vespalib::asciistream prefix; - prefix << "." << it->first.getIndex() << "."; - vespalib::asciistream ost; - it->second.serialize(ost, prefix.str(), false); - vespalib::stringref content = ost.str(); - if ( !content.empty()) { - out << " " << content; - } - } - } + if ((getNodeCount(NodeType::DISTRIBUTOR) + getNodeCount(NodeType::STORAGE)) == 0u) return; + + std::vector<NodeStatePair> nodeStates(_nodeStates.cbegin(), _nodeStates.cend()); + std::sort(nodeStates.begin(), nodeStates.end(), [](const NodeStatePair &a, const NodeStatePair &b) { return a.first < b.first; }); + serialize_nodes(out, sep, NodeType::DISTRIBUTOR, nodeStates); + serialize_nodes(out, sep, NodeType::STORAGE, nodeStates); } bool -ClusterState::operator==(const ClusterState& other) const +ClusterState::operator==(const ClusterState& other) const noexcept { return (_version == other._version && *_clusterState == *other._clusterState && @@ -260,17 +256,11 @@ ClusterState::operator==(const ClusterState& other) const } bool -ClusterState::operator!=(const ClusterState& other) const +ClusterState::operator!=(const ClusterState& other) const noexcept { return !(*this == other); } -uint16_t -ClusterState::getNodeCount(const NodeType& type) const -{ - return _nodeCount[type]; -} - namespace { [[noreturn]] void throwUnknownType(const Node & node) __attribute__((noinline)); void throwUnknownType(const Node & node) { @@ -282,7 +272,7 @@ const NodeState& ClusterState::getNodeState(const Node& node) const { // If it actually has an entry in map, return that - std::map<Node, NodeState>::const_iterator it = _nodeStates.find(node); + const auto it = _nodeStates.find(node); if (it != _nodeStates.end()) return it->second; // If beyond node count, the node is down. @@ -307,9 +297,7 @@ void ClusterState::setClusterState(const State& state) { if (!state.validClusterState()) { - throw vespalib::IllegalStateException( - state.toString(true) + " is not a legal cluster state", - VESPA_STRLOC); + throw vespalib::IllegalStateException(state.toString(true) + " is not a legal cluster state", VESPA_STRLOC); } _clusterState = &state; } @@ -319,17 +307,12 @@ ClusterState::setNodeState(const Node& node, const NodeState& state) { state.verifySupportForNodeType(node.getType()); if (node.getIndex() >= _nodeCount[node.getType()]) { - for (uint32_t i = _nodeCount[node.getType()]; i < node.getIndex(); ++i) - { - _nodeStates.insert(std::make_pair( - Node(node.getType(), i), - NodeState(node.getType(), State::DOWN))); + for (uint32_t i = _nodeCount[node.getType()]; i < node.getIndex(); ++i) { + _nodeStates.insert(std::make_pair(Node(node.getType(), i), NodeState(node.getType(), State::DOWN))); } _nodeCount[node.getType()] = node.getIndex() + 1; } - if (state == NodeState(node.getType(), State::UP) - && state.getDescription().size() == 0) - { + if ((state == NodeState(node.getType(), State::UP)) && state.getDescription().empty()) { _nodeStates.erase(node); } else { _nodeStates.insert(std::make_pair(node, state)); @@ -339,32 +322,34 @@ ClusterState::setNodeState(const Node& node, const NodeState& state) } void -ClusterState::print(std::ostream& out, bool verbose, - const std::string&) const +ClusterState::print(std::ostream& out, bool verbose, const std::string&) const { (void) verbose; vespalib::asciistream tmp; - serialize(tmp, false); + serialize(tmp); out << tmp.str(); } void ClusterState::removeExtraElements() { + removeExtraElements(NodeType::STORAGE); + removeExtraElements(NodeType::DISTRIBUTOR); +} + +void +ClusterState::removeExtraElements(const NodeType & type) +{ // Simplify the system state by removing the last indexes if the nodes // are down. - for (uint32_t i=0; i<2; ++i) { - const NodeType& type(i == 0 ? NodeType::STORAGE - : NodeType::DISTRIBUTOR); - for (int32_t index = _nodeCount[type]; index >= 0; --index) { - Node node(type, index - 1); - std::map<Node, NodeState>::iterator it(_nodeStates.find(node)); - if (it == _nodeStates.end()) break; - if (it->second.getState() != State::DOWN) break; - if (it->second.getDescription() != "") break; - _nodeStates.erase(it); - --_nodeCount[type]; - } + for (int32_t index = _nodeCount[type]; index >= 0; --index) { + Node node(type, index - 1); + const auto it = _nodeStates.find(node); + if (it == _nodeStates.end()) break; + if (it->second.getState() != State::DOWN) break; + if (it->second.getDescription() != "") break; + _nodeStates.erase(it); + --_nodeCount[type]; } } @@ -413,90 +398,89 @@ void ClusterState::printStateGroupwise(std::ostream& out, const Distribution& dist, bool verbose, const std::string& indent) const { - out << "ClusterState(Version: " << _version << ", Cluster state: " - << _clusterState->toString(true) << ", Distribution bits: " - << _distributionBits << ") {"; + out << "ClusterState(Version: " << _version << ", Cluster state: " << _clusterState->toString(true) + << ", Distribution bits: " << _distributionBits << ") {"; printStateGroupwise(out, dist.getNodeGraph(), verbose, indent + " ", true); out << "\n" << indent << "}"; } namespace { - template<typename T> - std::string getNumberSpec(const std::vector<T>& numbers) { - std::ostringstream ost; - bool first = true; - uint32_t firstInRange = numbers.size() == 0 ? 0 : numbers[0];; - uint32_t lastInRange = firstInRange; - for (uint32_t i=1; i<=numbers.size(); ++i) { - if (i < numbers.size() && numbers[i] == lastInRange + 1) { - ++lastInRange; + +template<typename T> +std::string getNumberSpec(const std::vector<T>& numbers) { + std::ostringstream ost; + bool first = true; + uint32_t firstInRange = numbers.size() == 0 ? 0 : numbers[0];; + uint32_t lastInRange = firstInRange; + for (uint32_t i=1; i<=numbers.size(); ++i) { + if (i < numbers.size() && numbers[i] == lastInRange + 1) { + ++lastInRange; + } else { + if (first) { + first = false; } else { - if (first) { - first = false; - } else { - ost << ","; - } - if (firstInRange == lastInRange) { - ost << firstInRange; - } else { - ost << firstInRange << "-" << lastInRange; - } - if (i < numbers.size()) { - firstInRange = lastInRange = numbers[i]; - } + ost << ","; + } + if (firstInRange == lastInRange) { + ost << firstInRange; + } else { + ost << firstInRange << "-" << lastInRange; + } + if (i < numbers.size()) { + firstInRange = lastInRange = numbers[i]; } } - return ost.str(); } + return ost.str(); +} + +} + +size_t +ClusterState::printStateGroupwise(std::ostream& out, const Group& group, bool verbose, + const std::string& indent, const NodeType& nodeType) const +{ + NodeState defState(nodeType, State::UP); + size_t printed = 0; + for (uint16_t nodeId : group.getNodes()) { + Node node(nodeType, nodeId); + const NodeState& state(getNodeState(node)); + if (state != defState) { + out << "\n" << indent << " " << node << ": "; + state.print(out, verbose, indent + " "); + printed++; + } + } + return printed; } void -ClusterState::printStateGroupwise(std::ostream& out, const Group& group, - bool verbose, const std::string& indent, - bool rootGroup) const +ClusterState::printStateGroupwise(std::ostream& out, const Group& group, bool verbose, + const std::string& indent, bool rootGroup) const { if (rootGroup) { out << "\n" << indent << "Top group"; } else { - out << "\n" << indent << "Group " << group.getIndex() << ": " - << group.getName(); + out << "\n" << indent << "Group " << group.getIndex() << ": " << group.getName(); if (group.getCapacity() != 1.0) { out << ", capacity " << group.getCapacity(); } } out << "."; if (group.isLeafGroup()) { - out << " " << group.getNodes().size() << " node" - << (group.getNodes().size() != 1 ? "s" : "") << " [" - << getNumberSpec(group.getNodes()) << "] {"; - bool printedAny = false; - for (uint32_t j=0; j<2; ++j) { - const NodeType& nodeType( - j == 0 ? NodeType::DISTRIBUTOR : NodeType::STORAGE); - NodeState defState(nodeType, State::UP); - for (uint32_t i=0; i<group.getNodes().size(); ++i) { - Node node(nodeType, group.getNodes()[i]); - const NodeState& state(getNodeState(node)); - if (state != defState) { - out << "\n" << indent << " " << node << ": "; - state.print(out, verbose, indent + " "); - printedAny = true; - } - } - } - if (!printedAny) { + out << " " << group.getNodes().size() << " node" << (group.getNodes().size() != 1 ? "s" : "") + << " [" << getNumberSpec(group.getNodes()) << "] {"; + size_t printed = printStateGroupwise(out, group, verbose, indent, NodeType::DISTRIBUTOR) + + printStateGroupwise(out, group, verbose, indent, NodeType::STORAGE); + if (printed == 0) { out << "\n" << indent << " All nodes in group up and available."; } } else { - const std::map<uint16_t, Group*>& children(group.getSubGroups()); - out << " " << children.size() << " branch" - << (children.size() != 1 ? "es" : "") << " with distribution " - << group.getDistributionSpec() << " {"; - for (std::map<uint16_t, Group*>::const_iterator it = children.begin(); - it != children.end(); ++it) - { - printStateGroupwise(out, *it->second, verbose, - indent + " ", false); + const auto & children(group.getSubGroups()); + out << " " << children.size() << " branch" << (children.size() != 1 ? "es" : "") + << " with distribution " << group.getDistributionSpec() << " {"; + for (const auto & child : children) { + printStateGroupwise(out, *child.second, verbose,indent + " ", false); } } out << "\n" << indent << "}"; diff --git a/vdslib/src/vespa/vdslib/state/clusterstate.h b/vdslib/src/vespa/vdslib/state/clusterstate.h index 3af5a45fcac..90ec7c1aa65 100644 --- a/vdslib/src/vespa/vdslib/state/clusterstate.h +++ b/vdslib/src/vespa/vdslib/state/clusterstate.h @@ -10,26 +10,21 @@ #include "node.h" #include "nodestate.h" -#include <map> +#include <vespa/vespalib/stllike/hash_map.h> +#include <array> namespace storage::lib { class Distribution; class Group; struct NodeData; +struct SeparatorPrinter; class ClusterState : public document::Printable { - uint32_t _version; - const State* _clusterState; - std::map<Node, NodeState> _nodeStates; - std::vector<uint16_t> _nodeCount; - vespalib::string _description; - uint16_t _distributionBits; - - void getTextualDifference(std::ostringstream& builder, const NodeType& type, - const ClusterState& other) const; - public: + using NodeStatePair = std::pair<Node, NodeState>; + using NodeMap = vespalib::hash_map<Node, NodeState>; + using NodeCounts = std::array<uint16_t, 2>; using CSP = std::shared_ptr<const ClusterState>; using SP = std::shared_ptr<ClusterState>; using UP = std::unique_ptr<ClusterState>; @@ -43,31 +38,29 @@ public: ~ClusterState(); std::string getTextualDifference(const ClusterState& other) const; - void serialize(vespalib::asciistream & out, bool ignoreNewFeatures) const; + void serialize(vespalib::asciistream & out) const; - bool operator==(const ClusterState& other) const; - bool operator!=(const ClusterState& other) const; + bool operator==(const ClusterState& other) const noexcept; + bool operator!=(const ClusterState& other) const noexcept; - uint32_t getVersion() const { return _version; } + uint32_t getVersion() const noexcept { return _version; } /** * Returns the smallest number above the highest node index found of the * given type that is not down. */ - uint16_t getNodeCount(const NodeType& type) const; - uint16_t getDistributionBitCount() const { return _distributionBits; } - const State& getClusterState() const { return *_clusterState; } + uint16_t getNodeCount(const NodeType& type) const noexcept { return _nodeCount[type]; } + uint16_t getDistributionBitCount() const noexcept { return _distributionBits; } + const State& getClusterState() const noexcept { return *_clusterState; } const NodeState& getNodeState(const Node& node) const; - void setVersion(uint32_t version) { _version = version; } + void setVersion(uint32_t version) noexcept { _version = version; } void setClusterState(const State& state); void setNodeState(const Node& node, const NodeState& state); - void setDistributionBitCount(uint16_t count) { _distributionBits = count; } + void setDistributionBitCount(uint16_t count) noexcept { _distributionBits = count; } void print(std::ostream& out, bool verbose, const std::string& indent) const override; - void printStateGroupwise(std::ostream& out, - const Distribution&, bool verbose = false, - const std::string& indent = "") const; + void printStateGroupwise(std::ostream& out, const Distribution&, bool verbose, const std::string& indent) const; private: // Preconditions: `key` and `value` MUST point into null-terminated strings. @@ -75,9 +68,18 @@ private: // Preconditions: `key` and `value` MUST point into null-terminated strings. bool parseSorD(vespalib::stringref key, vespalib::stringref value, NodeData & nodeData); void removeExtraElements(); - void printStateGroupwise(std::ostream& out, const Group&, bool verbose, - const std::string& indent, bool rootGroup) const; - + void removeExtraElements(const NodeType& type); + void printStateGroupwise(std::ostream& out, const Group&, bool verbose, const std::string& indent, bool rootGroup) const; + void getTextualDifference(std::ostringstream& builder, const NodeType& type, const ClusterState& other) const; + size_t printStateGroupwise(std::ostream& out, const Group&, bool verbose, const std::string& indent, const NodeType& type) const; + void serialize_nodes(vespalib::asciistream & out, SeparatorPrinter & sep, const NodeType & nodeType, + const std::vector<NodeStatePair> & nodeStates) const; + uint32_t _version; + NodeCounts _nodeCount; + const State* _clusterState; + NodeMap _nodeStates; + vespalib::string _description; + uint16_t _distributionBits; }; } diff --git a/vdslib/src/vespa/vdslib/state/node.h b/vdslib/src/vespa/vdslib/state/node.h index 2e33e54c638..49c8f0e641b 100644 --- a/vdslib/src/vespa/vdslib/state/node.h +++ b/vdslib/src/vespa/vdslib/state/node.h @@ -13,24 +13,25 @@ namespace storage::lib { class Node { const NodeType* _type; - uint16_t _index; + uint16_t _index; public: Node() noexcept : _type(&NodeType::STORAGE), _index(0) { } Node(const NodeType& type, uint16_t index) noexcept : _type(&type), _index(index) { } - const NodeType& getType() const { return *_type; } - uint16_t getIndex() const { return _index; } + const NodeType& getType() const noexcept { return *_type; } + uint16_t getIndex() const noexcept { return _index; } + uint32_t hash() const noexcept { return (_index << 1) | *_type; } - bool operator==(const Node& other) const { + bool operator==(const Node& other) const noexcept { return (other._index == _index && *other._type == *_type); } - bool operator!=(const Node& other) const { + bool operator!=(const Node& other) const noexcept { return (other._index != _index || *other._type != *_type); } - bool operator<(const Node& n) const { + bool operator<(const Node& n) const noexcept { if (*_type != *n._type) return (*_type < *n._type); return (_index < n._index); } diff --git a/vespa-athenz/pom.xml b/vespa-athenz/pom.xml index 7c3c982af84..66b369f00fe 100644 --- a/vespa-athenz/pom.xml +++ b/vespa-athenz/pom.xml @@ -275,6 +275,52 @@ <groupId>commons-codec</groupId> <artifactId>commons-codec</artifactId> </dependency> + <dependency> + <groupId>com.google.http-client</groupId> + <artifactId>google-http-client-apache-v2</artifactId> + <exclusions> + <exclusion> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpcore</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpclient</artifactId> + </exclusion> + <exclusion> + <groupId>com.google.http-client</groupId> + <artifactId>google-http-client</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>com.google.http-client</groupId> + <artifactId>google-http-client</artifactId> + <exclusions> + <exclusion> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpcore</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpclient</artifactId> + </exclusion> + <exclusion> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>com.google.auth</groupId> + <artifactId>google-auth-library-oauth2-http</artifactId> + <exclusions> + <exclusion> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + </exclusion> + </exclusions> + </dependency> </dependencies> <build> diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/gcp/GcpCredentials.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/gcp/GcpCredentials.java new file mode 100644 index 00000000000..bbdc3c2b372 --- /dev/null +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/gcp/GcpCredentials.java @@ -0,0 +1,180 @@ +package com.yahoo.vespa.athenz.gcp; + +import com.google.api.client.http.apache.v2.ApacheHttpTransport; +import com.google.auth.http.HttpTransportFactory; +import com.google.auth.oauth2.ExternalAccountCredentials; +import com.yahoo.security.token.TokenDomain; +import com.yahoo.security.token.TokenGenerator; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Slime; +import com.yahoo.slime.SlimeUtils; +import com.yahoo.vespa.athenz.api.AthenzDomain; +import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.apache.http.impl.client.HttpClientBuilder; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.util.Objects; + +public class GcpCredentials { + private static final TokenDomain domain = TokenDomain.of("athenz-gcp-oauth2-nonce"); + + final private InputStream tokenApiStream; + private final HttpTransportFactory httpTransportFactory; + + private GcpCredentials(Builder builder) { + String clientId = builder.athenzDomain.getName() + ".gcp"; + String audience = String.format("//iam.googleapis.com/projects/%s/locations/global/workloadIdentityPools/%s/providers/%s", + builder.projectNumber, builder.workloadPoolName, builder.workloadProviderName); + String serviceUrl = String.format("https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/%s@%s.iam.gserviceaccount.com:generateAccessToken", + builder.serviceAccountName, builder.projectName); + String scope = URLEncoder.encode(generateIdTokenScope(builder.athenzDomain.getName(), builder.role), StandardCharsets.UTF_8); + String redirectUri = URLEncoder.encode(generateRedirectUri(clientId, builder.redirectURISuffix), StandardCharsets.UTF_8); + String tokenUrl = String.format("%s/oauth2/auth?response_type=id_token&client_id=%s&redirect_uri=%s&scope=%s&nonce=%s&keyType=EC&fullArn=true&output=json", + builder.ztsUrl, clientId, redirectUri, scope, TokenGenerator.generateToken(domain, "", 32).secretTokenString()); + + tokenApiStream = createTokenAPIStream(audience, serviceUrl, tokenUrl, builder.tokenLifetimeSeconds); + SSLConnectionSocketFactory sslConnectionSocketFactory = new SSLConnectionSocketFactory(builder.identityProvider.getIdentitySslContext()); + HttpClientBuilder httpClientBuilder = ApacheHttpTransport.newDefaultHttpClientBuilder() + .setSSLSocketFactory(sslConnectionSocketFactory); + httpTransportFactory = () -> new ApacheHttpTransport(httpClientBuilder.build()); + } + + public ExternalAccountCredentials getCredential() throws IOException { + return ExternalAccountCredentials.fromStream(tokenApiStream, httpTransportFactory); + } + + private InputStream createTokenAPIStream(final String audience, final String serviceUrl, final String tokenUrl, + int tokenLifetimeSeconds) { + + Slime root = new Slime(); + Cursor c = root.setObject(); + + c.setString("type", "external_account"); + c.setString("audience", audience); + c.setString("subject_token_type", "urn:ietf:params:oauth:token-type:jwt"); + c.setString("token_url", "https://sts.googleapis.com/v1/token"); + + c.setString("service_account_impersonation_url", serviceUrl); + Cursor sai = c.setObject("service_account_impersonation"); + sai.setLong("token_lifetime_seconds", tokenLifetimeSeconds); + + Cursor credentialSource = c.setObject("credential_source"); + credentialSource.setString("url", tokenUrl); + + Cursor credentialSourceFormat = credentialSource.setObject("format"); + credentialSourceFormat.setString("type", "json"); + credentialSourceFormat.setString("subject_token_field_name", "id_token"); + + try { + return new ByteArrayInputStream(SlimeUtils.toJsonBytes(root)); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private static String generateIdTokenScope(final String domainName, String roleName) { + StringBuilder scope = new StringBuilder(256); + scope.append("openid"); + scope.append(' ').append(domainName).append(":role.").append(roleName); + return scope.toString(); + } + + private static String generateRedirectUri(final String clientId, String uriSuffix) { + int idx = clientId.lastIndexOf('.'); + if (idx == -1) { + return ""; + } + final String dashDomain = clientId.substring(0, idx).replace('.', '-'); + final String service = clientId.substring(idx + 1); + return "https://" + service + "." + dashDomain + "." + uriSuffix; + } + + + public static class Builder { + private String ztsUrl; + private ServiceIdentityProvider identityProvider; + private String redirectURISuffix; + private AthenzDomain athenzDomain; + private String role; + private String projectName; + private String projectNumber; + private String serviceAccountName; + + private int tokenLifetimeSeconds = 3600; // default to 1 hour lifetime + private String workloadPoolName = "athenz"; + private String workloadProviderName = "athenz"; + + public GcpCredentials build() { + Objects.requireNonNull(ztsUrl); + Objects.requireNonNull(identityProvider); + Objects.requireNonNull(redirectURISuffix); + Objects.requireNonNull(athenzDomain); + Objects.requireNonNull(role); + Objects.requireNonNull(projectName); + Objects.requireNonNull(projectNumber); + Objects.requireNonNull(serviceAccountName); + + return new GcpCredentials(this); + } + + public Builder setZtsUrl(String ztsUrl) { + this.ztsUrl = ztsUrl; + return this; + } + + public Builder identityProvider(ServiceIdentityProvider provider) { + this.identityProvider = provider; + return this; + } + + public Builder redirectURISuffix(String redirectURISuffix) { + this.redirectURISuffix = redirectURISuffix; + return this; + } + + public Builder athenzDomain(AthenzDomain athenzDomain) { + this.athenzDomain = athenzDomain; + return this; + } + + public Builder role(String gcpRole) { + this.role = gcpRole; + return this; + } + + public Builder projectName(String projectName) { + this.projectName = projectName; + return this; + } + + public Builder projectNumber(String projectNumber) { + this.projectNumber = projectNumber; + return this; + } + + public Builder serviceAccountName(String serviceAccountName) { + this.serviceAccountName = serviceAccountName; + return this; + } + + public Builder tokenLifetimeSeconds(int tokenLifetimeSeconds) { + this.tokenLifetimeSeconds = tokenLifetimeSeconds; + return this; + } + + public Builder workloadPoolName(String workloadPoolName) { + this.workloadPoolName = workloadPoolName; + return this; + } + + public Builder workloadProviderName(String workloadProviderName) { + this.workloadProviderName = workloadProviderName; + return this; + } + } +} diff --git a/vespa-dependencies-enforcer/allowed-maven-dependencies.txt b/vespa-dependencies-enforcer/allowed-maven-dependencies.txt index 7684e3ea2ae..cb2806b66d8 100644 --- a/vespa-dependencies-enforcer/allowed-maven-dependencies.txt +++ b/vespa-dependencies-enforcer/allowed-maven-dependencies.txt @@ -24,10 +24,17 @@ com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:2.15.2 com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:2.15.2 com.fasterxml.jackson.module:jackson-module-jaxb-annotations:2.15.2 com.github.spotbugs:spotbugs-annotations:3.1.9 +com.google.auth:google-auth-library-credentials:1.19.0 +com.google.auth:google-auth-library-oauth2-http:1.19.0 +com.google.auto.value:auto-value-annotations:1.10.1 com.google.code.findbugs:jsr305:3.0.2 +com.google.code.gson:gson:2.10 com.google.errorprone:error_prone_annotations:2.18.0 com.google.guava:failureaccess:1.0.1 com.google.guava:guava:32.1.1-jre +com.google.http-client:google-http-client:1.43.3 +com.google.http-client:google-http-client-apache-v2:1.43.3 +com.google.http-client:google-http-client-gson:1.42.3 com.google.inject:guice:4.2.3:no_aop com.google.j2objc:j2objc-annotations:2.8 com.google.protobuf:protobuf-java:3.21.7 @@ -53,6 +60,7 @@ commons-io:commons-io:2.11.0 commons-logging:commons-logging:1.2 io.airlift:airline:0.9 io.dropwizard.metrics:metrics-core:3.2.5 +io.grpc:grpc-context:1.27.2 io.jsonwebtoken:jjwt-api:0.11.5 io.jsonwebtoken:jjwt-impl:0.11.5 io.jsonwebtoken:jjwt-jackson:0.11.5 @@ -67,6 +75,8 @@ io.netty:netty-transport:4.1.94.Final io.netty:netty-transport-classes-epoll:4.1.94.Final io.netty:netty-transport-native-epoll:4.1.94.Final io.netty:netty-transport-native-unix-common:4.1.94.Final +io.opencensus:opencensus-api:0.31.1 +io.opencensus:opencensus-contrib-http-util:0.31.1 io.prometheus:simpleclient:0.6.0 io.prometheus:simpleclient_common:0.6.0 javax.annotation:javax.annotation-api:1.2 diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/JettyCluster.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/JettyCluster.java index 1d9fd9d1805..819115fd867 100644 --- a/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/JettyCluster.java +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/JettyCluster.java @@ -158,6 +158,7 @@ class JettyCluster implements Cluster { return pool; }); HttpClient httpClient = new HttpClient(transport); + httpClient.setMaxRequestsQueuedPerDestination(Integer.MAX_VALUE); httpClient.setFollowRedirects(false); httpClient.setUserAgentField( new HttpField(HttpHeader.USER_AGENT, String.format("vespa-feed-client/%s (Jetty)", Vespa.VERSION))); diff --git a/vespalib/src/tests/arrayref/arrayref_test.cpp b/vespalib/src/tests/arrayref/arrayref_test.cpp index bd8646b2f99..8c41d38b292 100644 --- a/vespalib/src/tests/arrayref/arrayref_test.cpp +++ b/vespalib/src/tests/arrayref/arrayref_test.cpp @@ -53,4 +53,24 @@ TEST("require that references can be unconstified") { EXPECT_EQUAL(data[1], 5); } +TEST("require that std::array references can be constified") { + std::array<int,3> data({1,2,3}); + const ArrayRef<int> array_ref(data); + ConstArrayRef<int> const_ref(array_ref); + EXPECT_EQUAL(const_ref.size(), 3u); + EXPECT_EQUAL(const_ref.end() - const_ref.begin(), 3); + EXPECT_EQUAL(const_ref[2], 3); +} + +TEST("require that references can be unconstified") { + std::array<int, 3> data({1,2,3}); + const ConstArrayRef<int> const_ref(data); + ArrayRef<int> array_ref = unconstify(const_ref); + EXPECT_EQUAL(array_ref.size(), 3u); + EXPECT_EQUAL(array_ref.end() - array_ref.begin(), 3); + EXPECT_EQUAL(array_ref[1], 2); + array_ref[1] = 5; + EXPECT_EQUAL(data[1], 5); +} + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/vespa/vespalib/datastore/buffer_type.cpp b/vespalib/src/vespa/vespalib/datastore/buffer_type.cpp index 1b087a01c58..5c26fd42fbb 100644 --- a/vespalib/src/vespa/vespalib/datastore/buffer_type.cpp +++ b/vespalib/src/vespa/vespalib/datastore/buffer_type.cpp @@ -56,6 +56,9 @@ BufferTypeBase::BufferTypeBase(uint32_t entry_size_in, { } +BufferTypeBase::BufferTypeBase(BufferTypeBase &&rhs) noexcept = default; +BufferTypeBase & BufferTypeBase::operator=(BufferTypeBase &&rhs) noexcept = default; + BufferTypeBase::~BufferTypeBase() { assert(_holdBuffers == 0); diff --git a/vespalib/src/vespa/vespalib/datastore/buffer_type.h b/vespalib/src/vespa/vespalib/datastore/buffer_type.h index 7b23a238ba2..33ee26ad296 100644 --- a/vespalib/src/vespa/vespalib/datastore/buffer_type.h +++ b/vespalib/src/vespa/vespalib/datastore/buffer_type.h @@ -37,8 +37,8 @@ public: BufferTypeBase(const BufferTypeBase &rhs) = delete; BufferTypeBase & operator=(const BufferTypeBase &rhs) = delete; - BufferTypeBase(BufferTypeBase &&rhs) noexcept = default; - BufferTypeBase & operator=(BufferTypeBase &&rhs) noexcept = default; + BufferTypeBase(BufferTypeBase &&rhs) noexcept; + BufferTypeBase & operator=(BufferTypeBase &&rhs) noexcept; BufferTypeBase(uint32_t entry_size_in, uint32_t buffer_underflow_size_in, uint32_t arraySize, uint32_t min_entries, uint32_t max_entries) noexcept; BufferTypeBase(uint32_t entry_size_in, uint32_t buffer_underflow_size_in, uint32_t arraySize, uint32_t min_entries, uint32_t max_entries, uint32_t num_entries_for_new_buffer, float allocGrowFactor) noexcept; diff --git a/vespalib/src/vespa/vespalib/stllike/hash_map.cpp b/vespalib/src/vespa/vespalib/stllike/hash_map.cpp index 9540a47eec3..abb88fe674f 100644 --- a/vespalib/src/vespa/vespalib/stllike/hash_map.cpp +++ b/vespalib/src/vespa/vespalib/stllike/hash_map.cpp @@ -16,6 +16,7 @@ VESPALIB_HASH_MAP_INSTANTIATE(vespalib::string, double); VESPALIB_HASH_MAP_INSTANTIATE(int64_t, int32_t); VESPALIB_HASH_MAP_INSTANTIATE(int64_t, uint32_t); VESPALIB_HASH_MAP_INSTANTIATE(int32_t, uint32_t); +VESPALIB_HASH_MAP_INSTANTIATE(uint16_t, uint32_t); VESPALIB_HASH_MAP_INSTANTIATE(uint32_t, int32_t); VESPALIB_HASH_MAP_INSTANTIATE(uint32_t, uint32_t); VESPALIB_HASH_MAP_INSTANTIATE(uint64_t, uint32_t); diff --git a/vespalib/src/vespa/vespalib/stllike/hash_map.h b/vespalib/src/vespa/vespalib/stllike/hash_map.h index 889093a9550..c4f60f879d7 100644 --- a/vespalib/src/vespa/vespalib/stllike/hash_map.h +++ b/vespalib/src/vespa/vespalib/stllike/hash_map.h @@ -35,6 +35,8 @@ public: constexpr iterator end() noexcept { return _ht.end(); } constexpr const_iterator begin() const noexcept { return _ht.begin(); } constexpr const_iterator end() const noexcept { return _ht.end(); } + constexpr const_iterator cbegin() const noexcept { return _ht.begin(); } + constexpr const_iterator cend() const noexcept { return _ht.end(); } constexpr size_t capacity() const noexcept { return _ht.capacity(); } constexpr size_t size() const noexcept { return _ht.size(); } constexpr bool empty() const noexcept { return _ht.empty(); } diff --git a/vespalib/src/vespa/vespalib/stllike/hash_set.cpp b/vespalib/src/vespa/vespalib/stllike/hash_set.cpp index 8812af426bf..54614329a97 100644 --- a/vespalib/src/vespa/vespalib/stllike/hash_set.cpp +++ b/vespalib/src/vespa/vespalib/stllike/hash_set.cpp @@ -6,6 +6,8 @@ namespace vespalib { } +VESPALIB_HASH_SET_INSTANTIATE(int16_t); +VESPALIB_HASH_SET_INSTANTIATE(uint16_t); VESPALIB_HASH_SET_INSTANTIATE(int32_t); VESPALIB_HASH_SET_INSTANTIATE(uint32_t); VESPALIB_HASH_SET_INSTANTIATE(uint64_t); diff --git a/vespalib/src/vespa/vespalib/stllike/hash_set_insert.hpp b/vespalib/src/vespa/vespalib/stllike/hash_set_insert.hpp index 6d5b7ed8b05..77e46bbf9e8 100644 --- a/vespalib/src/vespa/vespalib/stllike/hash_set_insert.hpp +++ b/vespalib/src/vespa/vespalib/stllike/hash_set_insert.hpp @@ -9,7 +9,7 @@ namespace vespalib { template<typename K, typename H, typename EQ, typename M> template<typename InputIterator> hash_set<K, H, EQ, M>::hash_set(InputIterator first, InputIterator last) - : _ht(0) + : _ht(last - first) { insert(first, last); } @@ -18,7 +18,6 @@ template<typename K, typename H, typename EQ, typename M> template<typename InputIt> void hash_set<K, H, EQ, M>::insert(InputIt first, InputIt last) { - _ht.resize(last - first + capacity()); for (; first < last; first++) { insert(*first); } diff --git a/vespalib/src/vespa/vespalib/util/arrayref.h b/vespalib/src/vespa/vespalib/util/arrayref.h index 319947e4cd9..9057f56fac0 100644 --- a/vespalib/src/vespa/vespalib/util/arrayref.h +++ b/vespalib/src/vespa/vespalib/util/arrayref.h @@ -3,6 +3,7 @@ #include <cstddef> #include <vector> +#include <array> namespace vespalib { @@ -17,6 +18,8 @@ public: constexpr ArrayRef(T * v, size_t sz) noexcept : _v(v), _sz(sz) { } template<typename A=std::allocator<T>> ArrayRef(std::vector<T, A> & v) noexcept : _v(v.data()), _sz(v.size()) { } + template<size_t SZ> + ArrayRef(std::array<T, SZ> & v) noexcept : _v(v.data()), _sz(SZ) { } T & operator [] (size_t i) noexcept { return _v[i]; } const T & operator [] (size_t i) const noexcept { return _v[i]; } T * data() noexcept { return _v; } @@ -36,6 +39,8 @@ public: constexpr ConstArrayRef(const T *v, size_t sz) noexcept : _v(v), _sz(sz) { } template<typename A=std::allocator<T>> ConstArrayRef(const std::vector<T, A> & v) noexcept : _v(v.data()), _sz(v.size()) { } + template<size_t SZ> + ConstArrayRef(const std::array<T, SZ> & v) noexcept : _v(v.data()), _sz(SZ) { } ConstArrayRef(const ArrayRef<T> & v) noexcept : _v(v.data()), _sz(v.size()) { } constexpr ConstArrayRef() noexcept : _v(nullptr), _sz(0) {} const T & operator [] (size_t i) const noexcept { return _v[i]; } diff --git a/vespalib/src/vespa/vespalib/util/mmap_file_allocator.cpp b/vespalib/src/vespa/vespalib/util/mmap_file_allocator.cpp index 2c0d0f4339d..51a639a3c4e 100644 --- a/vespalib/src/vespa/vespalib/util/mmap_file_allocator.cpp +++ b/vespalib/src/vespa/vespalib/util/mmap_file_allocator.cpp @@ -2,12 +2,16 @@ #include "mmap_file_allocator.h" #include "round_up_to_page_size.h" +#include "exceptions.h" +#include "stringfmt.h" #include <vespa/vespalib/stllike/hash_map.hpp> #include <fcntl.h> #include <sys/mman.h> #include <cassert> #include <filesystem> +using vespalib::make_string_short::fmt; + namespace vespalib::alloc { MmapFileAllocator::MmapFileAllocator(const vespalib::string& dir_name) @@ -50,12 +54,12 @@ MmapFileAllocator::alloc(size_t sz) const } sz = round_up_to_page_size(sz); uint64_t offset = alloc_area(sz); - void *buf = mmap(nullptr, sz, - PROT_READ | PROT_WRITE, - MAP_SHARED, - _file.getFileDescriptor(), - offset); - assert(buf != MAP_FAILED); + void *buf = mmap(nullptr, sz, PROT_READ | PROT_WRITE, MAP_SHARED, _file.getFileDescriptor(), offset); + if (buf == MAP_FAILED) { + throw IoException(fmt("Failed mmap(nullptr, %zu, PROT_READ | PROT_WRITE, MAP_SHARED, %s(fd=%d), %lu). Reason given by OS = '%s'", + sz, _file.getFilename().c_str(), _file.getFileDescriptor(), offset, getLastErrorString().c_str()), + IoException::getErrorType(errno), VESPA_STRLOC); + } assert(buf != nullptr); // Register allocation auto ins_res = _allocations.insert(std::make_pair(buf, SizeAndOffset(sz, offset))); |