diff options
145 files changed, 1768 insertions, 1918 deletions
diff --git a/client/go/internal/cli/cmd/document.go b/client/go/internal/cli/cmd/document.go index c31f8c34d14..1e5d1c30f6e 100644 --- a/client/go/internal/cli/cmd/document.go +++ b/client/go/internal/cli/cmd/document.go @@ -171,7 +171,7 @@ https://docs.vespa.ai/en/reference/document-json-format.html#document-operations When this returns successfully, the document is guaranteed to be visible in any subsequent get or query operation. -To feed with high throughput, https://docs.vespa.ai/en/vespa-feed-client.html +To feed with high throughput, https://docs.vespa.ai/en/reference/vespa-cli/vespa_feed.html should be used instead of this.`, Example: `$ vespa document src/test/resources/A-Head-Full-of-Dreams.json`, DisableAutoGenTag: true, diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/Quota.java b/config-model-api/src/main/java/com/yahoo/config/model/api/Quota.java index 7ef92bba7e9..20940989618 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/Quota.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/Quota.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.model.api; +import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; import com.yahoo.slime.SlimeUtils; @@ -50,10 +51,13 @@ public class Quota { public Slime toSlime() { var slime = new Slime(); - var root = slime.setObject(); + toSlime(slime.setObject()); + return slime; + } + + public void toSlime(Cursor root) { maxClusterSize.ifPresent(clusterSize -> root.setLong("clusterSize", clusterSize)); budget.ifPresent(b -> root.setString("budget", b.toPlainString())); - return slime; } public static Quota unlimited() { return UNLIMITED; } diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java index 3b715c63105..dbcd1cea2fa 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java @@ -63,6 +63,7 @@ public class MockApplicationPackage implements ApplicationPackage { private final boolean failOnValidateXml; private final QueryProfileRegistry queryProfileRegistry; private final ApplicationMetaData applicationMetaData; + private final TenantName tenantName; private DeploymentSpec deploymentSpec = null; @@ -70,7 +71,7 @@ public class MockApplicationPackage implements ApplicationPackage { Map<Path, MockApplicationFile> files, String schemaDir, String deploymentSpec, String validationOverrides, boolean failOnValidateXml, - String queryProfile, String queryProfileType) { + String queryProfile, String queryProfileType, TenantName tenantName) { this.root = root; this.hostsS = hosts; this.servicesS = services; @@ -85,19 +86,20 @@ public class MockApplicationPackage implements ApplicationPackage { applicationMetaData = new ApplicationMetaData("dir", 0L, false, - ApplicationId.from(TenantName.defaultName(), + ApplicationId.from(tenantName, ApplicationName.from(APPLICATION_NAME), InstanceName.defaultName()), "checksum", APPLICATION_GENERATION, 0L); + this.tenantName = tenantName; } /** Returns the root of this application package relative to the current dir */ protected File root() { return root; } @Override - public ApplicationId getApplicationId() { return ApplicationId.from("default", "mock-application", "default"); } + public ApplicationId getApplicationId() { return ApplicationId.from(tenantName.value(), "mock-application", "default"); } @Override public Reader getServices() { @@ -246,6 +248,7 @@ public class MockApplicationPackage implements ApplicationPackage { private boolean failOnValidateXml = false; private String queryProfile = null; private String queryProfileType = null; + private TenantName tenantName = TenantName.defaultName(); public Builder() { } @@ -323,10 +326,15 @@ public class MockApplicationPackage implements ApplicationPackage { return this; } + public Builder withTenantname(TenantName tenantName) { + this.tenantName = tenantName; + return this; + } + public ApplicationPackage build() { return new MockApplicationPackage(root, hosts, services, schemas, files, schemaDir, deploymentSpec, validationOverrides, failOnValidateXml, - queryProfile, queryProfileType); + queryProfile, queryProfileType, tenantName); } } diff --git a/config-model/src/main/java/com/yahoo/schema/RankProfile.java b/config-model/src/main/java/com/yahoo/schema/RankProfile.java index 69f32daef4a..35ef12f077a 100644 --- a/config-model/src/main/java/com/yahoo/schema/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/schema/RankProfile.java @@ -100,6 +100,7 @@ public class RankProfile implements Cloneable { private Double termwiseLimit = null; private Double postFilterThreshold = null; private Double approximateThreshold = null; + private Double targetHitsMaxAdjustmentFactor = null; /** The drop limit used to drop hits with rank score less than or equal to this value */ private double rankScoreDropLimit = -Double.MAX_VALUE; @@ -768,6 +769,7 @@ public class RankProfile implements Cloneable { public void setTermwiseLimit(double termwiseLimit) { this.termwiseLimit = termwiseLimit; } public void setPostFilterThreshold(double threshold) { this.postFilterThreshold = threshold; } public void setApproximateThreshold(double threshold) { this.approximateThreshold = threshold; } + public void setTargetHitsMaxAdjustmentFactor(double factor) { this.targetHitsMaxAdjustmentFactor = factor; } public OptionalDouble getTermwiseLimit() { if (termwiseLimit != null) return OptionalDouble.of(termwiseLimit); @@ -789,6 +791,13 @@ public class RankProfile implements Cloneable { return uniquelyInherited(p -> p.getApproximateThreshold(), l -> l.isPresent(), "approximate-threshold").orElse(OptionalDouble.empty()); } + public OptionalDouble getTargetHitsMaxAdjustmentFactor() { + if (targetHitsMaxAdjustmentFactor != null) { + return OptionalDouble.of(targetHitsMaxAdjustmentFactor); + } + return uniquelyInherited(p -> p.getTargetHitsMaxAdjustmentFactor(), l -> l.isPresent(), "target-hits-max-adjustment-factor").orElse(OptionalDouble.empty()); + } + /** Whether we should ignore the default rank features. Set to null to use inherited */ public void setIgnoreDefaultRankFeatures(Boolean ignoreDefaultRankFeatures) { this.ignoreDefaultRankFeatures = ignoreDefaultRankFeatures; diff --git a/config-model/src/main/java/com/yahoo/schema/derived/RawRankProfile.java b/config-model/src/main/java/com/yahoo/schema/derived/RawRankProfile.java index 82c0c9d516a..29bd454cc62 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/RawRankProfile.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/RawRankProfile.java @@ -153,6 +153,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { private final double termwiseLimit; private final OptionalDouble postFilterThreshold; private final OptionalDouble approximateThreshold; + private final OptionalDouble targetHitsMaxAdjustmentFactor; private final double rankScoreDropLimit; private final boolean enableNestedMultivalueGrouping; @@ -197,6 +198,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { enableNestedMultivalueGrouping = deployProperties.featureFlags().enableNestedMultivalueGrouping(); postFilterThreshold = compiled.getPostFilterThreshold(); approximateThreshold = compiled.getApproximateThreshold(); + targetHitsMaxAdjustmentFactor = compiled.getTargetHitsMaxAdjustmentFactor(); keepRankCount = compiled.getKeepRankCount(); rankScoreDropLimit = compiled.getRankScoreDropLimit(); ignoreDefaultRankFeatures = compiled.getIgnoreDefaultRankFeatures(); @@ -429,6 +431,9 @@ public class RawRankProfile implements RankProfilesConfig.Producer { if (approximateThreshold.isPresent()) { properties.add(new Pair<>("vespa.matching.global_filter.lower_limit", String.valueOf(approximateThreshold.getAsDouble()))); } + if (targetHitsMaxAdjustmentFactor.isPresent()) { + properties.add(new Pair<>("vespa.matching.nns.target_hits_max_adjustment_factor", String.valueOf(targetHitsMaxAdjustmentFactor.getAsDouble()))); + } if (matchPhaseSettings != null) { properties.add(new Pair<>("vespa.matchphase.degradation.attribute", matchPhaseSettings.getAttribute())); properties.add(new Pair<>("vespa.matchphase.degradation.ascendingorder", matchPhaseSettings.getAscending() + "")); diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedRanking.java b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedRanking.java index bdecf6332a0..c25d393c8bf 100644 --- a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedRanking.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedRanking.java @@ -65,6 +65,8 @@ public class ConvertParsedRanking { (value -> profile.setPostFilterThreshold(value)); parsed.getApproximateThreshold().ifPresent (value -> profile.setApproximateThreshold(value)); + parsed.getTargetHitsMaxAdjustmentFactor().ifPresent + (value -> profile.setTargetHitsMaxAdjustmentFactor(value)); parsed.getKeepRankCount().ifPresent (value -> profile.setKeepRankCount(value)); parsed.getMinHitsPerThread().ifPresent diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ParsedRankProfile.java b/config-model/src/main/java/com/yahoo/schema/parser/ParsedRankProfile.java index 2809ee0c633..1d06b993cdc 100644 --- a/config-model/src/main/java/com/yahoo/schema/parser/ParsedRankProfile.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ParsedRankProfile.java @@ -29,6 +29,7 @@ class ParsedRankProfile extends ParsedBlock { private Double termwiseLimit = null; private Double postFilterThreshold = null; private Double approximateThreshold = null; + private Double targetHitsMaxAdjustmentFactor = null; private final List<FeatureList> matchFeatures = new ArrayList<>(); private final List<FeatureList> rankFeatures = new ArrayList<>(); private final List<FeatureList> summaryFeatures = new ArrayList<>(); @@ -65,6 +66,7 @@ class ParsedRankProfile extends ParsedBlock { Optional<Double> getTermwiseLimit() { return Optional.ofNullable(this.termwiseLimit); } Optional<Double> getPostFilterThreshold() { return Optional.ofNullable(this.postFilterThreshold); } Optional<Double> getApproximateThreshold() { return Optional.ofNullable(this.approximateThreshold); } + Optional<Double> getTargetHitsMaxAdjustmentFactor() { return Optional.ofNullable(this.targetHitsMaxAdjustmentFactor); } List<FeatureList> getMatchFeatures() { return List.copyOf(this.matchFeatures); } List<FeatureList> getRankFeatures() { return List.copyOf(this.rankFeatures); } List<FeatureList> getSummaryFeatures() { return List.copyOf(this.summaryFeatures); } @@ -231,4 +233,9 @@ class ParsedRankProfile extends ParsedBlock { this.approximateThreshold = threshold; } + void setTargetHitsMaxAdjustmentFactor(double factor) { + verifyThat(targetHitsMaxAdjustmentFactor == null, "already has target-hits-max-adjustment-factor"); + this.targetHitsMaxAdjustmentFactor = factor; + } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Logserver.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Logserver.java index 7d7d0007b5e..2a0839e209d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/Logserver.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Logserver.java @@ -15,7 +15,6 @@ import java.util.Optional; */ public class Logserver extends AbstractService { - private static final long serialVersionUID = 1L; private static final String logArchiveDir = "$ROOT/logs/vespa/logarchive"; private String compressionType = "gzip"; @@ -32,7 +31,10 @@ public class Logserver extends AbstractService { @Override public void initService(DeployState deployState) { super.initService(deployState); - this.compressionType = deployState.featureFlags().logFileCompressionAlgorithm("gzip"); + // TODO Vespa 9: Change default to zstd everywhere + this.compressionType = deployState.isHosted() + ? deployState.featureFlags().logFileCompressionAlgorithm("zstd") + : deployState.featureFlags().logFileCompressionAlgorithm("gzip"); } /** diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/InfrastructureDeploymentValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/InfrastructureDeploymentValidator.java new file mode 100644 index 00000000000..842405e68f9 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/InfrastructureDeploymentValidator.java @@ -0,0 +1,30 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation; + +import com.yahoo.config.model.ConfigModelContext; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.model.VespaModel; + +import java.util.logging.Logger; + +/** + * Validator to check that only infrastructure tenant can use non-default application-type + * + * @author mortent + */ +public class InfrastructureDeploymentValidator extends Validator { + + private static final Logger log = Logger.getLogger(InfrastructureDeploymentValidator.class.getName()); + + @Override + public void validate(VespaModel model, DeployState deployState) { + // Allow the internally defined tenant owning all infrastructure applications + if (ApplicationId.global().tenant().equals(model.applicationPackage().getApplicationId().tenant())) return; + ConfigModelContext.ApplicationType applicationType = model.getAdmin().getApplicationType(); + if (applicationType != ConfigModelContext.ApplicationType.DEFAULT) { + log.warning("Tenant %s is not allowed to use application type %s".formatted(model.applicationPackage().getApplicationId().toFullString(), applicationType)); + throw new IllegalArgumentException("Tenant is not allowed to override application type"); + } + } +} diff --git a/config-model/src/main/javacc/SchemaParser.jj b/config-model/src/main/javacc/SchemaParser.jj index b2cb258c0ab..42eeabb5ac7 100644 --- a/config-model/src/main/javacc/SchemaParser.jj +++ b/config-model/src/main/javacc/SchemaParser.jj @@ -326,6 +326,7 @@ TOKEN : | < TERMWISE_LIMIT: "termwise-limit" > | < POST_FILTER_THRESHOLD: "post-filter-threshold" > | < APPROXIMATE_THRESHOLD: "approximate-threshold" > +| < TARGET_HITS_MAX_ADJUSTMENT_FACTOR: "target-hits-max-adjustment-factor" > | < KEEP_RANK_COUNT: "keep-rank-count" > | < RANK_SCORE_DROP_LIMIT: "rank-score-drop-limit" > | < CONSTANTS: "constants" > @@ -1727,6 +1728,7 @@ void rankProfileItem(ParsedSchema schema, ParsedRankProfile profile) : { } | termwiseLimit(profile) | postFilterThreshold(profile) | approximateThreshold(profile) + | targetHitsMaxAdjustmentFactor(profile) | rankFeatures(profile) | rankProperties(profile) | secondPhase(profile) @@ -2190,6 +2192,19 @@ void approximateThreshold(ParsedRankProfile profile) : } /** + * This rule consumes a target-hits-max-adjustment-factor statement for a rank profile. + * + * @param profile the rank profile to modify + */ +void targetHitsMaxAdjustmentFactor(ParsedRankProfile profile) : +{ + double factor; +} +{ + (<TARGET_HITS_MAX_ADJUSTMENT_FACTOR> <COLON> factor = floatValue()) { profile.setTargetHitsMaxAdjustmentFactor(factor); } +} + +/** * Consumes a rank-properties block of a rank profile. There * is a little trick within this rule to allow the final rank property * to skip the terminating newline token. @@ -2641,6 +2656,7 @@ String identifierWithDash() : | <SECOND_PHASE> | <STRUCT_FIELD> | <SUMMARY_TO> + | <TARGET_HITS_MAX_ADJUSTMENT_FACTOR> | <TERMWISE_LIMIT> | <UPPER_BOUND> ) { return token.image; } diff --git a/config-model/src/main/resources/schema/content.rnc b/config-model/src/main/resources/schema/content.rnc index bb0e39a41ab..dff24745778 100644 --- a/config-model/src/main/resources/schema/content.rnc +++ b/config-model/src/main/resources/schema/content.rnc @@ -6,11 +6,11 @@ include "searchchains.rnc" Redundancy = element redundancy { attribute reply-after { xsd:nonNegativeInteger }? & - xsd:nonNegativeInteger + xsd:integer { minInclusive = "1" maxInclusive = "65534" } } MinRedundancy = element min-redundancy { - xsd:nonNegativeInteger + xsd:integer { minInclusive = "1" maxInclusive = "65534" } } DistributionType = element distribution { diff --git a/config-model/src/test/java/com/yahoo/schema/RankProfileTestCase.java b/config-model/src/test/java/com/yahoo/schema/RankProfileTestCase.java index 85225f0d255..380b458ea8c 100644 --- a/config-model/src/test/java/com/yahoo/schema/RankProfileTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/RankProfileTestCase.java @@ -459,17 +459,9 @@ public class RankProfileTestCase extends AbstractSchemaTestCase { } private void verifyApproximateNearestNeighborThresholdSettings(Double postFilterThreshold, Double approximateThreshold) throws ParseException { - var rankProfileRegistry = new RankProfileRegistry(); - var props = new TestProperties(); - var queryProfileRegistry = new QueryProfileRegistry(); - var builder = new ApplicationBuilder(rankProfileRegistry, queryProfileRegistry, props); - builder.addSchema(createSDWithRankProfileThresholds(postFilterThreshold, approximateThreshold)); - builder.build(true); - - var schema = builder.getSchema(); - var rankProfile = rankProfileRegistry.get(schema, "my_profile"); - var rawRankProfile = new RawRankProfile(rankProfile, new LargeRankingExpressions(new MockFileRegistry()), queryProfileRegistry, - new ImportedMlModels(), new AttributeFields(schema), props); + var rp = createRankProfile(postFilterThreshold, approximateThreshold, null); + var rankProfile = rp.getFirst(); + var rawRankProfile = rp.getSecond(); if (postFilterThreshold != null) { assertEquals((double)postFilterThreshold, rankProfile.getPostFilterThreshold().getAsDouble(), 0.000001); @@ -488,13 +480,52 @@ public class RankProfileTestCase extends AbstractSchemaTestCase { } } - private String createSDWithRankProfileThresholds(Double postFilterThreshold, Double approximateThreshold) { + @Test + void target_hits_max_adjustment_factor_is_configurable() throws ParseException { + verifyTargetHitsMaxAdjustmentFactor(null); + verifyTargetHitsMaxAdjustmentFactor(2.0); + } + + private void verifyTargetHitsMaxAdjustmentFactor(Double targetHitsMaxAdjustmentFactor) throws ParseException { + var rp = createRankProfile(null, null, targetHitsMaxAdjustmentFactor); + var rankProfile = rp.getFirst(); + var rawRankProfile = rp.getSecond(); + if (targetHitsMaxAdjustmentFactor != null) { + assertEquals((double)targetHitsMaxAdjustmentFactor, rankProfile.getTargetHitsMaxAdjustmentFactor().getAsDouble(), 0.000001); + assertEquals(String.valueOf(targetHitsMaxAdjustmentFactor), findProperty(rawRankProfile.configProperties(), "vespa.matching.nns.target_hits_max_adjustment_factor").get()); + } else { + assertTrue(rankProfile.getTargetHitsMaxAdjustmentFactor().isEmpty()); + assertFalse(findProperty(rawRankProfile.configProperties(), "vespa.matching.nns.target_hits_max_adjustment_factor").isPresent()); + } + } + + private Pair<RankProfile, RawRankProfile> createRankProfile(Double postFilterThreshold, + Double approximateThreshold, + Double targetHitsMaxAdjustmentFactor) throws ParseException { + var rankProfileRegistry = new RankProfileRegistry(); + var props = new TestProperties(); + var queryProfileRegistry = new QueryProfileRegistry(); + var builder = new ApplicationBuilder(rankProfileRegistry, queryProfileRegistry, props); + builder.addSchema(createSDWithRankProfile(postFilterThreshold, approximateThreshold, targetHitsMaxAdjustmentFactor)); + builder.build(true); + + var schema = builder.getSchema(); + var rankProfile = rankProfileRegistry.get(schema, "my_profile"); + var rawRankProfile = new RawRankProfile(rankProfile, new LargeRankingExpressions(new MockFileRegistry()), queryProfileRegistry, + new ImportedMlModels(), new AttributeFields(schema), props); + return new Pair<>(rankProfile, rawRankProfile); + } + + private String createSDWithRankProfile(Double postFilterThreshold, + Double approximateThreshold, + Double targetHitsMaxAdjustmentFactor) { return joinLines( "search test {", " document test {}", " rank-profile my_profile {", - (postFilterThreshold != null ? (" post-filter-threshold: " + postFilterThreshold) : ""), - (approximateThreshold != null ? (" approximate-threshold: " + approximateThreshold) : ""), + (postFilterThreshold != null ? (" post-filter-threshold: " + postFilterThreshold) : ""), + (approximateThreshold != null ? (" approximate-threshold: " + approximateThreshold) : ""), + (targetHitsMaxAdjustmentFactor != null ? (" target-hits-max-adjustment-factor: " + targetHitsMaxAdjustmentFactor) : ""), " }", "}"); } diff --git a/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithOnnxTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithOnnxTestCase.java index 2f53dba7bb4..8db8f0710a0 100644 --- a/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithOnnxTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithOnnxTestCase.java @@ -4,6 +4,8 @@ package com.yahoo.schema.processing; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.TenantName; import com.yahoo.io.IOUtils; import com.yahoo.io.reader.NamedReader; import com.yahoo.path.Path; @@ -400,7 +402,7 @@ public class RankingExpressionWithOnnxTestCase { StoringApplicationPackage(Path applicationPackageWritableRoot, String queryProfile, String queryProfileType) { super(new File(applicationPackageWritableRoot.toString()), null, null, List.of(), Map.of(), null, - null, null, false, queryProfile, queryProfileType); + null, null, false, queryProfile, queryProfileType, TenantName.defaultName()); } @Override diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/InfrastructureDeploymentValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/InfrastructureDeploymentValidatorTest.java new file mode 100644 index 00000000000..0281d5cd6ee --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/InfrastructureDeploymentValidatorTest.java @@ -0,0 +1,48 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation; + +import com.yahoo.config.model.NullConfigModelRegistry; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.model.VespaModel; +import org.junit.jupiter.api.Test; +import org.xml.sax.SAXException; + +import java.io.IOException; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +public class InfrastructureDeploymentValidatorTest { + + @Test + public void allows_infrastructure_deployments() { + assertDoesNotThrow(() -> runValidator(ApplicationId.global().tenant())); + } + + @Test + public void prevents_non_infrastructure_deployments() { + IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, () -> runValidator(TenantName.defaultName())); + assertEquals("Tenant is not allowed to override application type", exception.getMessage()); + } + + private void runValidator(TenantName tenantName) throws IOException, SAXException { + String services = """ + <services version='1.0' application-type="hosted-infrastructure"> + <container id='default' version='1.0' /> + </services> + """; + var app = new MockApplicationPackage.Builder() + .withTenantname(tenantName) + .withServices(services) + .build(); + var deployState = new DeployState.Builder().applicationPackage(app).build(); + var model = new VespaModel(new NullConfigModelRegistry(), deployState); + + var validator = new InfrastructureDeploymentValidator(); + validator.validate(model, deployState); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 3502ece9cb7..c7e4022c668 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -167,7 +167,6 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye ConfigserverConfig configserverConfig, Orchestrator orchestrator, TesterClient testerClient, - Zone zone, HealthCheckerProvider healthCheckers, Metric metric, SecretStore secretStore, @@ -698,10 +697,10 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye Optional<String> applicationPackage = Optional.empty(); Optional<Session> session = getActiveSession(applicationId); if (session.isPresent()) { - FileReference applicationPackageReference = session.get().getApplicationPackageReference(); + Optional<FileReference> applicationPackageReference = session.get().getApplicationPackageReference(); File downloadDirectory = new File(Defaults.getDefaults().underVespaHome(configserverConfig().fileReferencesDir())); - if (applicationPackageReference != null && ! fileReferenceExistsOnDisk(downloadDirectory, applicationPackageReference)) - applicationPackage = Optional.of(applicationPackageReference.value()); + if (applicationPackageReference.isPresent() && ! fileReferenceExistsOnDisk(downloadDirectory, applicationPackageReference.get())) + applicationPackage = Optional.of(applicationPackageReference.get().value()); } return applicationPackage; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java index 0acf32d79a7..efa62625159 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java @@ -96,8 +96,8 @@ public class ZooKeeperClient { Path zkPath = getZooKeeperAppPath(USERAPP_ZK_SUBPATH).append(SCHEMAS_DIR); curator.create(zkPath); // Ensures that ranking expressions and other files are also written - writeDir(app.getFile(ApplicationPackage.SEARCH_DEFINITIONS_DIR), zkPath, true); - writeDir(app.getFile(ApplicationPackage.SCHEMAS_DIR), zkPath, true); + writeDir(app.getFile(ApplicationPackage.SEARCH_DEFINITIONS_DIR), zkPath); + writeDir(app.getFile(ApplicationPackage.SCHEMAS_DIR), zkPath); for (NamedReader sd : schemas) { curator.set(zkPath.append(sd.getName()), Utf8.toBytes(com.yahoo.io.IOUtils.readAll(sd.getReader()))); sd.getReader().close(); @@ -105,7 +105,7 @@ public class ZooKeeperClient { } /** - * Puts some of the application package files into ZK - see write(app). + * Writes some application package files into ZK - see write(app). * * @param app the application package to use as input. * @throws java.io.IOException if not able to write to Zookeeper @@ -118,45 +118,40 @@ public class ZooKeeperClient { writeFile(app.getFile(Path.fromString(VALIDATION_OVERRIDES.getName())), getZooKeeperAppPath(USERAPP_ZK_SUBPATH)); writeDir(app.getFile(RULES_DIR), getZooKeeperAppPath(USERAPP_ZK_SUBPATH).append(RULES_DIR), - (path) -> path.getName().endsWith(ApplicationPackage.RULES_NAME_SUFFIX), - true); + (path) -> path.getName().endsWith(ApplicationPackage.RULES_NAME_SUFFIX)); writeDir(app.getFile(QUERY_PROFILES_DIR), getZooKeeperAppPath(USERAPP_ZK_SUBPATH).append(QUERY_PROFILES_DIR), - xmlFilter, true); + xmlFilter); writeDir(app.getFile(PAGE_TEMPLATES_DIR), getZooKeeperAppPath(USERAPP_ZK_SUBPATH).append(PAGE_TEMPLATES_DIR), - xmlFilter, true); + xmlFilter); writeDir(app.getFile(Path.fromString(SEARCHCHAINS_DIR)), getZooKeeperAppPath(USERAPP_ZK_SUBPATH).append(SEARCHCHAINS_DIR), - xmlFilter, true); + xmlFilter); writeDir(app.getFile(Path.fromString(DOCPROCCHAINS_DIR)), getZooKeeperAppPath(USERAPP_ZK_SUBPATH).append(DOCPROCCHAINS_DIR), - xmlFilter, true); + xmlFilter); writeDir(app.getFile(Path.fromString(ROUTINGTABLES_DIR)), getZooKeeperAppPath(USERAPP_ZK_SUBPATH).append(ROUTINGTABLES_DIR), - xmlFilter, true); + xmlFilter); writeDir(app.getFile(MODELS_GENERATED_REPLICATED_DIR), - getZooKeeperAppPath(USERAPP_ZK_SUBPATH).append(MODELS_GENERATED_REPLICATED_DIR), - true); + getZooKeeperAppPath(USERAPP_ZK_SUBPATH).append(MODELS_GENERATED_REPLICATED_DIR)); writeDir(app.getFile(SECURITY_DIR), - getZooKeeperAppPath(USERAPP_ZK_SUBPATH).append(SECURITY_DIR), - true); + getZooKeeperAppPath(USERAPP_ZK_SUBPATH).append(SECURITY_DIR)); } - private void writeDir(ApplicationFile file, Path zooKeeperAppPath, boolean recurse) throws IOException { - writeDir(file, zooKeeperAppPath, (__) -> true, recurse); + private void writeDir(ApplicationFile file, Path zooKeeperAppPath) throws IOException { + writeDir(file, zooKeeperAppPath, (__) -> true); } - private void writeDir(ApplicationFile dir, Path path, ApplicationFile.PathFilter filenameFilter, boolean recurse) throws IOException { + private void writeDir(ApplicationFile dir, Path path, ApplicationFile.PathFilter filenameFilter) throws IOException { if ( ! dir.isDirectory()) return; for (ApplicationFile file : listFiles(dir, filenameFilter)) { String name = file.getPath().getName(); if (name.startsWith(".")) continue; //.svn , .git ... if (file.isDirectory()) { curator.create(path.append(name)); - if (recurse) { - writeDir(file, path.append(name), filenameFilter, recurse); - } + writeDir(file, path.append(name), filenameFilter); } else { writeFile(file, path); } @@ -202,9 +197,7 @@ public class ZooKeeperClient { if (files == null || files.isEmpty()) { curator.create(getZooKeeperAppPath(USERAPP_ZK_SUBPATH + "/" + userInclude)); } - writeDir(dir, - getZooKeeperAppPath(USERAPP_ZK_SUBPATH + "/" + userInclude), - xmlFilter, true); + writeDir(dir, getZooKeeperAppPath(USERAPP_ZK_SUBPATH + "/" + userInclude), xmlFilter); } } @@ -249,7 +242,7 @@ public class ZooKeeperClient { .forEach(path -> curator.delete(getZooKeeperAppPath(path))); } catch (Exception e) { logger.log(Level.WARNING, "Could not clean up in zookeeper: " + Exceptions.toMessageString(e)); - //Might be called in an exception handler before re-throw, so do not throw here. + // Might be called in an exception handler before re-throw, so do not throw here. } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java index da18c4e4fcc..6fe133958f5 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java @@ -24,12 +24,14 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.attribute.BasicFileAttributes; import java.time.Clock; +import java.util.Optional; import java.util.concurrent.TimeUnit; import java.util.function.Function; import java.util.logging.Level; import java.util.logging.Logger; import static com.yahoo.yolean.Exceptions.uncheck; +import static java.util.logging.Level.INFO; /** * Global file directory, holding files for file distribution for all deployed applications. @@ -40,7 +42,6 @@ public class FileDirectory extends AbstractComponent { private static final Logger log = Logger.getLogger(FileDirectory.class.getName()); private final Locks<FileReference> locks = new Locks<>(1, TimeUnit.MINUTES); - private final File root; @Inject @@ -67,7 +68,7 @@ public class FileDirectory extends AbstractComponent { } } - static private class Filter implements FilenameFilter { + private static class Filter implements FilenameFilter { @Override public boolean accept(File dir, String name) { return !".".equals(name) && !"..".equals(name) ; @@ -78,17 +79,23 @@ public class FileDirectory extends AbstractComponent { return root.getAbsolutePath() + "/" + ref.value(); } - public File getFile(FileReference reference) { + public Optional<File> getFile(FileReference reference) { ensureRootExist(); File dir = new File(getPath(reference)); - if (!dir.exists()) - throw new IllegalArgumentException("File reference '" + reference.value() + "' with absolute path '" + dir.getAbsolutePath() + "' does not exist."); - if (!dir.isDirectory()) - throw new IllegalArgumentException("File reference '" + reference.value() + "' with absolute path '" + dir.getAbsolutePath() + "' is not a directory."); - File [] files = dir.listFiles(new Filter()); - if (files == null || files.length == 0) - throw new IllegalArgumentException("File reference '" + reference.value() + "' with absolute path '" + dir.getAbsolutePath() + " does not contain any files"); - return files[0]; + if (!dir.exists()) { + log.log(INFO, "File reference '" + reference.value() + "' ('" + dir.getAbsolutePath() + "') does not exist."); + return Optional.empty(); + } + if (!dir.isDirectory()) { + log.log(INFO, "File reference '" + reference.value() + "' ('" + dir.getAbsolutePath() + ")' is not a directory."); + return Optional.empty(); + } + File[] files = dir.listFiles(new Filter()); + if (files == null || files.length == 0) { + log.log(INFO, "File reference '" + reference.value() + "' ('" + dir.getAbsolutePath() + "') does not contain any files"); + return Optional.empty(); + } + return Optional.of(files[0]); } public File getRoot() { return root; } @@ -136,7 +143,7 @@ public class FileDirectory extends AbstractComponent { private void deleteDirRecursively(File dir) { log.log(Level.FINE, "Will delete dir " + dir); if ( ! IOUtils.recursiveDeleteDir(dir)) - log.log(Level.INFO, "Failed to delete " + dir); + log.log(INFO, "Failed to delete " + dir); } // Check if we should add file, it might already exist diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java index dcd2720ae3e..e45c3a8e380 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java @@ -19,6 +19,7 @@ import com.yahoo.vespa.filedistribution.FileReferenceData; import com.yahoo.vespa.filedistribution.FileReferenceDownload; import com.yahoo.vespa.filedistribution.LazyFileReferenceData; import com.yahoo.vespa.filedistribution.LazyTemporaryStorageFileReferenceData; + import java.io.File; import java.io.IOException; import java.io.UncheckedIOException; @@ -27,6 +28,7 @@ import java.nio.file.Path; import java.time.Duration; import java.time.Instant; import java.util.List; +import java.util.Optional; import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -42,7 +44,6 @@ import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType.gzip; import static com.yahoo.vespa.filedistribution.FileReferenceData.Type; import static com.yahoo.vespa.filedistribution.FileReferenceData.Type.compressed; -import static com.yahoo.yolean.Exceptions.uncheck; public class FileServer { @@ -108,26 +109,24 @@ public class FileServer { } private boolean hasFile(FileReference reference) { - try { - return fileDirectory.getFile(reference).exists(); - } catch (IllegalArgumentException e) { - log.log(Level.FINE, () -> "Failed locating " + reference + ": " + e.getMessage()); - } + Optional<File> file = fileDirectory.getFile(reference); + if (file.isPresent()) + return file.get().exists(); + + log.log(Level.FINE, () -> "Failed locating " + reference); return false; } FileDirectory getRootDir() { return fileDirectory; } - void startFileServing(FileReference reference, Receiver target, Set<CompressionType> acceptedCompressionTypes) { - File file = fileDirectory.getFile(reference); - if ( ! file.exists()) return; - + void startFileServing(FileReference reference, File file, Receiver target, Set<CompressionType> acceptedCompressionTypes) { + var absolutePath = file.getAbsolutePath(); try (FileReferenceData fileData = fileReferenceData(reference, acceptedCompressionTypes, file)) { - log.log(Level.FINE, () -> "Start serving " + reference.value() + " with file '" + file.getAbsolutePath() + "'"); + log.log(Level.FINE, () -> "Start serving " + reference.value() + " with file '" + absolutePath + "'"); target.receive(fileData, new ReplayStatus(0, "OK")); - log.log(Level.FINE, () -> "Done serving " + reference.value() + " with file '" + file.getAbsolutePath() + "'"); + log.log(Level.FINE, () -> "Done serving " + reference.value() + " with file '" + absolutePath + "'"); } catch (IOException ioe) { - throw new UncheckedIOException("For " + reference.value() + ": failed reading file '" + file.getAbsolutePath() + "'" + + throw new UncheckedIOException("For " + reference.value() + ": failed reading file '" + absolutePath + "'" + " for sending to '" + target.toString() + "'. ", ioe); } catch (Exception e) { throw new RuntimeException("Failed serving " + reference.value() + " to '" + target + "': ", e); @@ -177,10 +176,10 @@ public class FileServer { try { var fileReferenceDownload = new FileReferenceDownload(fileReference, client, downloadFromOtherSourceIfNotFound); - boolean fileExists = hasFileDownloadIfNeeded(fileReferenceDownload); - if ( ! fileExists) return NOT_FOUND; + var file = getFileDownloadIfNeeded(fileReferenceDownload); + if (file.isEmpty()) return NOT_FOUND; - startFileServing(fileReference, receiver, acceptedCompressionTypes); + startFileServing(fileReference, file.get(), receiver, acceptedCompressionTypes); } catch (Exception e) { log.warning("Failed serving file reference '" + fileReference + "', request from " + client + " failed with: " + e.getMessage()); return TRANSFER_FAILED; @@ -199,9 +198,11 @@ public class FileServer { acceptedCompressionTypes + ", compression types server can use: " + compressionTypes); } - boolean hasFileDownloadIfNeeded(FileReferenceDownload fileReferenceDownload) { + public Optional<File> getFileDownloadIfNeeded(FileReferenceDownload fileReferenceDownload) { FileReference fileReference = fileReferenceDownload.fileReference(); - if (hasFile(fileReference)) return true; + Optional<File> file = fileDirectory.getFile(fileReference); + if (file.isPresent()) + return file; if (fileReferenceDownload.downloadFromOtherSourceIfNotFound()) { log.log(Level.FINE, "File not found, downloading from another source"); @@ -210,13 +211,13 @@ public class FileServer { FileReferenceDownload newDownload = new FileReferenceDownload(fileReference, fileReferenceDownload.client(), false); - boolean fileExists = downloader.getFile(newDownload).isPresent(); - if ( ! fileExists) + file = downloader.getFile(newDownload); + if (file.isEmpty()) log.log(Level.INFO, "Failed downloading '" + fileReferenceDownload + "'"); - return fileExists; + return file; } else { log.log(Level.FINE, "File not found, will not download from another source"); - return false; + return Optional.empty(); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java index 22ef6cc2547..031574bec77 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java @@ -66,15 +66,15 @@ public class ApplicationPackageMaintainer extends ConfigServerMaintainer { Optional<Session> session = applicationRepository.getActiveSession(applicationId); if (session.isEmpty()) continue; // App might be deleted after call to listApplications() or not activated yet (bootstrap phase) - FileReference appFileReference = session.get().getApplicationPackageReference(); - if (appFileReference != null) { + Optional<FileReference> appFileReference = session.get().getApplicationPackageReference(); + if (appFileReference.isPresent()) { long sessionId = session.get().getSessionId(); attempts++; - if (!fileReferenceExistsOnDisk(downloadDirectory, appFileReference)) { + if (!fileReferenceExistsOnDisk(downloadDirectory, appFileReference.get())) { log.fine(() -> "Downloading application package with file reference " + appFileReference + " for " + applicationId + " (session " + sessionId + ")"); - FileReferenceDownload download = new FileReferenceDownload(appFileReference, + FileReferenceDownload download = new FileReferenceDownload(appFileReference.get(), this.getClass().getSimpleName(), false); if (fileDownloader.getFile(download).isEmpty()) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java index b627fe9ba3b..eb359f9ffc6 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java @@ -94,14 +94,10 @@ public abstract class Session implements Comparable<Session> { * @return log preamble */ public String logPre() { - Optional<ApplicationId> applicationId; + Optional<ApplicationId> applicationId = getOptionalApplicationId(); + // We might not be able to read application id from zookeeper // e.g. when the app has been deleted. Use tenant name in that case. - try { - applicationId = Optional.of(getApplicationId()); - } catch (Exception e) { - applicationId = Optional.empty(); - } return applicationId .filter(appId -> ! appId.equals(ApplicationId.defaultId())) .map(TenantRepository::logPre) @@ -116,46 +112,6 @@ public abstract class Session implements Comparable<Session> { return sessionZooKeeperClient.readActivatedTime(); } - public void setApplicationId(ApplicationId applicationId) { - sessionZooKeeperClient.writeApplicationId(applicationId); - } - - void setApplicationPackageReference(FileReference applicationPackageReference) { - sessionZooKeeperClient.writeApplicationPackageReference(Optional.ofNullable(applicationPackageReference)); - } - - public void setVespaVersion(Version version) { - sessionZooKeeperClient.writeVespaVersion(version); - } - - public void setDockerImageRepository(Optional<DockerImage> dockerImageRepository) { - sessionZooKeeperClient.writeDockerImageRepository(dockerImageRepository); - } - - public void setAthenzDomain(Optional<AthenzDomain> athenzDomain) { - sessionZooKeeperClient.writeAthenzDomain(athenzDomain); - } - - public void setQuota(Optional<Quota> quota) { - sessionZooKeeperClient.writeQuota(quota); - } - - public void setTenantSecretStores(List<TenantSecretStore> tenantSecretStores) { - sessionZooKeeperClient.writeTenantSecretStores(tenantSecretStores); - } - - public void setOperatorCertificates(List<X509Certificate> operatorCertificates) { - sessionZooKeeperClient.writeOperatorCertificates(operatorCertificates); - } - - public void setCloudAccount(Optional<CloudAccount> cloudAccount) { - sessionZooKeeperClient.writeCloudAccount(cloudAccount); - } - - public void setDataplaneTokens(List<DataplaneToken> dataplaneTokens) { - sessionZooKeeperClient.writeDataplaneTokens(dataplaneTokens); - } - /** Returns application id read from ZooKeeper. Will throw RuntimeException if not found */ public ApplicationId getApplicationId() { return sessionZooKeeperClient.readApplicationId(); } @@ -168,7 +124,7 @@ public abstract class Session implements Comparable<Session> { } } - public FileReference getApplicationPackageReference() {return sessionZooKeeperClient.readApplicationPackageReference(); } + public Optional<FileReference> getApplicationPackageReference() { return sessionZooKeeperClient.readApplicationPackageReference(); } public Optional<DockerImage> getDockerImageRepository() { return sessionZooKeeperClient.readDockerImageRepository(); } @@ -202,6 +158,8 @@ public abstract class Session implements Comparable<Session> { return sessionZooKeeperClient.readDataplaneTokens(); } + public SessionZooKeeperClient getSessionZooKeeperClient() { return sessionZooKeeperClient; } + private Transaction createSetStatusTransaction(Status status) { return sessionZooKeeperClient.createWriteStatusTransaction(status); } @@ -226,7 +184,7 @@ public abstract class Session implements Comparable<Session> { return getApplicationPackage().getFile(relativePath); } - Optional<ApplicationSet> applicationSet() { return Optional.empty(); }; + Optional<ApplicationSet> applicationSet() { return Optional.empty(); } private void markSessionEdited() { setStatus(Session.Status.NEW); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionData.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionData.java new file mode 100644 index 00000000000..1fb72e1253e --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionData.java @@ -0,0 +1,87 @@ +package com.yahoo.vespa.config.server.session; + +import com.yahoo.component.Version; +import com.yahoo.config.FileReference; +import com.yahoo.config.model.api.Quota; +import com.yahoo.config.model.api.TenantSecretStore; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.AthenzDomain; +import com.yahoo.config.provision.CloudAccount; +import com.yahoo.config.provision.DataplaneToken; +import com.yahoo.config.provision.DockerImage; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Slime; +import com.yahoo.slime.SlimeUtils; +import com.yahoo.vespa.config.server.tenant.DataplaneTokenSerializer; +import com.yahoo.vespa.config.server.tenant.OperatorCertificateSerializer; +import com.yahoo.vespa.config.server.tenant.TenantSecretStoreSerializer; + +import java.io.IOException; +import java.security.cert.X509Certificate; +import java.util.List; +import java.util.Optional; + +/** + * Data class for session information, typically parameters supplied in a deployment request that needs + * to be persisted in ZooKeeper. These will be used when creating a new session based on an existing one. + * + * @author hmusum + */ +public record SessionData(ApplicationId applicationId, + Optional<FileReference> applicationPackageReference, + Version version, + Optional<DockerImage> dockerImageRepository, + Optional<AthenzDomain> athenzDomain, + Optional<Quota> quota, + List<TenantSecretStore> tenantSecretStores, + List<X509Certificate> operatorCertificates, + Optional<CloudAccount> cloudAccount, + List<DataplaneToken> dataplaneTokens) { + + // NOTE: Any state added here MUST also be propagated in com.yahoo.vespa.config.server.deploy.Deployment.prepare() + static final String APPLICATION_ID_PATH = "applicationId"; + static final String APPLICATION_PACKAGE_REFERENCE_PATH = "applicationPackageReference"; + static final String VERSION_PATH = "version"; + static final String CREATE_TIME_PATH = "createTime"; + static final String DOCKER_IMAGE_REPOSITORY_PATH = "dockerImageRepository"; + static final String ATHENZ_DOMAIN = "athenzDomain"; + static final String QUOTA_PATH = "quota"; + static final String TENANT_SECRET_STORES_PATH = "tenantSecretStores"; + static final String OPERATOR_CERTIFICATES_PATH = "operatorCertificates"; + static final String CLOUD_ACCOUNT_PATH = "cloudAccount"; + static final String DATAPLANE_TOKENS_PATH = "dataplaneTokens"; + static final String SESSION_DATA_PATH = "sessionData"; + + public byte[] toJson() { + try { + Slime slime = new Slime(); + toSlime(slime.setObject()); + return SlimeUtils.toJsonBytes(slime); + } + catch (IOException e) { + throw new RuntimeException("Serialization of session data to json failed", e); + } + } + + private void toSlime(Cursor object) { + object.setString(APPLICATION_ID_PATH, applicationId.serializedForm()); + applicationPackageReference.ifPresent(ref -> object.setString(APPLICATION_PACKAGE_REFERENCE_PATH, ref.value())); + object.setString(VERSION_PATH, version.toString()); + object.setLong(CREATE_TIME_PATH, System.currentTimeMillis()); + dockerImageRepository.ifPresent(image -> object.setString(DOCKER_IMAGE_REPOSITORY_PATH, image.asString())); + athenzDomain.ifPresent(domain -> object.setString(ATHENZ_DOMAIN, domain.value())); + quota.ifPresent(q -> q.toSlime(object.setObject(QUOTA_PATH))); + + Cursor tenantSecretStoresArray = object.setArray(TENANT_SECRET_STORES_PATH); + TenantSecretStoreSerializer.toSlime(tenantSecretStores, tenantSecretStoresArray); + + Cursor operatorCertificatesArray = object.setArray(OPERATOR_CERTIFICATES_PATH); + OperatorCertificateSerializer.toSlime(operatorCertificates, operatorCertificatesArray); + + cloudAccount.ifPresent(account -> object.setString(CLOUD_ACCOUNT_PATH, account.value())); + + Cursor dataplaneTokensArray = object.setArray(DATAPLANE_TOKENS_PATH); + DataplaneTokenSerializer.toSlime(dataplaneTokens, dataplaneTokensArray); + } + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java index ae87a0dd182..8d45ac7e8f1 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java @@ -36,6 +36,7 @@ import com.yahoo.vespa.config.server.ConfigServerSpec; import com.yahoo.vespa.config.server.TimeoutBudget; import com.yahoo.vespa.config.server.application.ApplicationSet; import com.yahoo.vespa.config.server.configchange.ConfigChangeActions; +import com.yahoo.vespa.config.server.deploy.ZooKeeperClient; import com.yahoo.vespa.config.server.deploy.ZooKeeperDeployer; import com.yahoo.vespa.config.server.filedistribution.FileDistributionFactory; import com.yahoo.vespa.config.server.host.HostValidator; @@ -49,7 +50,9 @@ import com.yahoo.vespa.config.server.tenant.EndpointCertificateMetadataStore; import com.yahoo.vespa.config.server.tenant.EndpointCertificateRetriever; import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.flags.BooleanFlag; import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.model.application.validation.BundleValidator; import org.xml.sax.SAXException; import javax.xml.parsers.ParserConfigurationException; @@ -71,6 +74,8 @@ import java.util.logging.Logger; import java.util.stream.Collectors; import java.util.zip.ZipException; +import static com.yahoo.vespa.config.server.session.SessionZooKeeperClient.getSessionPath; + /** * A SessionPreparer is responsible for preparing a session given an application package. * @@ -90,6 +95,7 @@ public class SessionPreparer { private final SecretStore secretStore; private final FlagSource flagSource; private final ExecutorService executor; + private final BooleanFlag writeSessionData; public SessionPreparer(ModelFactoryRegistry modelFactoryRegistry, FileDistributionFactory fileDistributionFactory, @@ -111,6 +117,7 @@ public class SessionPreparer { this.secretStore = secretStore; this.flagSource = flagSource; this.executor = executor; + this.writeSessionData = Flags.WRITE_CONFIG_SERVER_SESSION_DATA_AS_ONE_BLOB.bindTo(flagSource); } ExecutorService getExecutor() { return executor; } @@ -335,7 +342,7 @@ public class SessionPreparer { writeStateToZooKeeper(sessionZooKeeperClient, preprocessedApplicationPackage, applicationId, - filereference, + Optional.of(filereference), dockerImageRepository, vespaVersion, logger, @@ -377,7 +384,7 @@ public class SessionPreparer { private void writeStateToZooKeeper(SessionZooKeeperClient zooKeeperClient, ApplicationPackage applicationPackage, ApplicationId applicationId, - FileReference fileReference, + Optional<FileReference> fileReference, Optional<DockerImage> dockerImageRepository, Version vespaVersion, DeployLogger deployLogger, @@ -389,20 +396,22 @@ public class SessionPreparer { List<X509Certificate> operatorCertificates, Optional<CloudAccount> cloudAccount, List<DataplaneToken> dataplaneTokens) { - ZooKeeperDeployer zkDeployer = zooKeeperClient.createDeployer(deployLogger); + Path sessionPath = getSessionPath(applicationId.tenant(), zooKeeperClient.sessionId()); + ZooKeeperDeployer zkDeployer = new ZooKeeperDeployer(new ZooKeeperClient(curator, deployLogger, sessionPath)); try { zkDeployer.deploy(applicationPackage, fileRegistryMap, allocatedHosts); - // Note: When changing the below you need to also change similar calls in SessionRepository.createSessionFromExisting() - zooKeeperClient.writeApplicationId(applicationId); - zooKeeperClient.writeApplicationPackageReference(Optional.of(fileReference)); - zooKeeperClient.writeVespaVersion(vespaVersion); - zooKeeperClient.writeDockerImageRepository(dockerImageRepository); - zooKeeperClient.writeAthenzDomain(athenzDomain); - zooKeeperClient.writeQuota(quota); - zooKeeperClient.writeTenantSecretStores(tenantSecretStores); - zooKeeperClient.writeOperatorCertificates(operatorCertificates); - zooKeeperClient.writeCloudAccount(cloudAccount); - zooKeeperClient.writeDataplaneTokens(dataplaneTokens); + new SessionSerializer().write(zooKeeperClient, + applicationId, + fileReference, + dockerImageRepository, + vespaVersion, + athenzDomain, + quota, + tenantSecretStores, + operatorCertificates, + cloudAccount, + dataplaneTokens, + writeSessionData); } catch (RuntimeException | IOException e) { zkDeployer.cleanup(); throw new RuntimeException("Error preparing session", e); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java index f82aa405380..1af728919d9 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java @@ -6,7 +6,6 @@ import com.google.common.collect.Multiset; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.concurrent.DaemonThreadFactory; import com.yahoo.concurrent.StripedExecutor; -import com.yahoo.config.FileReference; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.api.ConfigDefinitionRepo; @@ -27,7 +26,6 @@ import com.yahoo.vespa.config.server.application.ApplicationSet; import com.yahoo.vespa.config.server.application.TenantApplications; import com.yahoo.vespa.config.server.configchange.ConfigChangeActions; import com.yahoo.vespa.config.server.deploy.TenantFileSystemDirs; -import com.yahoo.vespa.config.server.filedistribution.FileDirectory; import com.yahoo.vespa.config.server.filedistribution.FileDistributionFactory; import com.yahoo.vespa.config.server.http.InvalidApplicationException; import com.yahoo.vespa.config.server.http.UnknownVespaVersionException; @@ -41,7 +39,9 @@ import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.vespa.config.server.zookeeper.SessionCounter; import com.yahoo.vespa.config.server.zookeeper.ZKApplication; import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.flags.BooleanFlag; import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.LongFlag; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.flags.UnboundStringFlag; @@ -127,6 +127,7 @@ public class SessionRepository { private final ConfigDefinitionRepo configDefinitionRepo; private final int maxNodeSize; private final LongFlag expiryTimeFlag; + private final BooleanFlag writeSessionData; public SessionRepository(TenantName tenantName, TenantApplications applicationRepo, @@ -168,7 +169,8 @@ public class SessionRepository { this.modelFactoryRegistry = modelFactoryRegistry; this.configDefinitionRepo = configDefinitionRepo; this.maxNodeSize = maxNodeSize; - expiryTimeFlag = PermanentFlags.CONFIG_SERVER_SESSION_EXPIRY_TIME.bindTo(flagSource); + this.expiryTimeFlag = PermanentFlags.CONFIG_SERVER_SESSION_EXPIRY_TIME.bindTo(flagSource); + this.writeSessionData = Flags.WRITE_CONFIG_SERVER_SESSION_DATA_AS_ONE_BLOB.bindTo(flagSource); loadSessions(); // Needs to be done before creating cache below this.directoryCache = curator.createDirectoryCache(sessionsPath.getAbsolute(), false, false, zkCacheExecutor); @@ -266,24 +268,14 @@ public class SessionRepository { boolean internalRedeploy, TimeoutBudget timeoutBudget, DeployLogger deployLogger) { - ApplicationId existingApplicationId = existingSession.getApplicationId(); + ApplicationId applicationId = existingSession.getApplicationId(); File existingApp = getSessionAppDir(existingSession.getSessionId()); LocalSession session = createSessionFromApplication(existingApp, - existingApplicationId, + applicationId, internalRedeploy, timeoutBudget, deployLogger); - // Note: Setters below need to be kept in sync with calls in SessionPreparer.writeStateToZooKeeper() - session.setApplicationId(existingApplicationId); - session.setApplicationPackageReference(existingSession.getApplicationPackageReference()); - session.setVespaVersion(existingSession.getVespaVersion()); - session.setDockerImageRepository(existingSession.getDockerImageRepository()); - session.setAthenzDomain(existingSession.getAthenzDomain()); - session.setQuota(existingSession.getQuota()); - session.setTenantSecretStores(existingSession.getTenantSecretStores()); - session.setOperatorCertificates(existingSession.getOperatorCertificates()); - session.setCloudAccount(existingSession.getCloudAccount()); - session.setDataplaneTokens(existingSession.getDataplaneTokens()); + write(existingSession, session, applicationId); return session; } @@ -534,7 +526,6 @@ public class SessionRepository { private ApplicationSet loadApplication(Session session, Optional<ApplicationSet> previousApplicationSet) { log.log(Level.FINE, () -> "Loading application for " + session); SessionZooKeeperClient sessionZooKeeperClient = createSessionZooKeeperClient(session.getSessionId()); - ApplicationPackage applicationPackage = sessionZooKeeperClient.loadApplicationPackage(); ActivatedModelsBuilder builder = new ActivatedModelsBuilder(session.getTenantName(), session.getSessionId(), sessionZooKeeperClient, @@ -550,9 +541,9 @@ public class SessionRepository { modelFactoryRegistry, configDefinitionRepo); return ApplicationSet.fromList(builder.buildModels(session.getApplicationId(), - sessionZooKeeperClient.readDockerImageRepository(), - sessionZooKeeperClient.readVespaVersion(), - applicationPackage, + session.getDockerImageRepository(), + session.getVespaVersion(), + sessionZooKeeperClient.loadApplicationPackage(), new AllocatedHostsFromAllModels(), clock.instant())); } @@ -578,6 +569,24 @@ public class SessionRepository { }); } + // ---------------- Serialization ---------------------------------------------------------------- + + private void write(Session existingSession, LocalSession session, ApplicationId applicationId) { + SessionSerializer sessionSerializer = new SessionSerializer(); + sessionSerializer.write(session.getSessionZooKeeperClient(), + applicationId, + existingSession.getApplicationPackageReference(), + existingSession.getDockerImageRepository(), + existingSession.getVespaVersion(), + existingSession.getAthenzDomain(), + existingSession.getQuota(), + existingSession.getTenantSecretStores(), + existingSession.getOperatorCertificates(), + existingSession.getCloudAccount(), + existingSession.getDataplaneTokens(), + writeSessionData); + } + // ---------------- Common stuff ---------------------------------------------------------------- public void deleteExpiredSessions(Map<ApplicationId, Long> activeSessions) { @@ -854,23 +863,18 @@ public class SessionRepository { } SessionZooKeeperClient sessionZKClient = createSessionZooKeeperClient(sessionId); - FileReference fileReference = sessionZKClient.readApplicationPackageReference(); + var fileReference = sessionZKClient.readApplicationPackageReference(); log.log(Level.FINE, () -> "File reference for session id " + sessionId + ": " + fileReference); - if (fileReference == null) return; + if (fileReference.isEmpty()) return; + + Optional<File> sessionDir = fileDistributionFactory.fileDirectory().getFile(fileReference.get()); + // We cannot be guaranteed that the file reference exists (it could be that it has not + // been downloaded yet), and e.g. when bootstrapping we cannot throw an exception in that case + if (sessionDir.isEmpty()) return; - File sessionDir; - FileDirectory fileDirectory = fileDistributionFactory.fileDirectory(); - try { - sessionDir = fileDirectory.getFile(fileReference); - } catch (IllegalArgumentException e) { - // We cannot be guaranteed that the file reference exists (it could be that it has not - // been downloaded yet), and e.g. when bootstrapping we cannot throw an exception in that case - log.log(Level.FINE, () -> "File reference for session id " + sessionId + ": " + fileReference + " not found"); - return; - } ApplicationId applicationId = sessionZKClient.readApplicationId(); log.log(Level.FINE, () -> "Creating local session for tenant '" + tenantName + "' with session id " + sessionId); - createLocalSession(sessionDir, applicationId, sessionId); + createLocalSession(sessionDir.get(), applicationId, sessionId); } private Optional<Long> getActiveSessionId(ApplicationId applicationId) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionSerializer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionSerializer.java new file mode 100644 index 00000000000..1202b2bd08b --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionSerializer.java @@ -0,0 +1,53 @@ +package com.yahoo.vespa.config.server.session; + +import com.yahoo.component.Version; +import com.yahoo.config.FileReference; +import com.yahoo.config.model.api.Quota; +import com.yahoo.config.model.api.TenantSecretStore; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.AthenzDomain; +import com.yahoo.config.provision.CloudAccount; +import com.yahoo.config.provision.DataplaneToken; +import com.yahoo.config.provision.DockerImage; +import com.yahoo.vespa.flags.BooleanFlag; + +import java.security.cert.X509Certificate; +import java.util.List; +import java.util.Optional; + +/** + * Serialization and deserialization of session data to/from ZooKeeper. + * @author hmusum + */ +public class SessionSerializer { + + void write(SessionZooKeeperClient zooKeeperClient, ApplicationId applicationId, + Optional<FileReference> fileReference, Optional<DockerImage> dockerImageRepository, + Version vespaVersion, Optional<AthenzDomain> athenzDomain, Optional<Quota> quota, + List<TenantSecretStore> tenantSecretStores, List<X509Certificate> operatorCertificates, + Optional<CloudAccount> cloudAccount, List<DataplaneToken> dataplaneTokens, + BooleanFlag writeSessionData) { + zooKeeperClient.writeApplicationId(applicationId); + zooKeeperClient.writeApplicationPackageReference(fileReference); + zooKeeperClient.writeVespaVersion(vespaVersion); + zooKeeperClient.writeDockerImageRepository(dockerImageRepository); + zooKeeperClient.writeAthenzDomain(athenzDomain); + zooKeeperClient.writeQuota(quota); + zooKeeperClient.writeTenantSecretStores(tenantSecretStores); + zooKeeperClient.writeOperatorCertificates(operatorCertificates); + zooKeeperClient.writeCloudAccount(cloudAccount); + zooKeeperClient.writeDataplaneTokens(dataplaneTokens); + if (writeSessionData.value()) + zooKeeperClient.writeSessionData(new SessionData(applicationId, + fileReference, + vespaVersion, + dockerImageRepository, + athenzDomain, + quota, + tenantSecretStores, + operatorCertificates, + cloudAccount, + dataplaneTokens)); + } + +}
\ No newline at end of file diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java index 23b6fe075fa..7d1a7ceae4e 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java @@ -6,7 +6,6 @@ import com.yahoo.component.Version; import com.yahoo.component.Vtag; import com.yahoo.config.FileReference; import com.yahoo.config.application.api.ApplicationPackage; -import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.api.ConfigDefinitionRepo; import com.yahoo.config.model.api.Quota; import com.yahoo.config.model.api.TenantSecretStore; @@ -23,8 +22,6 @@ import com.yahoo.text.Utf8; import com.yahoo.transaction.Transaction; import com.yahoo.vespa.config.server.NotFoundException; import com.yahoo.vespa.config.server.UserConfigDefinitionRepo; -import com.yahoo.vespa.config.server.deploy.ZooKeeperClient; -import com.yahoo.vespa.config.server.deploy.ZooKeeperDeployer; import com.yahoo.vespa.config.server.filedistribution.AddFileInterface; import com.yahoo.vespa.config.server.filedistribution.MockFileManager; import com.yahoo.vespa.config.server.tenant.CloudAccountSerializer; @@ -45,6 +42,18 @@ import java.util.List; import java.util.Optional; import java.util.logging.Level; +import static com.yahoo.vespa.config.server.session.SessionData.APPLICATION_ID_PATH; +import static com.yahoo.vespa.config.server.session.SessionData.APPLICATION_PACKAGE_REFERENCE_PATH; +import static com.yahoo.vespa.config.server.session.SessionData.ATHENZ_DOMAIN; +import static com.yahoo.vespa.config.server.session.SessionData.CLOUD_ACCOUNT_PATH; +import static com.yahoo.vespa.config.server.session.SessionData.CREATE_TIME_PATH; +import static com.yahoo.vespa.config.server.session.SessionData.DATAPLANE_TOKENS_PATH; +import static com.yahoo.vespa.config.server.session.SessionData.DOCKER_IMAGE_REPOSITORY_PATH; +import static com.yahoo.vespa.config.server.session.SessionData.OPERATOR_CERTIFICATES_PATH; +import static com.yahoo.vespa.config.server.session.SessionData.QUOTA_PATH; +import static com.yahoo.vespa.config.server.session.SessionData.SESSION_DATA_PATH; +import static com.yahoo.vespa.config.server.session.SessionData.TENANT_SECRET_STORES_PATH; +import static com.yahoo.vespa.config.server.session.SessionData.VERSION_PATH; import static com.yahoo.vespa.config.server.zookeeper.ZKApplication.USER_DEFCONFIGS_ZK_SUBPATH; import static com.yahoo.vespa.curator.Curator.CompletionWaiter; import static com.yahoo.yolean.Exceptions.uncheck; @@ -61,18 +70,6 @@ public class SessionZooKeeperClient { // NOTE: Any state added here MUST also be propagated in com.yahoo.vespa.config.server.deploy.Deployment.prepare() - static final String APPLICATION_ID_PATH = "applicationId"; - static final String APPLICATION_PACKAGE_REFERENCE_PATH = "applicationPackageReference"; - private static final String VERSION_PATH = "version"; - private static final String CREATE_TIME_PATH = "createTime"; - private static final String DOCKER_IMAGE_REPOSITORY_PATH = "dockerImageRepository"; - private static final String ATHENZ_DOMAIN = "athenzDomain"; - private static final String QUOTA_PATH = "quota"; - private static final String TENANT_SECRET_STORES_PATH = "tenantSecretStores"; - private static final String OPERATOR_CERTIFICATES_PATH = "operatorCertificates"; - private static final String CLOUD_ACCOUNT_PATH = "cloudAccount"; - private static final String DATAPLANE_TOKENS_PATH = "dataplaneTokens"; - private final Curator curator; private final TenantName tenantName; private final long sessionId; @@ -180,11 +177,8 @@ public class SessionZooKeeperClient { reference -> curator.set(applicationPackageReferencePath(), Utf8.toBytes(reference.value()))); } - FileReference readApplicationPackageReference() { - Optional<byte[]> data = curator.getData(applicationPackageReferencePath()); - if (data.isEmpty()) return null; // This should not happen. - - return new FileReference(Utf8.toString(data.get())); + Optional<FileReference> readApplicationPackageReference() { + return curator.getData(applicationPackageReferencePath()).map(d -> new FileReference(Utf8.toString(d))); } private Path applicationPackageReferencePath() { @@ -227,6 +221,10 @@ public class SessionZooKeeperClient { curator.set(versionPath(), Utf8.toBytes(version.toString())); } + public void writeSessionData(SessionData sessionData) { + curator.set(sessionPath.append(SESSION_DATA_PATH), sessionData.toJson()); + } + public Version readVespaVersion() { Optional<byte[]> data = curator.getData(versionPath()); // TODO: Empty version should not be possible any more - verify and remove @@ -261,11 +259,6 @@ public class SessionZooKeeperClient { .orElseThrow(() -> new IllegalStateException("Allocated hosts does not exists")); } - public ZooKeeperDeployer createDeployer(DeployLogger logger) { - ZooKeeperClient zkClient = new ZooKeeperClient(curator, logger, sessionPath); - return new ZooKeeperDeployer(zkClient); - } - public Transaction createWriteStatusTransaction(Session.Status status) { CuratorTransaction transaction = new CuratorTransaction(curator); if (curator.exists(sessionStatusPath)) { @@ -368,7 +361,7 @@ public class SessionZooKeeperClient { transaction.commit(); } - private static Path getSessionPath(TenantName tenantName, long sessionId) { + static Path getSessionPath(TenantName tenantName, long sessionId) { return TenantRepository.getSessionsPath(tenantName).append(String.valueOf(sessionId)); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/DataplaneTokenSerializer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/DataplaneTokenSerializer.java index ef41512f979..3b819da6237 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/DataplaneTokenSerializer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/DataplaneTokenSerializer.java @@ -54,6 +54,11 @@ public class DataplaneTokenSerializer { public static Slime toSlime(List<DataplaneToken> dataplaneTokens) { Slime slime = new Slime(); Cursor root = slime.setArray(); + toSlime(dataplaneTokens, root); + return slime; + } + + public static void toSlime(List<DataplaneToken> dataplaneTokens, Cursor root) { for (DataplaneToken token : dataplaneTokens) { Cursor cursor = root.addObject(); cursor.setString(ID_FIELD, token.tokenId()); @@ -65,6 +70,6 @@ public class DataplaneTokenSerializer { val.setString(EXPIRATION_FIELD, v.expiration().map(Instant::toString).orElse("<none>")); }); } - return slime; } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/OperatorCertificateSerializer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/OperatorCertificateSerializer.java index 232dd2e5fe7..e5a969bb948 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/OperatorCertificateSerializer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/OperatorCertificateSerializer.java @@ -1,8 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - package com.yahoo.vespa.config.server.tenant; -import com.yahoo.config.model.api.ApplicationRoles; import com.yahoo.security.X509CertificateUtils; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; @@ -11,21 +9,28 @@ import com.yahoo.slime.SlimeUtils; import java.security.cert.X509Certificate; import java.util.List; -import java.util.stream.Collectors; +/** + * Serializer for operator certificates. + * The certificates are serialized as a list of PEM strings. + * @author tokle + */ public class OperatorCertificateSerializer { private final static String certificateField = "certificates"; - public static Slime toSlime(List<X509Certificate> certificateList) { Slime slime = new Slime(); var root = slime.setObject(); Cursor array = root.setArray(certificateField); + toSlime(certificateList, array); + return slime; + } + + public static void toSlime(List<X509Certificate> certificateList, Cursor array) { certificateList.stream() .map(X509CertificateUtils::toPem) .forEach(array::addString); - return slime; } public static List<X509Certificate> fromSlime(Inspector object) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantSecretStoreSerializer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantSecretStoreSerializer.java index 262192ad6c4..b8df5073a3e 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantSecretStoreSerializer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantSecretStoreSerializer.java @@ -30,10 +30,14 @@ public class TenantSecretStoreSerializer { public static Slime toSlime(List<TenantSecretStore> tenantSecretStores) { Slime slime = new Slime(); Cursor cursor = slime.setArray(); - tenantSecretStores.forEach(tenantSecretStore -> toSlime(tenantSecretStore, cursor.addObject())); + toSlime(tenantSecretStores, cursor); return slime; } + public static void toSlime(List<TenantSecretStore> tenantSecretStores, Cursor cursor) { + tenantSecretStores.forEach(tenantSecretStore -> toSlime(tenantSecretStore, cursor.addObject())); + } + public static void toSlime(TenantSecretStore tenantSecretStore, Cursor object) { object.setString(awsIdField, tenantSecretStore.getAwsId()); object.setString(nameField, tenantSecretStore.getName()); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileDirectoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileDirectoryTest.java index 649d382ddb6..040df208323 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileDirectoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileDirectoryTest.java @@ -37,9 +37,9 @@ public class FileDirectoryTest { FileReference foo = createFile("foo"); FileReference bar = createFile("bar"); - assertTrue(fileDirectory.getFile(foo).exists()); + assertTrue(fileDirectory.getFile(foo).get().exists()); assertEquals("ea315b7acac56246", foo.value()); - assertTrue(fileDirectory.getFile(bar).exists()); + assertTrue(fileDirectory.getFile(bar).get().exists()); assertEquals("2b8e97f15c854e1d", bar.value()); } @@ -49,7 +49,7 @@ public class FileDirectoryTest { File subDirectory = new File(temporaryFolder.getRoot(), subdirName); createFileInSubDir(subDirectory, "foo", "some content"); FileReference fileReference = fileDirectory.addFile(subDirectory); - File dir = fileDirectory.getFile(fileReference); + File dir = fileDirectory.getFile(fileReference).get(); assertTrue(dir.exists()); assertTrue(new File(dir, "foo").exists()); assertFalse(new File(dir, "doesnotexist").exists()); @@ -58,7 +58,7 @@ public class FileDirectoryTest { // Change contents of a file, file reference value should change createFileInSubDir(subDirectory, "foo", "new content"); FileReference fileReference2 = fileDirectory.addFile(subDirectory); - dir = fileDirectory.getFile(fileReference2); + dir = fileDirectory.getFile(fileReference2).get(); assertTrue(new File(dir, "foo").exists()); assertNotEquals(fileReference + " should not be equal to " + fileReference2, fileReference, fileReference2); assertEquals("e5d4b3fe5ee3ede3", fileReference2.value()); @@ -66,7 +66,7 @@ public class FileDirectoryTest { // Add a file, should be available and file reference should have another value createFileInSubDir(subDirectory, "bar", "some other content"); FileReference fileReference3 = fileDirectory.addFile(subDirectory); - dir = fileDirectory.getFile(fileReference3); + dir = fileDirectory.getFile(fileReference3).get(); assertTrue(new File(dir, "foo").exists()); assertTrue(new File(dir, "bar").exists()); assertEquals("894bced3fc9d199b", fileReference3.value()); @@ -78,7 +78,7 @@ public class FileDirectoryTest { File subDirectory = new File(temporaryFolder.getRoot(), subdirName); createFileInSubDir(subDirectory, "foo", "some content"); FileReference fileReference = fileDirectory.addFile(subDirectory); - File dir = fileDirectory.getFile(fileReference); + File dir = fileDirectory.getFile(fileReference).get(); assertTrue(dir.exists()); File foo = new File(dir, "foo"); assertTrue(foo.exists()); @@ -90,7 +90,7 @@ public class FileDirectoryTest { try { Thread.sleep(1000);} catch (InterruptedException e) {/*ignore */} // Needed since we have timestamp resolution of 1 second Files.delete(Paths.get(fileDirectory.getPath(fileReference)).resolve("subdir").resolve("foo")); fileReference = fileDirectory.addFile(subDirectory); - dir = fileDirectory.getFile(fileReference); + dir = fileDirectory.getFile(fileReference).get(); File foo2 = new File(dir, "foo"); assertTrue(dir.exists()); assertTrue(foo2.exists()); @@ -107,7 +107,7 @@ public class FileDirectoryTest { File subDirectory = new File(temporaryFolder.getRoot(), subdirName); createFileInSubDir(subDirectory, "foo", "some content"); FileReference fileReference = fileDirectory.addFile(subDirectory); - File dir = fileDirectory.getFile(fileReference); + File dir = fileDirectory.getFile(fileReference).get(); assertTrue(dir.exists()); File foo = new File(dir, "foo"); assertTrue(foo.exists()); @@ -119,7 +119,7 @@ public class FileDirectoryTest { // Add a file that already exists, nothing should happen createFileInSubDir(subDirectory, "foo", "some content"); // same as before, nothing should happen FileReference fileReference3 = fileDirectory.addFile(subDirectory); - dir = fileDirectory.getFile(fileReference3); + dir = fileDirectory.getFile(fileReference3).get(); assertTrue(new File(dir, "foo").exists()); assertEquals("bebc5a1aee74223d", fileReference3.value()); // same hash diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileServerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileServerTest.java index c17b68c6d12..373b39c8365 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileServerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileServerTest.java @@ -61,9 +61,9 @@ public class FileServerTest { String dir = "123"; assertFalse(fileServer.hasFile(dir)); FileReferenceDownload foo = new FileReferenceDownload(new FileReference(dir), "test"); - assertFalse(fileServer.hasFileDownloadIfNeeded(foo)); + assertFalse(fileServer.getFileDownloadIfNeeded(foo).isPresent()); writeFile(dir); - assertTrue(fileServer.hasFileDownloadIfNeeded(foo)); + assertTrue(fileServer.getFileDownloadIfNeeded(foo).isPresent()); } @Test @@ -79,7 +79,9 @@ public class FileServerTest { File dir = getFileServerRootDir(); IOUtils.writeFile(dir + "/12y/f1", "dummy-data", true); CompletableFuture<byte []> content = new CompletableFuture<>(); - fileServer.startFileServing(new FileReference("12y"), new FileReceiver(content), Set.of(gzip)); + FileReference fileReference = new FileReference("12y"); + var file = fileServer.getFileDownloadIfNeeded(new FileReferenceDownload(fileReference, "test")); + fileServer.startFileServing(fileReference, file.get(), new FileReceiver(content), Set.of(gzip)); assertEquals(new String(content.get()), "dummy-data"); } @@ -90,7 +92,9 @@ public class FileServerTest { File dir = getFileServerRootDir(); IOUtils.writeFile(dir + "/subdir/12z/f1", "dummy-data-2", true); CompletableFuture<byte []> content = new CompletableFuture<>(); - fileServer.startFileServing(new FileReference("subdir"), new FileReceiver(content), Set.of(gzip, lz4)); + FileReference fileReference = new FileReference("subdir"); + var file = fileServer.getFileDownloadIfNeeded(new FileReferenceDownload(fileReference, "test")); + fileServer.startFileServing(fileReference, file.get(), new FileReceiver(content), Set.of(gzip, lz4)); // Decompress with lz4 and check contents var compressor = new FileReferenceCompressor(FileReferenceData.Type.compressed, lz4); @@ -139,14 +143,16 @@ public class FileServerTest { FailingFileReceiver fileReceiver = new FailingFileReceiver(content); // Should fail the first time, see FailingFileReceiver + FileReference reference = new FileReference("12y"); + var file = fileServer.getFileDownloadIfNeeded(new FileReferenceDownload(reference, "test")); try { - fileServer.startFileServing(new FileReference("12y"), fileReceiver, Set.of(gzip)); + fileServer.startFileServing(reference, file.get(), fileReceiver, Set.of(gzip)); fail("Should have failed"); } catch (RuntimeException e) { // expected } - fileServer.startFileServing(new FileReference("12y"), fileReceiver, Set.of(gzip)); + fileServer.startFileServing(reference, file.get(), fileReceiver, Set.of(gzip)); assertEquals(new String(content.get()), "dummy-data"); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java index 52d5ba16562..0158aa1961d 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java @@ -67,8 +67,8 @@ import java.util.OptionalInt; import java.util.Set; import java.util.logging.Level; +import static com.yahoo.vespa.config.server.session.SessionData.APPLICATION_PACKAGE_REFERENCE_PATH; import static com.yahoo.vespa.config.server.session.SessionPreparer.PrepareResult; -import static com.yahoo.vespa.config.server.session.SessionZooKeeperClient.APPLICATION_PACKAGE_REFERENCE_PATH; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClientTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClientTest.java index 4a7aeafab7e..569b6624815 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClientTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClientTest.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.config.server.session; import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.component.Version; import com.yahoo.config.FileReference; import com.yahoo.config.model.api.Quota; import com.yahoo.config.model.api.TenantSecretStore; @@ -16,10 +17,13 @@ import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; + import java.time.Instant; import java.util.List; import java.util.Optional; +import static com.yahoo.vespa.config.server.session.SessionData.APPLICATION_ID_PATH; +import static com.yahoo.vespa.config.server.session.SessionData.SESSION_DATA_PATH; import static com.yahoo.vespa.config.server.zookeeper.ZKApplication.SESSIONSTATE_ZK_SUBPATH; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -87,7 +91,7 @@ public class SessionZooKeeperClientTest { int sessionId = 3; SessionZooKeeperClient zkc = createSessionZKClient(sessionId); zkc.writeApplicationId(id); - Path path = sessionPath(sessionId).append(SessionZooKeeperClient.APPLICATION_ID_PATH); + Path path = sessionPath(sessionId).append(APPLICATION_ID_PATH); assertTrue(curator.exists(path)); assertEquals(id.serializedForm(), Utf8.toString(curator.getData(path).get())); } @@ -135,7 +139,7 @@ public class SessionZooKeeperClientTest { final FileReference testRef = new FileReference("test-ref"); SessionZooKeeperClient zkc = createSessionZKClient(3); zkc.writeApplicationPackageReference(Optional.of(testRef)); - assertEquals(testRef, zkc.readApplicationPackageReference()); + assertEquals(testRef, zkc.readApplicationPackageReference().get()); } @Test @@ -157,9 +161,30 @@ public class SessionZooKeeperClientTest { assertEquals(secretStores, zkc.readTenantSecretStores()); } + @Test + public void require_that_session_data_is_written_to_zk() { + int sessionId = 2; + SessionZooKeeperClient zkc = createSessionZKClient(sessionId); + zkc.writeSessionData(new SessionData(ApplicationId.defaultId(), + Optional.of(new FileReference("foo")), + Version.fromString("8.195.1"), + Optional.empty(), + Optional.empty(), + Optional.empty(), + List.of(), + List.of(), + Optional.empty(), + List.of())); + Path path = sessionPath(sessionId).append(SESSION_DATA_PATH); + assertTrue(curator.exists(path)); + String data = Utf8.toString(curator.getData(path).get()); + assertTrue(data.contains("{\"applicationId\":\"default:default:default\",\"applicationPackageReference\":\"foo\",\"version\":\"8.195.1\",\"createTime\":")); + assertTrue(data.contains(",\"tenantSecretStores\":[],\"operatorCertificates\":[],\"dataplaneTokens\":[]}")); + } + private void assertApplicationIdParse(long sessionId, String idString, String expectedIdString) { SessionZooKeeperClient zkc = createSessionZKClient(sessionId); - Path path = sessionPath(sessionId).append(SessionZooKeeperClient.APPLICATION_ID_PATH); + Path path = sessionPath(sessionId).append(APPLICATION_ID_PATH); curator.set(path, Utf8.toBytes(idString)); assertEquals(expectedIdString, zkc.readApplicationId().serializedForm()); } diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json index 0f440957dfd..cdb660f294a 100644 --- a/container-search/abi-spec.json +++ b/container-search/abi-spec.json @@ -6981,12 +6981,14 @@ "public java.lang.Integer getMinHitsPerThread()", "public java.lang.Double getPostFilterThreshold()", "public java.lang.Double getApproximateThreshold()", + "public java.lang.Double getTargetHitsMaxAdjustmentFactor()", "public void setTermwiselimit(double)", "public void setNumThreadsPerSearch(int)", "public void setNumSearchPartitions(int)", "public void setMinHitsPerThread(int)", "public void setPostFilterThreshold(double)", "public void setApproximateThreshold(double)", + "public void setTargetHitsMaxAdjustmentFactor(double)", "public void prepare(com.yahoo.search.query.ranking.RankProperties)", "public com.yahoo.search.query.ranking.Matching clone()", "public boolean equals(java.lang.Object)", @@ -7000,6 +7002,7 @@ "public static final java.lang.String MINHITSPERTHREAD", "public static final java.lang.String POST_FILTER_THRESHOLD", "public static final java.lang.String APPROXIMATE_THRESHOLD", + "public static final java.lang.String TARGET_HITS_MAX_ADJUSTMENT_FACTOR", "public java.lang.Double termwiseLimit" ] }, diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/ReconfigurableDispatcher.java b/container-search/src/main/java/com/yahoo/search/dispatch/ReconfigurableDispatcher.java index 625a8bcb6da..c86c21d677f 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/ReconfigurableDispatcher.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/ReconfigurableDispatcher.java @@ -1,20 +1,17 @@ package com.yahoo.search.dispatch; import com.yahoo.component.ComponentId; +import com.yahoo.component.annotation.Inject; import com.yahoo.config.subscription.ConfigSubscriber; +import com.yahoo.container.QrConfig; import com.yahoo.container.handler.VipStatus; -import com.yahoo.messagebus.network.rpc.SlobrokConfigSubscriber; import com.yahoo.vespa.config.search.DispatchConfig; import com.yahoo.vespa.config.search.DispatchNodesConfig; import com.yahoo.yolean.UncheckedInterruptedException; -import java.util.Objects; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; -import static java.util.Objects.requireNonNull; - /** * @author jonmv */ @@ -22,10 +19,20 @@ public class ReconfigurableDispatcher extends Dispatcher { private final ConfigSubscriber subscriber; - public ReconfigurableDispatcher(ComponentId clusterId, DispatchConfig dispatchConfig, VipStatus vipStatus) { + @Inject + public ReconfigurableDispatcher(ComponentId clusterId, DispatchConfig dispatchConfig, QrConfig qrConfig, VipStatus vipStatus) { super(clusterId, dispatchConfig, new DispatchNodesConfig.Builder().build(), vipStatus); this.subscriber = new ConfigSubscriber(); - this.subscriber.subscribe(this::updateWithNewConfig, DispatchNodesConfig.class, clusterId.stringValue()); + CountDownLatch configured = new CountDownLatch(1); + this.subscriber.subscribe(config -> { updateWithNewConfig(config); configured.countDown(); }, + DispatchNodesConfig.class, configId(clusterId, qrConfig)); + try { + if ( ! configured.await(1, TimeUnit.MINUTES)) + throw new IllegalStateException("timed out waiting for initial dispatch nodes config for " + clusterId.getName()); + } + catch (InterruptedException e) { + throw new UncheckedInterruptedException("interrupted waiting for initial dispatch nodes config for " + clusterId.getName(), e); + } } @Override @@ -34,4 +41,8 @@ public class ReconfigurableDispatcher extends Dispatcher { super.deconstruct(); } + private static String configId(ComponentId clusterId, QrConfig qrConfig) { + return qrConfig.clustername() + "/component/" + clusterId.getName(); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java index 800b3a1ba89..99d6959441a 100644 --- a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java +++ b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java @@ -91,6 +91,7 @@ public class QueryProperties extends Properties { addDualCasedRM(map, Matching.MINHITSPERTHREAD, GetterSetter.of(query -> query.getRanking().getMatching().getMinHitsPerThread(), (query, value) -> query.getRanking().getMatching().setMinHitsPerThread(asInteger(value, 0)))); addDualCasedRM(map, Matching.POST_FILTER_THRESHOLD, GetterSetter.of(query -> query.getRanking().getMatching().getPostFilterThreshold(), (query, value) -> query.getRanking().getMatching().setPostFilterThreshold(asDouble(value, 1.0)))); addDualCasedRM(map, Matching.APPROXIMATE_THRESHOLD, GetterSetter.of(query -> query.getRanking().getMatching().getApproximateThreshold(), (query, value) -> query.getRanking().getMatching().setApproximateThreshold(asDouble(value, 0.05)))); + addDualCasedRM(map, Matching.TARGET_HITS_MAX_ADJUSTMENT_FACTOR, GetterSetter.of(query -> query.getRanking().getMatching().getTargetHitsMaxAdjustmentFactor(), (query, value) -> query.getRanking().getMatching().setTargetHitsMaxAdjustmentFactor(asDouble(value, 20.0)))); map.put(CompoundName.fromComponents(Ranking.RANKING, Ranking.MATCH_PHASE, MatchPhase.ATTRIBUTE), GetterSetter.of(query -> query.getRanking().getMatchPhase().getAttribute(), (query, value) -> query.getRanking().getMatchPhase().setAttribute(asString(value, null)))); map.put(CompoundName.fromComponents(Ranking.RANKING, Ranking.MATCH_PHASE, MatchPhase.ASCENDING), GetterSetter.of(query -> query.getRanking().getMatchPhase().getAscending(), (query, value) -> query.getRanking().getMatchPhase().setAscending(asBoolean(value, false)))); diff --git a/container-search/src/main/java/com/yahoo/search/query/ranking/Matching.java b/container-search/src/main/java/com/yahoo/search/query/ranking/Matching.java index 35fbd52f967..4d21f32d16d 100644 --- a/container-search/src/main/java/com/yahoo/search/query/ranking/Matching.java +++ b/container-search/src/main/java/com/yahoo/search/query/ranking/Matching.java @@ -24,6 +24,7 @@ public class Matching implements Cloneable { public static final String MINHITSPERTHREAD = "minHitsPerThread"; public static final String POST_FILTER_THRESHOLD = "postFilterThreshold"; public static final String APPROXIMATE_THRESHOLD = "approximateThreshold"; + public static final String TARGET_HITS_MAX_ADJUSTMENT_FACTOR = "targetHitsMaxAdjustmentFactor"; static { argumentType =new QueryProfileType(Ranking.MATCHING); @@ -35,6 +36,7 @@ public class Matching implements Cloneable { argumentType.addField(new FieldDescription(MINHITSPERTHREAD, "integer")); argumentType.addField(new FieldDescription(POST_FILTER_THRESHOLD, "double")); argumentType.addField(new FieldDescription(APPROXIMATE_THRESHOLD, "double")); + argumentType.addField(new FieldDescription(TARGET_HITS_MAX_ADJUSTMENT_FACTOR, "double")); argumentType.freeze(); } @@ -46,6 +48,7 @@ public class Matching implements Cloneable { private Integer minHitsPerThread = null; private Double postFilterThreshold = null; private Double approximateThreshold = null; + private Double targetHitsMaxAdjustmentFactor = null; public Double getTermwiseLimit() { return termwiseLimit; } public Integer getNumThreadsPerSearch() { return numThreadsPerSearch; } @@ -53,6 +56,7 @@ public class Matching implements Cloneable { public Integer getMinHitsPerThread() { return minHitsPerThread; } public Double getPostFilterThreshold() { return postFilterThreshold; } public Double getApproximateThreshold() { return approximateThreshold; } + public Double getTargetHitsMaxAdjustmentFactor() { return targetHitsMaxAdjustmentFactor; } public void setTermwiselimit(double value) { if ((value < 0.0) || (value > 1.0)) { @@ -75,6 +79,9 @@ public class Matching implements Cloneable { public void setApproximateThreshold(double threshold) { approximateThreshold = threshold; } + public void setTargetHitsMaxAdjustmentFactor(double factor) { + targetHitsMaxAdjustmentFactor = factor; + } /** Internal operation - DO NOT USE */ public void prepare(RankProperties rankProperties) { @@ -97,6 +104,9 @@ public class Matching implements Cloneable { if (approximateThreshold != null) { rankProperties.put("vespa.matching.global_filter.lower_limit", String.valueOf(approximateThreshold)); } + if (targetHitsMaxAdjustmentFactor != null) { + rankProperties.put("vespa.matching.nns.target_hits_max_adjustment_factor", String.valueOf(targetHitsMaxAdjustmentFactor)); + } } @Override @@ -119,12 +129,14 @@ public class Matching implements Cloneable { Objects.equals(numSearchPartitions, matching.numSearchPartitions) && Objects.equals(minHitsPerThread, matching.minHitsPerThread) && Objects.equals(postFilterThreshold, matching.postFilterThreshold) && - Objects.equals(approximateThreshold, matching.approximateThreshold); + Objects.equals(approximateThreshold, matching.approximateThreshold) && + Objects.equals(targetHitsMaxAdjustmentFactor, matching.targetHitsMaxAdjustmentFactor); } @Override public int hashCode() { - return Objects.hash(termwiseLimit, numThreadsPerSearch, numSearchPartitions, minHitsPerThread, postFilterThreshold, approximateThreshold); + return Objects.hash(termwiseLimit, numThreadsPerSearch, numSearchPartitions, minHitsPerThread, + postFilterThreshold, approximateThreshold, targetHitsMaxAdjustmentFactor); } } diff --git a/container-search/src/test/java/com/yahoo/search/query/MatchingTestCase.java b/container-search/src/test/java/com/yahoo/search/query/MatchingTestCase.java index e3a1eb18a33..37d0e9e1072 100644 --- a/container-search/src/test/java/com/yahoo/search/query/MatchingTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/MatchingTestCase.java @@ -20,6 +20,7 @@ public class MatchingTestCase { assertNull(query.getRanking().getMatching().getMinHitsPerThread()); assertNull(query.getRanking().getMatching().getPostFilterThreshold()); assertNull(query.getRanking().getMatching().getApproximateThreshold()); + assertNull(query.getRanking().getMatching().getTargetHitsMaxAdjustmentFactor()); } @Test @@ -30,13 +31,15 @@ public class MatchingTestCase { "&ranking.matching.numSearchPartitions=13" + "&ranking.matching.minHitsPerThread=3" + "&ranking.matching.postFilterThreshold=0.8" + - "&ranking.matching.approximateThreshold=0.3"); + "&ranking.matching.approximateThreshold=0.3" + + "&ranking.matching.targetHitsMaxAdjustmentFactor=2.5"); assertEquals(Double.valueOf(0.7), query.getRanking().getMatching().getTermwiseLimit()); assertEquals(Integer.valueOf(17), query.getRanking().getMatching().getNumThreadsPerSearch()); assertEquals(Integer.valueOf(13), query.getRanking().getMatching().getNumSearchPartitions()); assertEquals(Integer.valueOf(3), query.getRanking().getMatching().getMinHitsPerThread()); assertEquals(Double.valueOf(0.8), query.getRanking().getMatching().getPostFilterThreshold()); assertEquals(Double.valueOf(0.3), query.getRanking().getMatching().getApproximateThreshold()); + assertEquals(Double.valueOf(2.5), query.getRanking().getMatching().getTargetHitsMaxAdjustmentFactor()); query.prepare(); assertEquals("0.7", query.getRanking().getProperties().get("vespa.matching.termwise_limit").get(0)); @@ -45,6 +48,7 @@ public class MatchingTestCase { assertEquals("3", query.getRanking().getProperties().get("vespa.matching.minhitsperthread").get(0)); assertEquals("0.8", query.getRanking().getProperties().get("vespa.matching.global_filter.upper_limit").get(0)); assertEquals("0.3", query.getRanking().getProperties().get("vespa.matching.global_filter.lower_limit").get(0)); + assertEquals("2.5", query.getRanking().getProperties().get("vespa.matching.nns.target_hits_max_adjustment_factor").get(0)); } @Test diff --git a/dependency-versions/pom.xml b/dependency-versions/pom.xml index 5f75b042722..074cb2190da 100644 --- a/dependency-versions/pom.xml +++ b/dependency-versions/pom.xml @@ -110,7 +110,7 @@ <org.json.vespa.version>20230227</org.json.vespa.version> <org.lz4.vespa.version>1.8.0</org.lz4.vespa.version> <prometheus.client.vespa.version>0.6.0</prometheus.client.vespa.version> - <protobuf.vespa.version>3.21.7</protobuf.vespa.version> + <protobuf.vespa.version>3.24.0</protobuf.vespa.version> <spifly.vespa.version>1.3.6</spifly.vespa.version> <surefire.vespa.version>3.0.0-M9</surefire.vespa.version> <wiremock.vespa.version>2.35.0</wiremock.vespa.version> diff --git a/document/src/test/java/com/yahoo/document/DocumentTestCase.java b/document/src/test/java/com/yahoo/document/DocumentTestCase.java index 33b77cb1878..e5f6453c581 100644 --- a/document/src/test/java/com/yahoo/document/DocumentTestCase.java +++ b/document/src/test/java/com/yahoo/document/DocumentTestCase.java @@ -42,7 +42,7 @@ import static org.junit.Assert.fail; /** * Test for Document and all its features, including (de)serialization. * - * @author <a href="thomasg@yahoo-inc.com>Thomas Gundersen</a> + * @author Thomas Gundersen * @author bratseth */ public class DocumentTestCase extends DocumentTestCaseBase { diff --git a/document/src/tests/serialization/vespadocumentserializer_test.cpp b/document/src/tests/serialization/vespadocumentserializer_test.cpp index 1839005d720..03878f43e4b 100644 --- a/document/src/tests/serialization/vespadocumentserializer_test.cpp +++ b/document/src/tests/serialization/vespadocumentserializer_test.cpp @@ -686,7 +686,7 @@ void deserializeAndCheck(const string &file_name, FieldValueT &value, const string &field_name) { File file(file_name); file.open(File::READONLY); - vector<char> content(file.stat()._size); + vector<char> content(file.getFileSize()); size_t r = file.read(&content[0], content.size(), 0); ASSERT_EQUAL(content.size(), r); diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 4e995e9b392..436104c590c 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -325,11 +325,6 @@ public class Flags { "Takes effect on next run of CertPoolMaintainer" ); - public static final UnboundStringFlag CONTAINER_IMAGE_PULL_IO_MAX = defineStringFlag( - "container-image-pull-io-max", "", List.of("freva"), "2023-08-04", "2023-09-15", - "The value (excluding the device name) of io.max cgroup used by container image pull, e.g. 'wiops=100', or 'wbps=10000 riops=20', or empty for unlimited", - "Takes effect at next host-admin tick"); - public static final UnboundBooleanFlag ENABLE_THE_ONE_THAT_SHOULD_NOT_BE_NAMED = defineFeatureFlag( "enable-the-one-that-should-not-be-named", false, List.of("hmusum"), "2023-05-08", "2023-09-15", "Whether to enable the one program that should not be named", diff --git a/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java b/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java index 5a791522977..b50e71154eb 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java @@ -352,14 +352,6 @@ public class PermanentFlags { "Takes effect immediately", TENANT_ID); - // TODO: Remove when not in use anymore, replaced by KEEP_FILE_REFERENCES_DAYS - public static final UnboundIntFlag KEEP_FILE_REFERENCES_ON_TENANT_NODES = defineIntFlag( - "keep-file-references-on-tenant-nodes", 30, - "How many days to keep file references on tenant nodes (based on last modification time)", - "Takes effect on restart of Docker container", - APPLICATION_ID - ); - public static final UnboundIntFlag KEEP_FILE_REFERENCES_DAYS = defineIntFlag( "keep-file-references-days", 30, "How many days to keep file references on tenant nodes (based on last modification time)", @@ -393,7 +385,7 @@ public class PermanentFlags { "Takes effect immediately"); public static final UnboundBooleanFlag DROP_CACHES = defineFeatureFlag( - "drop-caches", false, + "drop-caches", true, "Drop caches on tenant hosts", "Takes effect on next tick", // The application ID is the exclusive application ID associated with the host, diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToEpochSecondExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToEpochSecondExpression.java new file mode 100644 index 00000000000..c8106148630 --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToEpochSecondExpression.java @@ -0,0 +1,51 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.datatypes.LongFieldValue; +import java.time.Instant; + +/** + * Converts ISO-8601 formatted date string to UNIX Epoch Time in seconds + * + * @author bergum + */ + +public class ToEpochSecondExpression extends Expression { + public ToEpochSecondExpression() { + super(DataType.STRING); //only accept string input + } + + @Override + protected void doExecute(ExecutionContext context) { + String inputString = String.valueOf(context.getValue()); + long epochTime = Instant.parse(inputString).getEpochSecond(); + context.setValue(new LongFieldValue(epochTime)); + } + + @Override + protected void doVerify(VerificationContext context) { + context.setValueType(createdOutputType()); + } + + @Override + public DataType createdOutputType() { + return DataType.LONG; + } + + @Override + public String toString() { + return "to_epoch_second"; + } + + @Override + public boolean equals(Object obj) { + return obj instanceof ToEpochSecondExpression; + } + + @Override + public int hashCode() { + return getClass().hashCode(); + } + +} diff --git a/indexinglanguage/src/main/javacc/IndexingParser.jj b/indexinglanguage/src/main/javacc/IndexingParser.jj index a039ad137ee..d559d9b7260 100644 --- a/indexinglanguage/src/main/javacc/IndexingParser.jj +++ b/indexinglanguage/src/main/javacc/IndexingParser.jj @@ -198,6 +198,7 @@ TOKEN : <TO_INT: "to_int"> | <TO_LONG: "to_long"> | <TO_POS: "to_pos"> | + <TO_EPOCH_SECOND: "to_epoch_second"> | <TO_STRING: "to_string"> | <TO_WSET: "to_wset"> | <TO_BOOL: "to_bool"> | @@ -338,6 +339,7 @@ Expression value() : val = toIntExp() | val = toLongExp() | val = toPosExp() | + val = toEpochSecondExp() | val = toStringExp() | val = toWsetExp() | val = toBoolExp() | @@ -713,6 +715,12 @@ Expression toPosExp() : { } { return new ToPositionExpression(); } } +Expression toEpochSecondExp() : { } +{ + ( <TO_EPOCH_SECOND> ) + { return new ToEpochSecondExpression(); } +} + Expression toStringExp() : { } { ( <TO_STRING> ) diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToEpochSecondExpressionTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToEpochSecondExpressionTestCase.java new file mode 100644 index 00000000000..7203afcc1a0 --- /dev/null +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToEpochSecondExpressionTestCase.java @@ -0,0 +1,51 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.LongFieldValue; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter; +import org.junit.Test; + +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerify; +import static com.yahoo.vespa.indexinglanguage.expressions.ExpressionAssert.assertVerifyThrows; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; + +public class ToEpochSecondExpressionTestCase { + @Test + public void requireThatHashCodeAndEqualsAreImplemented() { + Expression exp = new ToEpochSecondExpression(); + assertFalse(exp.equals(new Object())); + assertEquals(exp, new ToEpochSecondExpression()); + assertEquals(exp.hashCode(), new ToEpochSecondExpression().hashCode()); + } + + @Test + public void requireThatExpressionCanBeVerified() { + Expression exp = new ToEpochSecondExpression(); + assertVerify(DataType.STRING, exp, DataType.LONG); + assertVerifyThrows(DataType.INT, exp, "Expected string input, got int."); + assertVerifyThrows(null, exp, "Expected string input, got null."); + } + + @Test + public void requireThatValueIsConvertedWithMs() { + ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); + ctx.setValue(new StringFieldValue("2023-12-24T17:00:43.000Z")).execute(new ToEpochSecondExpression()); + FieldValue val = ctx.getValue(); + assertTrue(val instanceof LongFieldValue); + assertEquals(1703437243L, ((LongFieldValue)val).getLong()); + } + + @Test + public void requireThatValueIsConverted() { + ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); + ctx.setValue(new StringFieldValue("2023-12-24T17:00:43Z")).execute(new ToEpochSecondExpression()); + FieldValue val = ctx.getValue(); + assertTrue(val instanceof LongFieldValue); + assertEquals(1703437243L, ((LongFieldValue)val).getLong()); + } +} diff --git a/metrics/src/main/java/ai/vespa/metrics/set/DefaultMetrics.java b/metrics/src/main/java/ai/vespa/metrics/set/DefaultMetrics.java index cb3a13acb85..515b06de2d8 100644 --- a/metrics/src/main/java/ai/vespa/metrics/set/DefaultMetrics.java +++ b/metrics/src/main/java/ai/vespa/metrics/set/DefaultMetrics.java @@ -56,6 +56,7 @@ public class DefaultMetrics { addStorageMetrics(metrics); addDistributorMetrics(metrics); addClusterControllerMetrics(metrics); + addSentinelMetrics(metrics); addOtherMetrics(metrics); return Collections.unmodifiableSet(metrics); } @@ -154,7 +155,7 @@ public class DefaultMetrics { private static void addSentinelMetrics(Set<Metric> metrics) { // Metrics needed for alerting - addMetric(metrics, SentinelMetrics.SENTINEL_TOTAL_RESTARTS, EnumSet.of(sum, last)); // TODO: Vespa 9: Remove last + addMetric(metrics, SentinelMetrics.SENTINEL_TOTAL_RESTARTS, EnumSet.of(max, sum, last)); // TODO: Vespa 9: Remove last, sum? } private static void addOtherMetrics(Set<Metric> metrics) { diff --git a/metrics/src/main/java/ai/vespa/metrics/set/VespaMetricSet.java b/metrics/src/main/java/ai/vespa/metrics/set/VespaMetricSet.java index f9e37f4a85b..bc8567b8bf5 100644 --- a/metrics/src/main/java/ai/vespa/metrics/set/VespaMetricSet.java +++ b/metrics/src/main/java/ai/vespa/metrics/set/VespaMetricSet.java @@ -62,7 +62,7 @@ public class VespaMetricSet { Set<Metric> metrics = new LinkedHashSet<>(); addMetric(metrics, SentinelMetrics.SENTINEL_RESTARTS.count()); - addMetric(metrics, SentinelMetrics.SENTINEL_TOTAL_RESTARTS, EnumSet.of(sum, last)); // TODO: Vespa 9: Remove last + addMetric(metrics, SentinelMetrics.SENTINEL_TOTAL_RESTARTS, EnumSet.of(max, sum, last)); // TODO: Vespa 9: Remove last, sum? addMetric(metrics, SentinelMetrics.SENTINEL_UPTIME.last()); addMetric(metrics, SentinelMetrics.SENTINEL_RUNNING, EnumSet.of(count, last)); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java index 284306e1e8c..466ee65fcc1 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java @@ -486,7 +486,8 @@ public class NodeAgentImpl implements NodeAgent { // Run this here and now, even though we may immediately remove the container below. // This ensures these maintainers are run even if something fails or returns early. // These maintainers should also run immediately after starting the container (see below). - container.ifPresent(c -> runImportantContainerMaintainers(context, c)); + container.filter(c -> c.state().isRunning()) + .ifPresent(c -> runImportantContainerMaintainers(context, c)); switch (node.state()) { case ready, reserved, failed, inactive, parked -> { @@ -561,9 +562,9 @@ public class NodeAgentImpl implements NodeAgent { } } - private void runImportantContainerMaintainers(NodeAgentContext context, Container container) { + private void runImportantContainerMaintainers(NodeAgentContext context, Container runningContainer) { aclMaintainer.ifPresent(maintainer -> maintainer.converge(context)); - wireguardTasks.forEach(task -> task.converge(context, container.id())); + wireguardTasks.forEach(task -> task.converge(context, runningContainer.id())); } private static void logChangesToNodeSpec(NodeAgentContext context, NodeSpec lastNode, NodeSpec node) { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixUser.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixUser.java index 665bb4b8bbc..78fc4b151c7 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixUser.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixUser.java @@ -11,6 +11,7 @@ import java.util.Objects; public class UnixUser { public static final UnixUser ROOT = new UnixUser("root", 0, "root", 0); + public static final UnixUser VESPA = new UnixUser("vespa", 1000, "vespa", 1000); private final String name; private final int uid; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java index 602314bed96..eafaed2a217 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java @@ -95,8 +95,7 @@ public class NodeRepository extends AbstractComponent { metricsDb, orchestrator, config.useCuratorClientCache(), - zone.environment().isProduction() && !zone.cloud().dynamicProvisioning() && !zone.system().isCd() ? 1 : 0, - config.nodeCacheSize()); + zone.environment().isProduction() && !zone.cloud().dynamicProvisioning() && !zone.system().isCd() ? 1 : 0); } /** @@ -116,15 +115,14 @@ public class NodeRepository extends AbstractComponent { MetricsDb metricsDb, Orchestrator orchestrator, boolean useCuratorClientCache, - int spareCount, - long nodeCacheSize) { + int spareCount) { if (provisionServiceProvider.getHostProvisioner().isPresent() != zone.cloud().dynamicProvisioning()) throw new IllegalArgumentException(String.format( "dynamicProvisioning property must be 1-to-1 with availability of HostProvisioner, was: dynamicProvisioning=%s, hostProvisioner=%s", zone.cloud().dynamicProvisioning(), provisionServiceProvider.getHostProvisioner().map(__ -> "present").orElse("empty"))); this.flagSource = flagSource; - this.db = new CuratorDb(flavors, curator, clock, useCuratorClientCache, nodeCacheSize); + this.db = new CuratorDb(flavors, curator, clock, useCuratorClientCache); this.zone = zone; this.clock = clock; this.applications = new Applications(db); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java index 1ca81df824b..796bc2eeb92 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java @@ -208,6 +208,16 @@ public class Cluster { return minimum(ClusterModel.minScalingDuration(clusterSpec), totalDuration.dividedBy(completedEventCount)); } + /** The predicted time this cluster will stay in each resource configuration (including the scaling duration). */ + public Duration allocationDuration(ClusterSpec clusterSpec) { + if (scalingEvents.size() < 2) return Duration.ofHours(12); // Default + + long totalDurationMs = 0; + for (int i = 1; i < scalingEvents().size(); i++) + totalDurationMs += scalingEvents().get(i).at().toEpochMilli() - scalingEvents().get(i - 1).at().toEpochMilli(); + return Duration.ofMillis(totalDurationMs / (scalingEvents.size() - 1)); + } + private static Duration minimum(Duration smallestAllowed, Duration duration) { if (duration.minus(smallestAllowed).isNegative()) return smallestAllowed; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableClusterResources.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableResources.java index c19d76efb35..8069c9c089b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableClusterResources.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableResources.java @@ -10,13 +10,14 @@ import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeRepository; +import java.time.Duration; import java.util.List; import java.util.Optional; /** * @author bratseth */ -public class AllocatableClusterResources { +public class AllocatableResources { /** The node count in the cluster */ private final int nodes; @@ -32,9 +33,9 @@ public class AllocatableClusterResources { private final double fulfilment; /** Fake allocatable resources from requested capacity */ - public AllocatableClusterResources(ClusterResources requested, - ClusterSpec clusterSpec, - NodeRepository nodeRepository) { + public AllocatableResources(ClusterResources requested, + ClusterSpec clusterSpec, + NodeRepository nodeRepository) { this.nodes = requested.nodes(); this.groups = requested.groups(); this.realResources = nodeRepository.resourcesCalculator().requestToReal(requested.nodeResources(), nodeRepository.exclusiveAllocation(clusterSpec), false); @@ -43,7 +44,7 @@ public class AllocatableClusterResources { this.fulfilment = 1; } - public AllocatableClusterResources(NodeList nodes, NodeRepository nodeRepository) { + public AllocatableResources(NodeList nodes, NodeRepository nodeRepository) { this.nodes = nodes.size(); this.groups = (int)nodes.stream().map(node -> node.allocation().get().membership().cluster().group()).distinct().count(); this.realResources = averageRealResourcesOf(nodes.asList(), nodeRepository); // Average since we average metrics over nodes @@ -52,10 +53,10 @@ public class AllocatableClusterResources { this.fulfilment = 1; } - public AllocatableClusterResources(ClusterResources realResources, - NodeResources advertisedResources, - ClusterResources idealResources, - ClusterSpec clusterSpec) { + public AllocatableResources(ClusterResources realResources, + NodeResources advertisedResources, + ClusterResources idealResources, + ClusterSpec clusterSpec) { this.nodes = realResources.nodes(); this.groups = realResources.groups(); this.realResources = realResources.nodeResources(); @@ -64,12 +65,12 @@ public class AllocatableClusterResources { this.fulfilment = fulfilment(realResources, idealResources); } - private AllocatableClusterResources(int nodes, - int groups, - NodeResources realResources, - NodeResources advertisedResources, - ClusterSpec clusterSpec, - double fulfilment) { + private AllocatableResources(int nodes, + int groups, + NodeResources realResources, + NodeResources advertisedResources, + ClusterSpec clusterSpec, + double fulfilment) { this.nodes = nodes; this.groups = groups; this.realResources = realResources; @@ -79,16 +80,16 @@ public class AllocatableClusterResources { } /** Returns this with the redundant node or group removed from counts. */ - public AllocatableClusterResources withoutRedundancy() { + public AllocatableResources withoutRedundancy() { int groupSize = nodes / groups; int nodesAdjustedForRedundancy = nodes > 1 ? (groups == 1 ? nodes - 1 : nodes - groupSize) : nodes; int groupsAdjustedForRedundancy = nodes > 1 ? (groups == 1 ? 1 : groups - 1) : groups; - return new AllocatableClusterResources(nodesAdjustedForRedundancy, - groupsAdjustedForRedundancy, - realResources, - advertisedResources, - clusterSpec, - fulfilment); + return new AllocatableResources(nodesAdjustedForRedundancy, + groupsAdjustedForRedundancy, + realResources, + advertisedResources, + clusterSpec, + fulfilment); } /** @@ -112,6 +113,7 @@ public class AllocatableClusterResources { public ClusterSpec clusterSpec() { return clusterSpec; } + /** Returns the standard cost of these resources, in dollars per hour */ public double cost() { return nodes * advertisedResources.cost(); } /** @@ -128,11 +130,22 @@ public class AllocatableClusterResources { return (vcpuFulfilment + memoryGbFulfilment + diskGbFulfilment) / 3; } - public boolean preferableTo(AllocatableClusterResources other) { - if (this.fulfilment < 1 || other.fulfilment < 1) // always fulfil as much as possible - return this.fulfilment > other.fulfilment; + public boolean preferableTo(AllocatableResources other, ClusterModel model) { + if (other.fulfilment() < 1 || this.fulfilment() < 1) // always fulfil as much as possible + return this.fulfilment() > other.fulfilment(); - return this.cost() < other.cost(); // otherwise, prefer lower cost + return this.cost() * toHours(model.allocationDuration()) + this.costChangingFrom(model) + < + other.cost() * toHours(model.allocationDuration()) + other.costChangingFrom(model); + } + + private double toHours(Duration duration) { + return duration.toMillis() / 3600000.0; + } + + /** The estimated cost of changing from the given current resources to this. */ + public double costChangingFrom(ClusterModel model) { + return new ResourceChange(model, this).cost(); } @Override @@ -154,12 +167,13 @@ public class AllocatableClusterResources { .withBandwidthGbps(sum.bandwidthGbps() / nodes.size()); } - public static Optional<AllocatableClusterResources> from(ClusterResources wantedResources, - ApplicationId applicationId, - ClusterSpec clusterSpec, - Limits applicationLimits, - List<NodeResources> availableRealHostResources, - NodeRepository nodeRepository) { + public static Optional<AllocatableResources> from(ClusterResources wantedResources, + ApplicationId applicationId, + ClusterSpec clusterSpec, + Limits applicationLimits, + List<NodeResources> availableRealHostResources, + ClusterModel model, + NodeRepository nodeRepository) { var systemLimits = nodeRepository.nodeResourceLimits(); boolean exclusive = nodeRepository.exclusiveAllocation(clusterSpec); if (! exclusive) { @@ -193,8 +207,8 @@ public class AllocatableClusterResources { } else { // Return the cheapest flavor satisfying the requested resources, if any NodeResources cappedWantedResources = applicationLimits.cap(wantedResources.nodeResources()); - Optional<AllocatableClusterResources> best = Optional.empty(); - Optional<AllocatableClusterResources> bestDisregardingDiskLimit = Optional.empty(); + Optional<AllocatableResources> best = Optional.empty(); + Optional<AllocatableResources> bestDisregardingDiskLimit = Optional.empty(); for (Flavor flavor : nodeRepository.flavors().getFlavors()) { // Flavor decide resources: Real resources are the worst case real resources we'll get if we ask for these advertised resources NodeResources advertisedResources = nodeRepository.resourcesCalculator().advertisedResourcesOf(flavor); @@ -216,18 +230,18 @@ public class AllocatableClusterResources { if ( ! between(applicationLimits.min().nodeResources(), applicationLimits.max().nodeResources(), advertisedResources)) continue; if ( ! systemLimits.isWithinRealLimits(realResources, applicationId, clusterSpec)) continue; - var candidate = new AllocatableClusterResources(wantedResources.with(realResources), - advertisedResources, - wantedResources, - clusterSpec); + var candidate = new AllocatableResources(wantedResources.with(realResources), + advertisedResources, + wantedResources, + clusterSpec); if ( ! systemLimits.isWithinAdvertisedDiskLimits(advertisedResources, clusterSpec)) { // TODO: Remove when disk limit is enforced - if (bestDisregardingDiskLimit.isEmpty() || candidate.preferableTo(bestDisregardingDiskLimit.get())) { + if (bestDisregardingDiskLimit.isEmpty() || candidate.preferableTo(bestDisregardingDiskLimit.get(), model)) { bestDisregardingDiskLimit = Optional.of(candidate); } continue; } - if (best.isEmpty() || candidate.preferableTo(best.get())) { + if (best.isEmpty() || candidate.preferableTo(best.get(), model)) { best = Optional.of(candidate); } } @@ -237,13 +251,13 @@ public class AllocatableClusterResources { } } - private static AllocatableClusterResources calculateAllocatableResources(ClusterResources wantedResources, - NodeRepository nodeRepository, - ApplicationId applicationId, - ClusterSpec clusterSpec, - Limits applicationLimits, - boolean exclusive, - boolean bestCase) { + private static AllocatableResources calculateAllocatableResources(ClusterResources wantedResources, + NodeRepository nodeRepository, + ApplicationId applicationId, + ClusterSpec clusterSpec, + Limits applicationLimits, + boolean exclusive, + boolean bestCase) { var systemLimits = nodeRepository.nodeResourceLimits(); var advertisedResources = nodeRepository.resourcesCalculator().realToRequest(wantedResources.nodeResources(), exclusive, bestCase); advertisedResources = systemLimits.enlargeToLegal(advertisedResources, applicationId, clusterSpec, exclusive, true); // Ask for something legal @@ -255,10 +269,10 @@ public class AllocatableClusterResources { advertisedResources = advertisedResources.with(NodeResources.StorageType.remote); realResources = nodeRepository.resourcesCalculator().requestToReal(advertisedResources, exclusive, bestCase); } - return new AllocatableClusterResources(wantedResources.with(realResources), - advertisedResources, - wantedResources, - clusterSpec); + return new AllocatableResources(wantedResources.with(realResources), + advertisedResources, + wantedResources, + clusterSpec); } /** Returns true if the given resources could be allocated on any of the given host flavors */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java index 42bb16005ee..f650d8ec269 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java @@ -5,7 +5,6 @@ import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.IntRange; import com.yahoo.config.provision.NodeResources; import com.yahoo.vespa.hosted.provision.NodeRepository; -import com.yahoo.vespa.hosted.provision.provisioning.NodeResourceLimits; import java.util.Optional; @@ -35,21 +34,20 @@ public class AllocationOptimizer { * @return the best allocation, if there are any possible legal allocations, fulfilling the target * fully or partially, within the limits */ - public Optional<AllocatableClusterResources> findBestAllocation(Load loadAdjustment, - AllocatableClusterResources current, - ClusterModel clusterModel, - Limits limits) { + public Optional<AllocatableResources> findBestAllocation(Load loadAdjustment, + ClusterModel model, + Limits limits) { if (limits.isEmpty()) limits = Limits.of(new ClusterResources(minimumNodes, 1, NodeResources.unspecified()), new ClusterResources(maximumNodes, maximumNodes, NodeResources.unspecified()), IntRange.empty()); else - limits = atLeast(minimumNodes, limits).fullySpecified(current.clusterSpec(), nodeRepository, clusterModel.application().id()); - Optional<AllocatableClusterResources> bestAllocation = Optional.empty(); + limits = atLeast(minimumNodes, limits).fullySpecified(model.current().clusterSpec(), nodeRepository, model.application().id()); + Optional<AllocatableResources> bestAllocation = Optional.empty(); var availableRealHostResources = nodeRepository.zone().cloud().dynamicProvisioning() ? nodeRepository.flavors().getFlavors().stream().map(flavor -> flavor.resources()).toList() : nodeRepository.nodes().list().hosts().stream().map(host -> host.flavor().resources()) - .map(hostResources -> maxResourcesOf(hostResources, clusterModel)) + .map(hostResources -> maxResourcesOf(hostResources, model)) .toList(); for (int groups = limits.min().groups(); groups <= limits.max().groups(); groups++) { for (int nodes = limits.min().nodes(); nodes <= limits.max().nodes(); nodes++) { @@ -58,15 +56,16 @@ public class AllocationOptimizer { var resources = new ClusterResources(nodes, groups, nodeResourcesWith(nodes, groups, - limits, loadAdjustment, current, clusterModel)); - var allocatableResources = AllocatableClusterResources.from(resources, - clusterModel.application().id(), - current.clusterSpec(), - limits, - availableRealHostResources, - nodeRepository); + limits, loadAdjustment, model)); + var allocatableResources = AllocatableResources.from(resources, + model.application().id(), + model.current().clusterSpec(), + limits, + availableRealHostResources, + model, + nodeRepository); if (allocatableResources.isEmpty()) continue; - if (bestAllocation.isEmpty() || allocatableResources.get().preferableTo(bestAllocation.get())) + if (bestAllocation.isEmpty() || allocatableResources.get().preferableTo(bestAllocation.get(), model)) bestAllocation = allocatableResources; } } @@ -74,8 +73,8 @@ public class AllocationOptimizer { } /** Returns the max resources of a host one node may allocate. */ - private NodeResources maxResourcesOf(NodeResources hostResources, ClusterModel clusterModel) { - if (nodeRepository.exclusiveAllocation(clusterModel.clusterSpec())) return hostResources; + private NodeResources maxResourcesOf(NodeResources hostResources, ClusterModel model) { + if (nodeRepository.exclusiveAllocation(model.clusterSpec())) return hostResources; // static, shared hosts: Allocate at most half of the host cpu to simplify management return hostResources.withVcpu(hostResources.vcpu() / 2); } @@ -88,9 +87,8 @@ public class AllocationOptimizer { int groups, Limits limits, Load loadAdjustment, - AllocatableClusterResources current, - ClusterModel clusterModel) { - var loadWithTarget = clusterModel.loadAdjustmentWith(nodes, groups, loadAdjustment); + ClusterModel model) { + var loadWithTarget = model.loadAdjustmentWith(nodes, groups, loadAdjustment); // Leave some headroom above the ideal allocation to avoid immediately needing to scale back up if (loadAdjustment.cpu() < 1 && (1.0 - loadWithTarget.cpu()) < headroomRequiredToScaleDown) @@ -100,11 +98,11 @@ public class AllocationOptimizer { if (loadAdjustment.disk() < 1 && (1.0 - loadWithTarget.disk()) < headroomRequiredToScaleDown) loadAdjustment = loadAdjustment.withDisk(Math.min(1.0, loadAdjustment.disk() * (1.0 + headroomRequiredToScaleDown))); - loadWithTarget = clusterModel.loadAdjustmentWith(nodes, groups, loadAdjustment); + loadWithTarget = model.loadAdjustmentWith(nodes, groups, loadAdjustment); - var scaled = loadWithTarget.scaled(current.realResources().nodeResources()); + var scaled = loadWithTarget.scaled(model.current().realResources().nodeResources()); var nonScaled = limits.isEmpty() || limits.min().nodeResources().isUnspecified() - ? current.advertisedResources().nodeResources() + ? model.current().advertisedResources().nodeResources() : limits.min().nodeResources(); // min=max for non-scaled return nonScaled.withVcpu(scaled.vcpu()).withMemoryGb(scaled.memoryGb()).withDiskGb(scaled.diskGb()); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java index 32b59319a88..b5f86be68f6 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java @@ -54,40 +54,40 @@ public class Autoscaler { } private Autoscaling autoscale(Application application, Cluster cluster, NodeList clusterNodes, Limits limits) { - ClusterModel clusterModel = new ClusterModel(nodeRepository, - application, - clusterNodes.not().retired().clusterSpec(), - cluster, - clusterNodes, - nodeRepository.metricsDb(), - nodeRepository.clock()); - if (clusterModel.isEmpty()) return Autoscaling.empty(); + var model = new ClusterModel(nodeRepository, + application, + clusterNodes.not().retired().clusterSpec(), + cluster, + clusterNodes, + new AllocatableResources(clusterNodes.not().retired(), nodeRepository), + nodeRepository.metricsDb(), + nodeRepository.clock()); + if (model.isEmpty()) return Autoscaling.empty(); if (! limits.isEmpty() && cluster.minResources().equals(cluster.maxResources())) - return Autoscaling.dontScale(Autoscaling.Status.unavailable, "Autoscaling is not enabled", clusterModel); + return Autoscaling.dontScale(Autoscaling.Status.unavailable, "Autoscaling is not enabled", model); - if ( ! clusterModel.isStable(nodeRepository)) - return Autoscaling.dontScale(Status.waiting, "Cluster change in progress", clusterModel); + if ( ! model.isStable(nodeRepository)) + return Autoscaling.dontScale(Status.waiting, "Cluster change in progress", model); - var current = new AllocatableClusterResources(clusterNodes.not().retired(), nodeRepository); - var loadAdjustment = clusterModel.loadAdjustment(); + var loadAdjustment = model.loadAdjustment(); // Ensure we only scale down if we'll have enough headroom to not scale up again given a small load increase - var target = allocationOptimizer.findBestAllocation(loadAdjustment, current, clusterModel, limits); + var target = allocationOptimizer.findBestAllocation(loadAdjustment, model, limits); if (target.isEmpty()) - return Autoscaling.dontScale(Status.insufficient, "No allocations are possible within configured limits", clusterModel); + return Autoscaling.dontScale(Status.insufficient, "No allocations are possible within configured limits", model); - if (! worthRescaling(current.realResources(), target.get().realResources())) { + if (! worthRescaling(model.current().realResources(), target.get().realResources())) { if (target.get().fulfilment() < 0.9999999) - return Autoscaling.dontScale(Status.insufficient, "Configured limits prevents ideal scaling of this cluster", clusterModel); - else if ( ! clusterModel.safeToScaleDown() && clusterModel.idealLoad().any(v -> v < 1.0)) - return Autoscaling.dontScale(Status.ideal, "Cooling off before considering to scale down", clusterModel); + return Autoscaling.dontScale(Status.insufficient, "Configured limits prevents ideal scaling of this cluster", model); + else if ( ! model.safeToScaleDown() && model.idealLoad().any(v -> v < 1.0)) + return Autoscaling.dontScale(Status.ideal, "Cooling off before considering to scale down", model); else - return Autoscaling.dontScale(Status.ideal, "Cluster is ideally scaled (within configured limits)", clusterModel); + return Autoscaling.dontScale(Status.ideal, "Cluster is ideally scaled (within configured limits)", model); } - return Autoscaling.scaleTo(target.get().advertisedResources(), clusterModel); + return Autoscaling.scaleTo(target.get().advertisedResources(), model); } /** Returns true if it is worthwhile to make the given resource change, false if it is too insignificant */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaling.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaling.java index 0c86108b36c..fad280d6c29 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaling.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaling.java @@ -120,25 +120,25 @@ public class Autoscaling { } /** Creates an autoscaling conclusion which does not change the current allocation for a specified reason. */ - public static Autoscaling dontScale(Status status, String description, ClusterModel clusterModel) { + public static Autoscaling dontScale(Status status, String description, ClusterModel model) { return new Autoscaling(status, description, Optional.empty(), - clusterModel.at(), - clusterModel.peakLoad(), - clusterModel.idealLoad(), - clusterModel.metrics()); + model.at(), + model.peakLoad(), + model.idealLoad(), + model.metrics()); } /** Creates an autoscaling conclusion to scale. */ - public static Autoscaling scaleTo(ClusterResources target, ClusterModel clusterModel) { + public static Autoscaling scaleTo(ClusterResources target, ClusterModel model) { return new Autoscaling(Status.rescaling, "Rescaling initiated due to load changes", Optional.of(target), - clusterModel.at(), - clusterModel.peakLoad(), - clusterModel.idealLoad(), - clusterModel.metrics()); + model.at(), + model.peakLoad(), + model.idealLoad(), + model.metrics()); } public enum Status { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java index 0d64d4fbb10..8976dd9ff08 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java @@ -50,6 +50,7 @@ public class ClusterModel { private final Application application; private final ClusterSpec clusterSpec; private final Cluster cluster; + private final AllocatableResources current; private final CpuModel cpu = new CpuModel(); private final MemoryModel memory = new MemoryModel(); @@ -63,6 +64,7 @@ public class ClusterModel { private final Clock clock; private final Duration scalingDuration; + private final Duration allocationDuration; private final ClusterTimeseries clusterTimeseries; private final ClusterNodesTimeseries nodeTimeseries; private final Instant at; @@ -77,6 +79,7 @@ public class ClusterModel { ClusterSpec clusterSpec, Cluster cluster, NodeList clusterNodes, + AllocatableResources current, MetricsDb metricsDb, Clock clock) { this.nodeRepository = nodeRepository; @@ -84,8 +87,10 @@ public class ClusterModel { this.clusterSpec = clusterSpec; this.cluster = cluster; this.nodes = clusterNodes; + this.current = current; this.clock = clock; this.scalingDuration = cluster.scalingDuration(clusterSpec); + this.allocationDuration = cluster.allocationDuration(clusterSpec); this.clusterTimeseries = metricsDb.getClusterTimeseries(application.id(), cluster.id()); this.nodeTimeseries = new ClusterNodesTimeseries(scalingDuration(), cluster, nodes, metricsDb); this.at = clock.instant(); @@ -95,8 +100,10 @@ public class ClusterModel { Application application, ClusterSpec clusterSpec, Cluster cluster, + AllocatableResources current, Clock clock, Duration scalingDuration, + Duration allocationDuration, ClusterTimeseries clusterTimeseries, ClusterNodesTimeseries nodeTimeseries) { this.nodeRepository = nodeRepository; @@ -104,9 +111,11 @@ public class ClusterModel { this.clusterSpec = clusterSpec; this.cluster = cluster; this.nodes = NodeList.of(); + this.current = current; this.clock = clock; this.scalingDuration = scalingDuration; + this.allocationDuration = allocationDuration; this.clusterTimeseries = clusterTimeseries; this.nodeTimeseries = nodeTimeseries; this.at = clock.instant(); @@ -114,6 +123,7 @@ public class ClusterModel { public Application application() { return application; } public ClusterSpec clusterSpec() { return clusterSpec; } + public AllocatableResources current() { return current; } private ClusterNodesTimeseries nodeTimeseries() { return nodeTimeseries; } private ClusterTimeseries clusterTimeseries() { return clusterTimeseries; } @@ -127,6 +137,27 @@ public class ClusterModel { /** Returns the predicted duration of a rescaling of this cluster */ public Duration scalingDuration() { return scalingDuration; } + /** + * Returns the predicted duration of a resource change in this cluster, + * until we, or the application , will change it again. + */ + public Duration allocationDuration() { return allocationDuration; } + + public boolean isContent() { + return clusterSpec.type().isContent(); + } + + /** Returns the predicted duration of data redistribution in this cluster. */ + public Duration redistributionDuration() { + if (! isContent()) return Duration.ofMinutes(0); + return scalingDuration(); // TODO: Estimate separately + } + + /** Returns the predicted duration of replacing all the nodes in this cluster. */ + public Duration nodeReplacementDuration() { + return Duration.ofMinutes(5); // TODO: Estimate? + } + /** Returns the average of the peak load measurement in each dimension, from each node. */ public Load peakLoad() { return nodeTimeseries().peakLoad(); @@ -137,6 +168,10 @@ public class ClusterModel { return loadWith(nodeCount(), groupCount()); } + public boolean isExclusive() { + return nodeRepository.exclusiveAllocation(clusterSpec); + } + /** Returns the relative load adjustment that should be made to this cluster given available measurements. */ public Load loadAdjustment() { if (nodeTimeseries().measurementsPerNode() < 0.5) return Load.one(); // Don't change based on very little data @@ -237,16 +272,15 @@ public class ClusterModel { private Load adjustQueryDependentIdealLoadByBcpGroupInfo(Load ideal) { double currentClusterTotalVcpuPerGroup = nodes.not().retired().first().get().resources().vcpu() * groupSize(); - double targetQueryRateToHandle = ( canRescaleWithinBcpDeadline() ? averageQueryRate().orElse(0) : cluster.bcpGroupInfo().queryRate() ) * cluster.bcpGroupInfo().growthRateHeadroom() * trafficShiftHeadroom(); - double neededTotalVcpPerGroup = cluster.bcpGroupInfo().cpuCostPerQuery() * targetQueryRateToHandle / groupCount() + + double neededTotalVcpuPerGroup = cluster.bcpGroupInfo().cpuCostPerQuery() * targetQueryRateToHandle / groupCount() + ( 1 - cpu.queryFraction()) * cpu.idealLoad() * (clusterSpec.type().isContainer() ? 1 : groupSize()); - - double cpuAdjustment = neededTotalVcpPerGroup / currentClusterTotalVcpuPerGroup; - return ideal.withCpu(peakLoad().cpu() / cpuAdjustment); + // Max 1: Only use bcp group info if it indicates that we need to scale *up* + double cpuAdjustment = Math.max(1.0, neededTotalVcpuPerGroup / currentClusterTotalVcpuPerGroup); + return ideal.withCpu(ideal.cpu() / cpuAdjustment); } private boolean hasScaledIn(Duration period) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceChange.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceChange.java new file mode 100644 index 00000000000..7a26a217e61 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceChange.java @@ -0,0 +1,94 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.autoscale; + +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.NodeResources; + +import java.time.Duration; + +/** + * A resource change. + * + * @author bratseth + */ +public class ResourceChange { + + private final AllocatableResources from, to; + private final ClusterModel model; + + public ResourceChange(ClusterModel model, AllocatableResources to) { + this.from = model.current(); + this.to = to; + this.model = model; + } + + /** Returns the estimated total cost of this resource change (coming in addition to the "to" resource cost). */ + public double cost() { + if (model.isContent()) { + if (requiresNodeReplacement()) return toHours(model.redistributionDuration()) * from.cost(); + return toHours(model.redistributionDuration()) * from.advertisedResources().cost() * nodesToRetire(); + } + else { + if (requiresNodeReplacement()) return toHours(model.nodeReplacementDuration()) * from.cost(); + return 0; + } + } + + private boolean requiresRedistribution() { + if ( ! model.clusterSpec().type().isContent()) return false; + if (from.nodes() != to.nodes()) return true; + if (from.groups() != to.groups()) return true; + if (requiresNodeReplacement()) return true; + return false; + } + + /** + * Returns the estimated number of nodes that will be retired by this change, + * given that it is a content cluster and no node replacement is necessary. + * This is not necessarily always perfectly correct if this changes group layout. + */ + private int nodesToRetire() { + return Math.max(0, from.nodes() - to.nodes()); + } + + /** Returns true if the *existing* nodes of this needs to be replaced in this change. */ + private boolean requiresNodeReplacement() { + var fromNodes = from.advertisedResources().nodeResources(); + var toNodes = to.advertisedResources().nodeResources(); + + if (model.isExclusive()) { + return ! fromNodes.equals(toNodes); + } + else { + if ( ! fromNodes.justNonNumbers().equalsWhereSpecified(toNodes.justNonNumbers())) return true; + if ( ! canInPlaceResize()) return true; + return false; + } + } + + private double toHours(Duration duration) { + return duration.toMillis() / 3600000.0; + } + + private boolean canInPlaceResize() { + return canInPlaceResize(from.nodes(), from.advertisedResources().nodeResources(), + to.nodes(), to.advertisedResources().nodeResources(), + model.clusterSpec().type(), model.isExclusive(), from.groups() != to.groups()); + } + + public static boolean canInPlaceResize(int fromCount, NodeResources fromResources, + int toCount, NodeResources toResources, + ClusterSpec.Type type, boolean exclusive, boolean hasTopologyChange) { + if (exclusive) return false; // exclusive resources must match the host + + // Never allow in-place resize when also changing topology or decreasing cluster size + if (hasTopologyChange || toCount < fromCount) return false; + + // Do not allow increasing cluster size and decreasing node resources at the same time for content nodes + if (type.isContent() && toCount > fromCount && !toResources.satisfies(fromResources.justNumbers())) + return false; + + return true; + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java index 92f86325cf7..6a01a2bcd18 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java @@ -16,7 +16,7 @@ import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.applications.Application; import com.yahoo.vespa.hosted.provision.applications.Applications; import com.yahoo.vespa.hosted.provision.applications.Cluster; -import com.yahoo.vespa.hosted.provision.autoscale.AllocatableClusterResources; +import com.yahoo.vespa.hosted.provision.autoscale.AllocatableResources; import com.yahoo.vespa.hosted.provision.autoscale.Autoscaler; import com.yahoo.vespa.hosted.provision.autoscale.Autoscaling; import com.yahoo.vespa.hosted.provision.autoscale.NodeMetricSnapshot; @@ -87,7 +87,7 @@ public class AutoscalingMaintainer extends NodeRepositoryMaintainer { NodeList clusterNodes = nodeRepository().nodes().list(Node.State.active).owner(applicationId).cluster(clusterId); cluster = updateCompletion(cluster, clusterNodes); - var current = new AllocatableClusterResources(clusterNodes.not().retired(), nodeRepository()).advertisedResources(); + var current = new AllocatableResources(clusterNodes.not().retired(), nodeRepository()).advertisedResources(); // Autoscale unless an autoscaling is already in progress Autoscaling autoscaling = null; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDb.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDb.java index c388273b1a6..43a135a7e04 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDb.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDb.java @@ -88,8 +88,8 @@ public class CuratorDb { /** Simple cache for deserialized node objects, based on their ZK node version. */ private final Cache<Path, Pair<Integer, Node>> cachedNodes = CacheBuilder.newBuilder().recordStats().build(); - public CuratorDb(NodeFlavors flavors, Curator curator, Clock clock, boolean useCache, long nodeCacheSize) { - this.nodeSerializer = new NodeSerializer(flavors, nodeCacheSize); + public CuratorDb(NodeFlavors flavors, Curator curator, Clock clock, boolean useCache) { + this.nodeSerializer = new NodeSerializer(flavors); this.db = new CachingCurator(curator, root, useCache); this.clock = clock; this.provisionIndexCounter = new CuratorCounter(curator, root.append("provisionIndexCounter")); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java index 7e82ef55917..df39a0230b6 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java @@ -134,7 +134,7 @@ public class NodeSerializer { // ---------------- Serialization ---------------------------------------------------- - public NodeSerializer(NodeFlavors flavors, long cacheSize) { + public NodeSerializer(NodeFlavors flavors) { this.flavors = flavors; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java index 8a39f309935..5ce5bc8abd0 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java @@ -98,10 +98,7 @@ public class CapacityPolicies { Architecture architecture = adminClusterArchitecture(applicationId); if (nodeRepository.exclusiveAllocation(clusterSpec)) { - var resources = legacySmallestExclusiveResources(); //TODO: use 8Gb as default when no apps are using 4Gb - return versioned(clusterSpec, Map.of(new Version(0), resources, - new Version(8, 182, 12), resources.with(architecture), - new Version(8, 187), smallestExclusiveResources().with(architecture))); + return smallestExclusiveResources().with(architecture); } if (clusterSpec.id().value().equals("cluster-controllers")) { @@ -131,8 +128,7 @@ public class CapacityPolicies { // 1.32 fits floor(8/1.32) = 6 cluster controllers on each 8Gb host, and each will have // 1.32-(0.7+0.6)*(1.32/8) = 1.1 Gb real memory given current taxes. if (architecture == Architecture.x86_64) - return versioned(clusterSpec, Map.of(new Version(0), new NodeResources(0.25, 1.14, 10, 0.3), - new Version(8, 129, 4), new NodeResources(0.25, 1.32, 10, 0.3))); + return versioned(clusterSpec, Map.of(new Version(0), new NodeResources(0.25, 1.32, 10, 0.3))); else // arm64 nodes need more memory return versioned(clusterSpec, Map.of(new Version(0), new NodeResources(0.25, 1.50, 10, 0.3))); @@ -159,13 +155,6 @@ public class CapacityPolicies { } // The lowest amount of resources that can be exclusive allocated (i.e. a matching host flavor for this exists) - private NodeResources legacySmallestExclusiveResources() { - return (zone.cloud().name().equals(CloudName.GCP)) - ? new NodeResources(1, 4, 50, 0.3) - : new NodeResources(0.5, 4, 50, 0.3); - } - - // The lowest amount of resources that can be exclusive allocated (i.e. a matching host flavor for this exists) private NodeResources smallestExclusiveResources() { return (zone.cloud().name().equals(CloudName.GCP)) ? new NodeResources(2, 8, 50, 0.3) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java index 3d0c1069584..a67a513550a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java @@ -23,7 +23,7 @@ import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.applications.Application; import com.yahoo.vespa.hosted.provision.applications.Cluster; -import com.yahoo.vespa.hosted.provision.autoscale.AllocatableClusterResources; +import com.yahoo.vespa.hosted.provision.autoscale.AllocatableResources; import com.yahoo.vespa.hosted.provision.autoscale.AllocationOptimizer; import com.yahoo.vespa.hosted.provision.autoscale.ClusterModel; import com.yahoo.vespa.hosted.provision.autoscale.Limits; @@ -182,12 +182,12 @@ public class NodeRepositoryProvisioner implements Provisioner { .not().retired() .not().removable(); boolean firstDeployment = nodes.isEmpty(); - AllocatableClusterResources currentResources = + var current = firstDeployment // start at min, preserve current resources otherwise - ? new AllocatableClusterResources(initialResourcesFrom(requested, clusterSpec, application.id()), clusterSpec, nodeRepository) - : new AllocatableClusterResources(nodes, nodeRepository); - var clusterModel = new ClusterModel(nodeRepository, application, clusterSpec, cluster, nodes, nodeRepository.metricsDb(), nodeRepository.clock()); - return within(Limits.of(requested), currentResources, firstDeployment, clusterModel); + ? new AllocatableResources(initialResourcesFrom(requested, clusterSpec, application.id()), clusterSpec, nodeRepository) + : new AllocatableResources(nodes, nodeRepository); + var model = new ClusterModel(nodeRepository, application, clusterSpec, cluster, nodes, current, nodeRepository.metricsDb(), nodeRepository.clock()); + return within(Limits.of(requested), model, firstDeployment); } private ClusterResources initialResourcesFrom(Capacity requested, ClusterSpec clusterSpec, ApplicationId applicationId) { @@ -197,21 +197,19 @@ public class NodeRepositoryProvisioner implements Provisioner { /** Make the minimal adjustments needed to the current resources to stay within the limits */ private ClusterResources within(Limits limits, - AllocatableClusterResources current, - boolean firstDeployment, - ClusterModel clusterModel) { + ClusterModel model, + boolean firstDeployment) { if (limits.min().equals(limits.max())) return limits.min(); // Don't change current deployments that are still legal - if (! firstDeployment && current.advertisedResources().isWithin(limits.min(), limits.max())) - return current.advertisedResources(); + if (! firstDeployment && model.current().advertisedResources().isWithin(limits.min(), limits.max())) + return model.current().advertisedResources(); // Otherwise, find an allocation that preserves the current resources as well as possible return allocationOptimizer.findBestAllocation(Load.one(), - current, - clusterModel, + model, limits) - .orElseThrow(() -> newNoAllocationPossible(current.clusterSpec(), limits)) + .orElseThrow(() -> newNoAllocationPossible(model.current().clusterSpec(), limits)) .advertisedResources(); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java index cea0608013d..77f37cadc0b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java @@ -6,6 +6,7 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.autoscale.ResourceChange; import java.time.Duration; import java.util.Map; @@ -162,16 +163,11 @@ public interface NodeSpec { @Override public boolean canResize(NodeResources currentNodeResources, NodeResources currentSpareHostResources, ClusterSpec.Type type, boolean hasTopologyChange, int currentClusterSize) { - if (exclusive) return false; // exclusive resources must match the host - // Never allow in-place resize when also changing topology or decreasing cluster size - if (hasTopologyChange || count < currentClusterSize) return false; + return ResourceChange.canInPlaceResize(currentClusterSize, currentNodeResources, count, requestedNodeResources, + type, exclusive, hasTopologyChange) + && + currentSpareHostResources.add(currentNodeResources.justNumbers()).satisfies(requestedNodeResources); - // Do not allow increasing cluster size and decreasing node resources at the same time for content nodes - if (type.isContent() && count > currentClusterSize && !requestedNodeResources.satisfies(currentNodeResources.justNumbers())) - return false; - - // Otherwise, allowed as long as the host can satisfy the new requested resources - return currentSpareHostResources.add(currentNodeResources.justNumbers()).satisfies(requestedNodeResources); } @Override diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java index e3f67721eb5..90cf37aa876 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java @@ -96,7 +96,7 @@ public class MockNodeRepository extends NodeRepository { new MemoryMetricsDb(Clock.fixed(Instant.ofEpochMilli(123), ZoneId.of("Z"))), new OrchestratorMock(), true, - 0, 1000); + 0); this.flavors = flavors; defaultCloudAccount = zone.cloud().account(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java index 49702a7d4c1..bf714cd9df1 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java @@ -54,7 +54,7 @@ public class NodeRepositoryTester { new MemoryMetricsDb(clock), new OrchestratorMock(), true, - 0, 1000); + 0); } public NodeRepository nodeRepository() { return nodeRepository; } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/RealDataScenarioTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/RealDataScenarioTest.java index 1ed3c13cfff..f64e50310bb 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/RealDataScenarioTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/RealDataScenarioTest.java @@ -133,7 +133,7 @@ public class RealDataScenarioTest { } private static void initFromZk(NodeRepository nodeRepository, Path pathToZkSnapshot) { - NodeSerializer nodeSerializer = new NodeSerializer(nodeRepository.flavors(), 1000); + NodeSerializer nodeSerializer = new NodeSerializer(nodeRepository.flavors()); AtomicBoolean nodeNext = new AtomicBoolean(false); Pattern zkNodePathPattern = Pattern.compile(".?/provision/v1/nodes/[a-z0-9.-]+\\.(com|cloud).?"); Consumer<String> consumer = input -> { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java index d33857d1a1e..4e19d04ffac 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java @@ -6,6 +6,7 @@ import com.yahoo.config.provision.ClusterInfo; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.IntRange; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeResources.DiskSpeed; @@ -18,6 +19,7 @@ import com.yahoo.vespa.hosted.provision.provisioning.DynamicProvisioningTester; import org.junit.Test; import java.time.Duration; +import java.util.List; import java.util.Optional; import static com.yahoo.config.provision.NodeResources.DiskSpeed.fast; @@ -88,7 +90,7 @@ public class AutoscalingTest { fixture.tester().clock().advance(Duration.ofDays(7)); fixture.loader().applyCpuLoad(0.1f, 10); fixture.tester().assertResources("Scaling cpu down since usage has gone down significantly", - 6, 1, 1.1, 9.8, 390.2, + 9, 1, 1.0, 6.5, 243.9, fixture.autoscale()); } @@ -173,7 +175,7 @@ public class AutoscalingTest { fixture.setScalingDuration(Duration.ofHours(12)); // Fixture sets last completion to be 1 day into the past fixture.loader().applyLoad(new Load(1.0, 0.1, 1.0), 10); fixture.tester().assertResources("Scaling up (only) since resource usage is too high", - 8, 1, 7.1, 9.3, 75.4, + 5, 1, 11.7, 15.4, 132.0, fixture.autoscale()); } @@ -185,7 +187,7 @@ public class AutoscalingTest { fixture.tester.clock().advance(Duration.ofDays(2)); fixture.loader().applyLoad(new Load(1.0, 0.1, 1.0), 10); fixture.tester().assertResources("Scaling cpu and disk up and memory down", - 7, 1, 8.2, 4.0, 88.0, + 5, 1, 11.7, 4.0, 132.0, fixture.autoscale()); } @@ -208,7 +210,7 @@ public class AutoscalingTest { fixture.loader().applyCpuLoad(0.70, 1); fixture.loader().applyCpuLoad(0.01, 100); fixture.tester().assertResources("Scaling up since peak resource usage is too high", - 8, 1, 4.3, 7.4, 29.0, + 5, 1, 7.1, 12.3, 50.7, fixture.autoscale()); } @@ -232,7 +234,7 @@ public class AutoscalingTest { fixture.loader().applyCpuLoad(0.70, 1); fixture.loader().applyCpuLoad(0.01, 100); fixture.tester().assertResources("Scaling up cpu since peak resource usage is too high", - 8, 1, 4.3, 7.7, 34.3, + 5, 1, 7.1, 12.8, 60.0, fixture.autoscale()); } @@ -393,11 +395,10 @@ public class AutoscalingTest { .initialResources(Optional.of(now)) .capacity(Capacity.from(min, max)) .build(); - fixture.setScalingDuration(Duration.ofHours(6)); fixture.tester().clock().advance(Duration.ofDays(2)); - fixture.loader().applyCpuLoad(0.4, 240); + fixture.loader().applyCpuLoad(0.5, 240); fixture.tester().assertResources("Scaling cpu up", - 6, 6, 5.0, 7.4, 22.3, + 6, 6, 4.5, 7.4, 22.3, fixture.autoscale()); } @@ -460,7 +461,7 @@ public class AutoscalingTest { fixture.tester().clock().advance(Duration.ofDays(2)); fixture.loader().applyCpuLoad(1.0, 120); fixture.tester().assertResources("Suggesting above capacity limit", - 8, 1, 6.2, 7.4, 29.0, + 5, 1, 10.2, 12.3, 50.7, fixture.tester().suggest(fixture.applicationId, fixture.clusterSpec.id(), min, min)); } @@ -593,13 +594,12 @@ public class AutoscalingTest { .initialResources(Optional.of(now)) .capacity(Capacity.from(min, max)) .build(); - fixture.setScalingDuration(Duration.ofHours(6)); fixture.tester().clock().advance(Duration.ofDays(2)); Duration timePassed = fixture.loader().addCpuMeasurements(0.25, 120); fixture.tester().clock().advance(timePassed.negated()); fixture.loader().addLoadMeasurements(10, t -> t == 0 ? 200.0 : 100.0, t -> 10.0); - fixture.tester().assertResources("Scaling up cpu, others down, changing to 1 group is cheaper", - 7, 1, 3.2, 43.3, 129.8, + fixture.tester().assertResources("Changing to 1 group is cheaper", + 7, 1, 2.5, 43.3, 129.8, fixture.autoscale()); } @@ -650,11 +650,10 @@ public class AutoscalingTest { .initialResources(Optional.of(now)) .capacity(Capacity.from(min, max)) .build(); - fixture.setScalingDuration(Duration.ofHours(6)); fixture.tester().clock().advance(Duration.ofDays(2)); fixture.loader().applyLoad(new Load(0.16, 0.02, 0.5), 120); fixture.tester().assertResources("Scaling down memory", - 7, 1, 2.5, 4.0, 80.2, + 6, 1, 2.1, 4.0, 96.2, fixture.autoscale()); } @@ -710,16 +709,16 @@ public class AutoscalingTest { fixture.tester.clock().advance(timeAdded.negated()); fixture.loader().addCpuMeasurements(0.25, 200); fixture.tester().assertResources("Scale up since we assume we need 2x cpu for growth when no scaling time data", - 8, 1, 1.6, 7.4, 29.0, + 5, 1, 2.6, 12.3, 50.7, fixture.autoscale()); fixture.setScalingDuration(Duration.ofHours(8)); fixture.tester().clock().advance(Duration.ofDays(2)); timeAdded = fixture.loader().addLoadMeasurements(100, t -> 100.0 + (t < 50 ? t : 100 - t), t -> 0.0); fixture.tester.clock().advance(timeAdded.negated()); - fixture.loader().addCpuMeasurements(0.25, 200); + fixture.loader().addCpuMeasurements(0.20, 200); fixture.tester().assertResources("Scale down since observed growth is slower than scaling time", - 8, 1, 1.2, 7.4, 29.0, + 5, 1, 1.6, 12.3, 50.7, fixture.autoscale()); fixture.setScalingDuration(Duration.ofHours(8)); @@ -730,7 +729,7 @@ public class AutoscalingTest { fixture.tester.clock().advance(timeAdded.negated()); fixture.loader().addCpuMeasurements(0.25, 200); fixture.tester().assertResources("Scale up since observed growth is faster than scaling time", - 8, 1, 1.5, 7.4, 29.0, + 5, 1, 2.4, 12.3, 50.7, fixture.autoscale()); } @@ -747,7 +746,7 @@ public class AutoscalingTest { fixture.tester.clock().advance(timeAdded.negated()); fixture.loader().addCpuMeasurements(0.7, 200); fixture.tester().assertResources("Scale up slightly since observed growth is faster than scaling time, but we are not confident", - 8, 1, 1.3, 7.4, 29.0, + 5, 1, 2.2, 12.3, 50.7, fixture.autoscale()); } @@ -766,16 +765,16 @@ public class AutoscalingTest { fixture.tester.clock().advance(timeAdded.negated()); fixture.loader().addCpuMeasurements(0.4, 200); fixture.tester.assertResources("Query and write load is equal -> scale up somewhat", - 8, 1, 1.8, 7.4, 29.0, + 5, 1, 2.9, 12.3, 50.7, fixture.autoscale()); fixture.tester().clock().advance(Duration.ofDays(2)); timeAdded = fixture.loader().addLoadMeasurements(100, t -> t == 0 ? 800.0 : 400.0, t -> 100.0); fixture.tester.clock().advance(timeAdded.negated()); fixture.loader().addCpuMeasurements(0.4, 200); - // TODO: Ackhually, we scale down here - why? + // TODO: Ackhually, we scale up less here - why? fixture.tester().assertResources("Query load is 4x write load -> scale up more", - 8, 1, 1.4, 7.4, 29.0, + 5, 1, 2.2, 12.3, 50.7, fixture.autoscale()); fixture.tester().clock().advance(Duration.ofDays(2)); @@ -783,7 +782,7 @@ public class AutoscalingTest { fixture.tester.clock().advance(timeAdded.negated()); fixture.loader().addCpuMeasurements(0.4, 200); fixture.tester().assertResources("Write load is 10x query load -> scale down", - 6, 1, 1.1, 10.0, 40.5, + 5, 1, 1.3, 12.3, 50.7, fixture.autoscale()); fixture.tester().clock().advance(Duration.ofDays(2)); @@ -791,7 +790,7 @@ public class AutoscalingTest { fixture.tester.clock().advance(timeAdded.negated()); fixture.loader().addCpuMeasurements(0.4, 200); fixture.tester().assertResources("Query only -> larger", - 8, 1, 2.1, 7.4, 29.0, + 5, 1, 3.5, 12.3, 50.7, fixture.autoscale()); fixture.tester().clock().advance(Duration.ofDays(2)); @@ -954,4 +953,32 @@ public class AutoscalingTest { .build(); } + @Test + public void change_not_requiring_node_replacement_is_preferred() { + var min = new ClusterResources(5, 1, new NodeResources( 16, 64, 200, 1, DiskSpeed.fast, StorageType.remote)); + var max = new ClusterResources(6, 1, new NodeResources( 16, 64, 200, 1, DiskSpeed.fast, StorageType.remote)); + + List<Flavor> flavors = List.of(new Flavor("arm_16", new NodeResources( 16, 64, 200, 1, DiskSpeed.fast, StorageType.remote, NodeResources.Architecture.arm64)), + new Flavor("x86_16", new NodeResources( 16, 64, 200, 1, DiskSpeed.fast, StorageType.remote, NodeResources.Architecture.x86_64))); + var fixture = DynamicProvisioningTester.fixture() + .clusterType(ClusterSpec.Type.container) + .hostFlavors(flavors) + .awsZone(false, Environment.prod) + .capacity(Capacity.from(min, max)) + .initialResources(Optional.of(min.with(min.nodeResources().with(NodeResources.Architecture.x86_64)))) + .build(); + var nodes = fixture.nodes().not().retired().asList(); + assertEquals(5, nodes.size()); + assertEquals(NodeResources.Architecture.x86_64, nodes.get(0).resources().architecture()); + + fixture.tester().clock().advance(Duration.ofHours(5)); + fixture.loader().applyCpuLoad(0.27, 10); // trigger rescaling, but don't cause fulfilment < 1 + var autoscaling = fixture.autoscale(); + fixture.deploy(Capacity.from(autoscaling.resources().get())); + nodes = fixture.nodes().not().retired().asList(); + assertEquals(6, nodes.size()); + assertEquals("We stay with x86 even though the first matching flavor is arm", + NodeResources.Architecture.x86_64, nodes.get(0).resources().architecture()); + } + } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingUsingBcpGroupInfoTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingUsingBcpGroupInfoTest.java index 379dbb27d87..be7bc3c44a8 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingUsingBcpGroupInfoTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingUsingBcpGroupInfoTest.java @@ -32,7 +32,7 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.store(new BcpGroupInfo(100, 1.1, 0.3)); fixture.loader().addCpuMeasurements(0.7f, 10); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 8, 1, 4.0, 7.4, 29.0, + 8, 1, 3.4, 7.4, 29.0, fixture.autoscale()); // Higher query rate @@ -40,7 +40,7 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.store(new BcpGroupInfo(200, 1.1, 0.3)); fixture.loader().addCpuMeasurements(0.7f, 10); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 8, 1, 8.0, 7.4, 29.0, + 8, 1, 6.8, 7.4, 29.0, fixture.autoscale()); // Higher headroom @@ -48,7 +48,7 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.store(new BcpGroupInfo(100, 1.3, 0.3)); fixture.loader().addCpuMeasurements(0.7f, 10); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 8, 1, 4.8, 7.4, 29.0, + 8, 1, 4.0, 7.4, 29.0, fixture.autoscale()); // Higher per query cost @@ -56,7 +56,7 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.store(new BcpGroupInfo(100, 1.1, 0.45)); fixture.loader().addCpuMeasurements(0.7f, 10); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 8, 1, 6.0, 7.4, 29.0, + 8, 1, 5.1, 7.4, 29.0, fixture.autoscale()); // Bcp elsewhere is 0 - use local only @@ -85,7 +85,7 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.store(new BcpGroupInfo(100, 1.1, 0.3)); fixture.loader().addCpuMeasurements(0.7f, 10); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 3, 3, 10.5, 43.2, 190.0, + 3, 3, 11.7, 43.2, 190.0, fixture.autoscale()); // Higher query rate @@ -93,7 +93,7 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.store(new BcpGroupInfo(200, 1.1, 0.3)); fixture.loader().addCpuMeasurements(0.7f, 10); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 3, 3, 20.9, 43.2, 190.0, + 3, 3, 23.1, 43.2, 190.0, fixture.autoscale()); // Higher headroom @@ -101,7 +101,7 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.store(new BcpGroupInfo(100, 1.3, 0.3)); fixture.loader().addCpuMeasurements(0.7f, 10); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 3, 3, 12.4, 43.2, 190.0, + 3, 3, 13.8, 43.2, 190.0, fixture.autoscale()); // Higher per query cost @@ -109,7 +109,7 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.store(new BcpGroupInfo(100, 1.1, 0.45)); fixture.loader().addCpuMeasurements(0.7f, 10); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 3, 3, 15.7, 43.2, 190.0, + 3, 3, 17.4, 43.2, 190.0, fixture.autoscale()); } @@ -127,7 +127,7 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.store(new BcpGroupInfo(100, 1.1, 0.3)); fixture.loader().addCpuMeasurements(0.7f, 10); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 8, 1, 4.0, 16.0, 40.8, + 4, 1, 8.0, 16.0, 40.8, fixture.autoscale()); // Higher query rate (mem and disk changes are due to being assigned larger hosts where we get less overhead share @@ -135,7 +135,7 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.store(new BcpGroupInfo(200, 1.1, 0.3)); fixture.loader().addCpuMeasurements(0.7f, 10); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 8, 1, 8.0, 16.0, 40.8, + 7, 1, 8.0, 16.0, 40.8, fixture.autoscale()); // Higher headroom @@ -143,7 +143,7 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.store(new BcpGroupInfo(100, 1.3, 0.3)); fixture.loader().addCpuMeasurements(0.7f, 10); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 5, 1, 8.0, 16.0, 40.8, + 8, 1, 4.0, 16.0, 40.8, fixture.autoscale()); // Higher per query cost @@ -151,7 +151,7 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.store(new BcpGroupInfo(100, 1.1, 0.45)); fixture.loader().addCpuMeasurements(0.7f, 10); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 6, 1, 8.0, 16.0, 40.8, + 10, 1, 4.0, 16.0, 40.8, fixture.autoscale()); } @@ -173,7 +173,7 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.store(new BcpGroupInfo(100, 1.1, 0.45)); fixture.loader().addCpuMeasurements(0.7f, 10); fixture.tester().assertResources("No need for traffic shift headroom", - 2, 1, 2.0, 16.0, 40.8, + 3, 1, 4.0, 16.0, 40.8, fixture.autoscale()); } @@ -186,7 +186,7 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.store(new BcpGroupInfo(200, 1.3, 0.45)); fixture.loader().addCpuMeasurements(0.7f, 10); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 8, 1, 14.2, 7.4, 29.0, + 8, 1, 11.9, 7.4, 29.0, fixture.autoscale()); // Some local traffic @@ -196,7 +196,7 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.tester().clock().advance(duration1.negated()); fixture.loader().addQueryRateMeasurements(10, __ -> 10.0); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 8, 1, 6.9, 7.4, 29.0, + 8, 1, 6.8, 7.4, 29.0, fixture.autoscale()); // Enough local traffic to get half the votes @@ -206,7 +206,7 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.tester().clock().advance(duration2.negated()); fixture.loader().addQueryRateMeasurements(10, __ -> 50.0); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 8, 1, 2.9, 7.4, 29.0, + 8, 1, 3.0, 7.4, 29.0, fixture.autoscale()); // Mostly local @@ -270,6 +270,21 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.autoscale()); } + @Test + public void test_autoscaling_containers_with_some_local_traffic() { + var fixture = DynamicProvisioningTester.fixture().clusterType(ClusterSpec.Type.container).awsProdSetup(true).build(); + + // Some local traffic + fixture.tester().clock().advance(Duration.ofDays(2)); + fixture.store(new BcpGroupInfo(200, 1.9, 0.01)); + Duration duration1 = fixture.loader().addCpuMeasurements(0.58f, 10); + fixture.tester().clock().advance(duration1.negated()); + fixture.loader().addQueryRateMeasurements(10, __ -> 10.0); + fixture.tester().assertResources("Not scaling down due to group info, even though it contains much evidence queries are cheap", + 3, 1, 4.0, 16.0, 40.8, + fixture.autoscale()); + } + /** Tests with varying BCP group info parameters. */ @Test public void test_autoscaling_metrics() { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModelTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModelTest.java index ec084014a6a..f07d52a4a7f 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModelTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModelTest.java @@ -5,17 +5,12 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeResources; -import com.yahoo.config.provision.Zone; import com.yahoo.test.ManualClock; -import com.yahoo.vespa.curator.mock.MockCurator; -import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.applications.Application; import com.yahoo.vespa.hosted.provision.applications.Cluster; import com.yahoo.vespa.hosted.provision.applications.Status; import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester; -import com.yahoo.vespa.hosted.provision.testutils.MockNodeRepository; import org.junit.Test; import java.time.Duration; @@ -36,10 +31,10 @@ public class ClusterModelTest { public void unit_adjustment_should_cause_no_change() { var model = clusterModelWithNoData(); // 5 nodes, 1 group assertEquals(Load.one(), model.loadAdjustment()); - var target = model.loadAdjustment().scaled(resources()); + var target = model.loadAdjustment().scaled(nodeResources()); int testingNodes = 5 - 1; int currentNodes = 5 - 1; - assertEquals(resources(), model.loadWith(testingNodes, 1).scaled(Load.one().divide(model.loadWith(currentNodes, 1)).scaled(target))); + assertEquals(nodeResources(), model.loadWith(testingNodes, 1).scaled(Load.one().divide(model.loadWith(currentNodes, 1)).scaled(target))); } @Test @@ -91,16 +86,23 @@ public class ClusterModelTest { ManualClock clock = new ManualClock(); Application application = Application.empty(ApplicationId.from("t1", "a1", "i1")); ClusterSpec clusterSpec = clusterSpec(); - Cluster cluster = cluster(resources()); + Cluster cluster = cluster(); application = application.with(cluster); - return new ClusterModel(new ProvisioningTester.Builder().build().nodeRepository(), + var nodeRepository = new ProvisioningTester.Builder().build().nodeRepository(); + return new ClusterModel(nodeRepository, application.with(status), - clusterSpec, cluster, clock, Duration.ofMinutes(10), + clusterSpec, cluster, + new AllocatableResources(clusterResources(), clusterSpec, nodeRepository), + clock, Duration.ofMinutes(10), Duration.ofMinutes(5), timeseries(cluster,100, queryRate, writeRate, clock), ClusterNodesTimeseries.empty()); } - private NodeResources resources() { + private ClusterResources clusterResources() { + return new ClusterResources(5, 1, nodeResources()); + } + + private NodeResources nodeResources() { return new NodeResources(1, 10, 100, 1); } @@ -111,10 +113,10 @@ public class ClusterModelTest { .build(); } - private Cluster cluster(NodeResources resources) { + private Cluster cluster() { return Cluster.create(ClusterSpec.Id.from("test"), false, - Capacity.from(new ClusterResources(5, 1, resources))); + Capacity.from(clusterResources())); } /** Creates the given number of measurements, spaced 5 minutes between, using the given function */ diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/Fixture.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/Fixture.java index 33d3d3d50dc..78feba14fbf 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/Fixture.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/Fixture.java @@ -5,17 +5,14 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.Cloud; -import com.yahoo.config.provision.ClusterInfo; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.Flavor; -import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; -import com.yahoo.vespa.curator.mock.MockCurator; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.flags.custom.HostResources; @@ -29,7 +26,6 @@ import com.yahoo.vespa.hosted.provision.autoscale.awsnodes.AwsHostResourcesCalcu import com.yahoo.vespa.hosted.provision.autoscale.awsnodes.AwsNodeTypes; import com.yahoo.vespa.hosted.provision.provisioning.DynamicProvisioningTester; import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator; -import com.yahoo.vespa.hosted.provision.testutils.MockNodeRepository; import java.time.Duration; import java.util.Arrays; @@ -72,9 +68,9 @@ public class Fixture { return tester().nodeRepository().applications().get(applicationId).orElse(Application.empty(applicationId)); } - public AllocatableClusterResources currentResources() { - return new AllocatableClusterResources(tester.nodeRepository().nodes().list(Node.State.active).owner(applicationId).cluster(clusterId()), - tester.nodeRepository()); + public AllocatableResources currentResources() { + return new AllocatableResources(tester.nodeRepository().nodes().list(Node.State.active).owner(applicationId).cluster(clusterId()), + tester.nodeRepository()); } public Cluster cluster() { @@ -89,6 +85,7 @@ public class Fixture { clusterSpec, cluster(), nodes(), + new AllocatableResources(nodes(), tester.nodeRepository()), tester.nodeRepository().metricsDb(), tester.nodeRepository().clock()); } @@ -180,6 +177,7 @@ public class Fixture { new NodeResources(100, 1000, 1000, 1, NodeResources.DiskSpeed.any))); HostResourcesCalculator resourceCalculator = new DynamicProvisioningTester.MockHostResourcesCalculator(zone); final InMemoryFlagSource flagSource = new InMemoryFlagSource(); + boolean reversedFlavorOrder = false; int hostCount = 0; public Fixture.Builder zone(Zone zone) { @@ -228,12 +226,16 @@ public class Fixture { public Fixture.Builder awsSetup(boolean allowHostSharing, Environment environment) { return this.awsHostFlavors() .awsResourceCalculator() - .zone(new Zone(Cloud.builder().dynamicProvisioning(true) - .allowHostSharing(allowHostSharing) - .build(), - SystemName.Public, - environment, - RegionName.from("aws-eu-west-1a"))); + .awsZone(allowHostSharing, environment); + } + + public Fixture.Builder awsZone(boolean allowHostSharing, Environment environment) { + return zone(new Zone(Cloud.builder().dynamicProvisioning(true) + .allowHostSharing(allowHostSharing) + .build(), + SystemName.Public, + environment, + RegionName.from("aws-eu-west-1a"))); } public Fixture.Builder vespaVersion(Version version) { @@ -246,6 +248,11 @@ public class Fixture { return this; } + public Fixture.Builder hostFlavors(List<Flavor> hostFlavors) { + this.hostFlavors = hostFlavors; + return this; + } + /** Adds the host resources available on AWS. */ public Fixture.Builder awsHostFlavors() { this.hostFlavors = AwsNodeTypes.asFlavors(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java index 523feeeb303..eedf4946e3a 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java @@ -78,8 +78,7 @@ public class CapacityCheckerTester { new MemoryMetricsDb(clock), new OrchestratorMock(), true, - 0, - 1000); + 0); } private void updateCapacityChecker() { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java index 8aaf0eb20e7..3145675325b 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java @@ -75,7 +75,7 @@ public class ScalingSuggestionsMaintainerTest { assertEquals("8 nodes with [vcpu: 3.2, memory: 4.5 Gb, disk: 10.0 Gb, bandwidth: 0.1 Gbps, architecture: any]", suggestionOf(app1, cluster1, tester).resources().get().toString()); - assertEquals("8 nodes with [vcpu: 3.6, memory: 4.7 Gb, disk: 14.2 Gb, bandwidth: 0.1 Gbps, architecture: any]", + assertEquals("7 nodes with [vcpu: 4.1, memory: 5.3 Gb, disk: 16.5 Gb, bandwidth: 0.1 Gbps, architecture: any]", suggestionOf(app2, cluster2, tester).resources().get().toString()); // Utilization goes way down diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainerTest.java index a5ac2be72ee..6d67f39d9bb 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainerTest.java @@ -273,8 +273,7 @@ public class SpareCapacityMaintainerTest { new MemoryMetricsDb(clock), new OrchestratorMock(), true, - 1, - 1000); + 1); deployer = new MockDeployer(nodeRepository); maintainer = new SpareCapacityMaintainer(deployer, nodeRepository, metric, Duration.ofDays(1), maxIterations); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDbTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDbTest.java index c0d6ab90f06..e755f3c3cfc 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDbTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDbTest.java @@ -24,7 +24,7 @@ public class CuratorDbTest { private final Curator curator = new MockCurator(); private final CuratorDb zkClient = new CuratorDb( - FlavorConfigBuilder.createDummies("default"), curator, Clock.systemUTC(), true, 1000); + FlavorConfigBuilder.createDummies("default"), curator, Clock.systemUTC(), true); @Test public void can_read_stored_host_information() throws Exception { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java index 56f03423ad2..6e2d1e7fcd6 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java @@ -60,7 +60,7 @@ import static org.junit.Assert.assertTrue; public class NodeSerializerTest { private final NodeFlavors nodeFlavors = FlavorConfigBuilder.createDummies("default", "large", "ugccloud-container", "arm64", "gpu"); - private final NodeSerializer nodeSerializer = new NodeSerializer(nodeFlavors, 1000); + private final NodeSerializer nodeSerializer = new NodeSerializer(nodeFlavors); private final ManualClock clock = new ManualClock(); @Test diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java index bca48b19ccf..60dd9ce59ef 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java @@ -128,8 +128,7 @@ public class ProvisioningTester { new MemoryMetricsDb(clock), orchestrator, true, - spareCount, - 1000); + spareCount); this.provisioner = new NodeRepositoryProvisioner(nodeRepository, zone, provisionServiceProvider, new MockMetric()); this.capacityPolicies = new CapacityPolicies(nodeRepository); this.provisionLogger = new InMemoryProvisionLogger(); diff --git a/searchcore/src/tests/proton/matching/matching_test.cpp b/searchcore/src/tests/proton/matching/matching_test.cpp index b59384f1493..6ef462f80c4 100644 --- a/searchcore/src/tests/proton/matching/matching_test.cpp +++ b/searchcore/src/tests/proton/matching/matching_test.cpp @@ -1135,12 +1135,12 @@ TEST("require that docsum matcher can extract matching elements from single attr EXPECT_EQUAL(list[1], 3u); } -struct GlobalFilterParamsFixture { +struct AttributeBlueprintParamsFixture { BlueprintFactory factory; search::fef::test::IndexEnvironment index_env; RankSetup rank_setup; Properties rank_properties; - GlobalFilterParamsFixture(double lower_limit, double upper_limit) + AttributeBlueprintParamsFixture(double lower_limit, double upper_limit, double target_hits_max_adjustment_factor) : factory(), index_env(), rank_setup(factory, index_env), @@ -1148,32 +1148,37 @@ struct GlobalFilterParamsFixture { { rank_setup.set_global_filter_lower_limit(lower_limit); rank_setup.set_global_filter_upper_limit(upper_limit); + rank_setup.set_target_hits_max_adjustment_factor(target_hits_max_adjustment_factor); } - void set_query_properties(vespalib::stringref lower_limit, vespalib::stringref upper_limit) { + void set_query_properties(vespalib::stringref lower_limit, vespalib::stringref upper_limit, + vespalib::stringref target_hits_max_adjustment_factor) { rank_properties.add(GlobalFilterLowerLimit::NAME, lower_limit); rank_properties.add(GlobalFilterUpperLimit::NAME, upper_limit); + rank_properties.add(TargetHitsMaxAdjustmentFactor::NAME, target_hits_max_adjustment_factor); } AttributeBlueprintParams extract(uint32_t active_docids = 9, uint32_t docid_limit = 10) const { - return MatchToolsFactory::extract_global_filter_params(rank_setup, rank_properties, active_docids, docid_limit); + return MatchToolsFactory::extract_attribute_blueprint_params(rank_setup, rank_properties, active_docids, docid_limit); } }; -TEST_F("global filter params are extracted from rank profile", GlobalFilterParamsFixture(0.2, 0.8)) +TEST_F("attribute blueprint params are extracted from rank profile", AttributeBlueprintParamsFixture(0.2, 0.8, 5.0)) { auto params = f.extract(); EXPECT_EQUAL(0.2, params.global_filter_lower_limit); EXPECT_EQUAL(0.8, params.global_filter_upper_limit); + EXPECT_EQUAL(5.0, params.target_hits_max_adjustment_factor); } -TEST_F("global filter params are extracted from query", GlobalFilterParamsFixture(0.2, 0.8)) +TEST_F("attribute blueprint params are extracted from query", AttributeBlueprintParamsFixture(0.2, 0.8, 5.0)) { - f.set_query_properties("0.15", "0.75"); + f.set_query_properties("0.15", "0.75", "3.0"); auto params = f.extract(); EXPECT_EQUAL(0.15, params.global_filter_lower_limit); EXPECT_EQUAL(0.75, params.global_filter_upper_limit); + EXPECT_EQUAL(3.0, params.target_hits_max_adjustment_factor); } -TEST_F("global filter params are scaled with active hit ratio", GlobalFilterParamsFixture(0.2, 0.8)) +TEST_F("global filter params are scaled with active hit ratio", AttributeBlueprintParamsFixture(0.2, 0.8, 5.0)) { auto params = f.extract(5, 10); EXPECT_EQUAL(0.12, params.global_filter_lower_limit); diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp b/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp index c7cbdc29689..a353d4816f6 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp @@ -176,11 +176,11 @@ MatchToolsFactory(QueryLimiter & queryLimiter, const search::IDocumentMetaStoreContext::IReadGuard::SP * metaStoreReadGuard, bool is_search) : _queryLimiter(queryLimiter), - _global_filter_params(extract_global_filter_params(rankSetup, rankProperties, metaStore.getNumActiveLids(), searchContext.getDocIdLimit())), + _attribute_blueprint_params(extract_attribute_blueprint_params(rankSetup, rankProperties, metaStore.getNumActiveLids(), searchContext.getDocIdLimit())), _query(), _match_limiter(), _queryEnv(indexEnv, attributeContext, rankProperties, searchContext.getIndexes()), - _requestContext(doom, attributeContext, _queryEnv, _queryEnv.getObjectStore(), _global_filter_params, metaStoreReadGuard), + _requestContext(doom, attributeContext, _queryEnv, _queryEnv.getObjectStore(), _attribute_blueprint_params, metaStoreReadGuard), _mdl(), _rankSetup(rankSetup), _featureOverrides(featureOverrides), @@ -203,8 +203,8 @@ MatchToolsFactory(QueryLimiter & queryLimiter, _query.fetchPostings(); if (is_search) { _query.handle_global_filter(searchContext.getDocIdLimit(), - _global_filter_params.global_filter_lower_limit, - _global_filter_params.global_filter_upper_limit, + _attribute_blueprint_params.global_filter_lower_limit, + _attribute_blueprint_params.global_filter_upper_limit, thread_bundle, trace); } _query.freeze(); @@ -324,18 +324,20 @@ MatchToolsFactory::get_feature_rename_map() const } AttributeBlueprintParams -MatchToolsFactory::extract_global_filter_params(const RankSetup& rank_setup, const Properties& rank_properties, - uint32_t active_docids, uint32_t docid_limit) +MatchToolsFactory::extract_attribute_blueprint_params(const RankSetup& rank_setup, const Properties& rank_properties, + uint32_t active_docids, uint32_t docid_limit) { double lower_limit = GlobalFilterLowerLimit::lookup(rank_properties, rank_setup.get_global_filter_lower_limit()); double upper_limit = GlobalFilterUpperLimit::lookup(rank_properties, rank_setup.get_global_filter_upper_limit()); + double target_hits_max_adjustment_factor = TargetHitsMaxAdjustmentFactor::lookup(rank_properties, rank_setup.get_target_hits_max_adjustment_factor()); // Note that we count the reserved docid 0 as active. // This ensures that when searchable-copies=1, the ratio is 1.0. double active_hit_ratio = std::min(active_docids + 1, docid_limit) / static_cast<double>(docid_limit); return {lower_limit * active_hit_ratio, - upper_limit * active_hit_ratio}; + upper_limit * active_hit_ratio, + target_hits_max_adjustment_factor}; } AttributeOperationTask::AttributeOperationTask(const RequestContext & requestContext, diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_tools.h b/searchcore/src/vespa/searchcore/proton/matching/match_tools.h index db30ea8d2b2..681690d4c36 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_tools.h +++ b/searchcore/src/vespa/searchcore/proton/matching/match_tools.h @@ -121,7 +121,7 @@ private: using IIndexEnvironment = search::fef::IIndexEnvironment; using IDiversifier = search::queryeval::IDiversifier; QueryLimiter & _queryLimiter; - AttributeBlueprintParams _global_filter_params; + AttributeBlueprintParams _attribute_blueprint_params; Query _query; MaybeMatchPhaseLimiter::UP _match_limiter; std::unique_ptr<RangeQueryLocator> _rangeLocator; @@ -177,15 +177,15 @@ public: const StringStringMap & get_feature_rename_map() const; /** - * Extracts global filter parameters from the rank-profile and query. + * Extracts attribute blueprint parameters from the rank-profile and query. * - * These parameters are expected to be in the range [0.0, 1.0], which matches the range of the estimated hit ratio of the query. + * The global filter parameters are expected to be in the range [0.0, 1.0], which matches the range of the estimated hit ratio of the query. * When searchable-copies > 1, we must scale the parameters to match the effective range of the estimated hit ratio. * This is done by multiplying with the active hit ratio (active docids / docid limit). */ static AttributeBlueprintParams - extract_global_filter_params(const RankSetup& rank_setup, const Properties& rank_properties, - uint32_t active_docids, uint32_t docid_limit); + extract_attribute_blueprint_params(const RankSetup& rank_setup, const Properties& rank_properties, + uint32_t active_docids, uint32_t docid_limit); }; } diff --git a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp index 6ca7d298ee2..0475f8462fc 100644 --- a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp +++ b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp @@ -1320,15 +1320,16 @@ public: return *_query_tensor; } - std::unique_ptr<NearestNeighborBlueprint> make_blueprint(bool approximate = true, double global_filter_lower_limit = 0.05) { + std::unique_ptr<NearestNeighborBlueprint> make_blueprint(bool approximate = true, + double global_filter_lower_limit = 0.05, + double target_hits_max_adjustment_factor = 20.0) { search::queryeval::FieldSpec field("foo", 0, 0); auto bp = std::make_unique<NearestNeighborBlueprint>( field, std::make_unique<DistanceCalculator>(this->as_dense_tensor(), create_query_tensor(vec_2d(17, 42))), - 3, approximate, 5, - 100100.25, - global_filter_lower_limit, 1.0, _no_doom.get_doom()); + 3, approximate, 5, 100100.25, + global_filter_lower_limit, 1.0, target_hits_max_adjustment_factor, _no_doom.get_doom()); EXPECT_EQUAL(11u, bp->getState().estimate().estHits); EXPECT_EQUAL(100100.25 * 100100.25, bp->get_distance_threshold()); return bp; @@ -1362,6 +1363,19 @@ TEST_F("NN blueprint handles empty filter (post-filtering)", NearestNeighborBlue EXPECT_EQUAL(NNBA::INDEX_TOP_K, bp->get_algorithm()); } +TEST_F("NN blueprint adjustment of targetHits is bound (post-filtering)", NearestNeighborBlueprintFixture) +{ + auto bp = f.make_blueprint(true, 0.05, 3.5); + auto empty_filter = GlobalFilter::create(); + bp->set_global_filter(*empty_filter, 0.2); + // targetHits is adjusted based on the estimated hit ratio of the query, + // but bound by target-hits-max-adjustment-factor + EXPECT_EQUAL(3u, bp->get_target_hits()); + EXPECT_EQUAL(10u, bp->get_adjusted_target_hits()); + EXPECT_EQUAL(10u, bp->getState().estimate().estHits); + EXPECT_EQUAL(NNBA::INDEX_TOP_K, bp->get_algorithm()); +} + TEST_F("NN blueprint handles strong filter (pre-filtering)", NearestNeighborBlueprintFixture) { auto bp = f.make_blueprint(); diff --git a/searchlib/src/tests/queryeval/nearest_neighbor/nearest_neighbor_test.cpp b/searchlib/src/tests/queryeval/nearest_neighbor/nearest_neighbor_test.cpp index b9599a0c75d..f3545499231 100644 --- a/searchlib/src/tests/queryeval/nearest_neighbor/nearest_neighbor_test.cpp +++ b/searchlib/src/tests/queryeval/nearest_neighbor/nearest_neighbor_test.cpp @@ -126,11 +126,12 @@ SimpleResult find_matches(Fixture &env, const Value &qtv, double threshold = std auto dff = search::tensor::make_distance_function_factory(DistanceMetric::Euclidean, qtv.cells().type); auto df = dff->for_query_vector(qtv.cells()); threshold = df->convert_threshold(threshold); - DistanceCalculator dist_calc(attr, std::move(df)); NearestNeighborDistanceHeap dh(2); dh.set_distance_threshold(threshold); const GlobalFilter &filter = *env._global_filter; - auto search = NearestNeighborIterator::create(strict, tfmd, dist_calc, dh, filter); + auto search = NearestNeighborIterator::create(strict, tfmd, + std::make_unique<DistanceCalculator>(attr, qtv), + dh, filter); if (strict) { return SimpleResult().searchStrict(*search, attr.getNumDocs()); } else { @@ -253,10 +254,11 @@ std::vector<feature_t> get_rawscores(Fixture &env, const Value &qtv) { auto &tfmd = *(md->resolveTermField(0)); auto &attr = *(env._attr); auto dff = search::tensor::make_distance_function_factory(DistanceMetric::Euclidean, qtv.cells().type); - DistanceCalculator dist_calc(attr, dff->for_query_vector(qtv.cells())); NearestNeighborDistanceHeap dh(2); auto dummy_filter = GlobalFilter::create(); - auto search = NearestNeighborIterator::create(strict, tfmd, dist_calc, dh, *dummy_filter); + auto search = NearestNeighborIterator::create(strict, tfmd, + std::make_unique<DistanceCalculator>(attr, qtv), + dh, *dummy_filter); uint32_t limit = attr.getNumDocs(); uint32_t docid = 1; search->initRange(docid, limit); diff --git a/searchlib/src/tests/ranksetup/ranksetup_test.cpp b/searchlib/src/tests/ranksetup/ranksetup_test.cpp index 50d9d36f575..f708df0a862 100644 --- a/searchlib/src/tests/ranksetup/ranksetup_test.cpp +++ b/searchlib/src/tests/ranksetup/ranksetup_test.cpp @@ -533,6 +533,9 @@ void RankSetupTest::testRankSetup() env.getProperties().add(mutate::on_second_phase::Operation::NAME, "=7"); env.getProperties().add(mutate::on_summary::Attribute::NAME, "c"); env.getProperties().add(mutate::on_summary::Operation::NAME, "-=2"); + env.getProperties().add(matching::GlobalFilterLowerLimit::NAME, "0.3"); + env.getProperties().add(matching::GlobalFilterUpperLimit::NAME, "0.7"); + env.getProperties().add(matching::TargetHitsMaxAdjustmentFactor::NAME, "5.0"); RankSetup rs(_factory, env); EXPECT_FALSE(rs.has_match_features()); @@ -571,7 +574,9 @@ void RankSetupTest::testRankSetup() EXPECT_EQUAL(rs.getMutateOnSecondPhase()._operation, "=7"); EXPECT_EQUAL(rs.getMutateOnSummary()._attribute, "c"); EXPECT_EQUAL(rs.getMutateOnSummary()._operation, "-=2"); - + EXPECT_EQUAL(rs.get_global_filter_lower_limit(), 0.3); + EXPECT_EQUAL(rs.get_global_filter_upper_limit(), 0.7); + EXPECT_EQUAL(rs.get_target_hits_max_adjustment_factor(), 5.0); } bool diff --git a/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp b/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp index be631be6dca..453b7b321b9 100644 --- a/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp @@ -842,14 +842,16 @@ public: } try { auto calc = tensor::DistanceCalculator::make_with_validation(_attr, *query_tensor); + const auto& params = getRequestContext().get_attribute_blueprint_params(); setResult(std::make_unique<queryeval::NearestNeighborBlueprint>(_field, std::move(calc), n.get_target_num_hits(), n.get_allow_approximate(), n.get_explore_additional_hits(), n.get_distance_threshold(), - getRequestContext().get_attribute_blueprint_params().global_filter_lower_limit, - getRequestContext().get_attribute_blueprint_params().global_filter_upper_limit, + params.global_filter_lower_limit, + params.global_filter_upper_limit, + params.target_hits_max_adjustment_factor, getRequestContext().getDoom())); } catch (const vespalib::IllegalArgumentException& ex) { return fail_nearest_neighbor_term(n, ex.getMessage()); diff --git a/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_params.h b/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_params.h index 39f58c5382e..64213235c23 100644 --- a/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_params.h +++ b/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_params.h @@ -13,17 +13,21 @@ struct AttributeBlueprintParams { double global_filter_lower_limit; double global_filter_upper_limit; + double target_hits_max_adjustment_factor; AttributeBlueprintParams(double global_filter_lower_limit_in, - double global_filter_upper_limit_in) + double global_filter_upper_limit_in, + double target_hits_max_adjustment_factor_in) : global_filter_lower_limit(global_filter_lower_limit_in), - global_filter_upper_limit(global_filter_upper_limit_in) + global_filter_upper_limit(global_filter_upper_limit_in), + target_hits_max_adjustment_factor(target_hits_max_adjustment_factor_in) { } AttributeBlueprintParams() : AttributeBlueprintParams(fef::indexproperties::matching::GlobalFilterLowerLimit::DEFAULT_VALUE, - fef::indexproperties::matching::GlobalFilterUpperLimit::DEFAULT_VALUE) + fef::indexproperties::matching::GlobalFilterUpperLimit::DEFAULT_VALUE, + fef::indexproperties::matching::TargetHitsMaxAdjustmentFactor::DEFAULT_VALUE) { } }; diff --git a/searchlib/src/vespa/searchlib/fef/indexproperties.cpp b/searchlib/src/vespa/searchlib/fef/indexproperties.cpp index 8be44ce0a0c..7871e66970e 100644 --- a/searchlib/src/vespa/searchlib/fef/indexproperties.cpp +++ b/searchlib/src/vespa/searchlib/fef/indexproperties.cpp @@ -422,6 +422,22 @@ GlobalFilterUpperLimit::lookup(const Properties &props, double defaultValue) return lookupDouble(props, NAME, defaultValue); } +const vespalib::string TargetHitsMaxAdjustmentFactor::NAME("vespa.matching.nns.target_hits_max_adjustment_factor"); + +const double TargetHitsMaxAdjustmentFactor::DEFAULT_VALUE(20.0); + +double +TargetHitsMaxAdjustmentFactor::lookup(const Properties& props) +{ + return lookup(props, DEFAULT_VALUE); +} + +double +TargetHitsMaxAdjustmentFactor::lookup(const Properties& props, double defaultValue) +{ + return lookupDouble(props, NAME, defaultValue); +} + } // namespace matching namespace softtimeout { diff --git a/searchlib/src/vespa/searchlib/fef/indexproperties.h b/searchlib/src/vespa/searchlib/fef/indexproperties.h index f538e7bef2e..4f38a27d3fe 100644 --- a/searchlib/src/vespa/searchlib/fef/indexproperties.h +++ b/searchlib/src/vespa/searchlib/fef/indexproperties.h @@ -313,6 +313,21 @@ namespace matching { static double lookup(const Properties &props); static double lookup(const Properties &props, double defaultValue); }; + + /** + * Property to control the auto-adjustment of targetHits in a nearestNeighbor search using HNSW index with post-filtering. + * + * The targetHits is auto-adjusted in an effort to expose targetHits hits to first-phase ranking after post-filtering: + * adjustedTargetHits = min(targetHits / estimatedHitRatio, targetHits * targetHitsMaxAdjustmentFactor). + * + * This property ensures an upper bound of adjustedTargetHits, avoiding that the search in the HNSW index takes too long. + **/ + struct TargetHitsMaxAdjustmentFactor { + static const vespalib::string NAME; + static const double DEFAULT_VALUE; + static double lookup(const Properties &props); + static double lookup(const Properties &props, double defaultValue); + }; } namespace softtimeout { diff --git a/searchlib/src/vespa/searchlib/fef/ranksetup.cpp b/searchlib/src/vespa/searchlib/fef/ranksetup.cpp index 823e39199df..9d4e547feef 100644 --- a/searchlib/src/vespa/searchlib/fef/ranksetup.cpp +++ b/searchlib/src/vespa/searchlib/fef/ranksetup.cpp @@ -68,6 +68,7 @@ RankSetup::RankSetup(const BlueprintFactory &factory, const IIndexEnvironment &i _softTimeoutTailCost(0.1), _global_filter_lower_limit(0.0), _global_filter_upper_limit(1.0), + _target_hits_max_adjustment_factor(20.0), _mutateOnMatch(), _mutateOnFirstPhase(), _mutateOnSecondPhase(), @@ -121,6 +122,7 @@ RankSetup::configure() setSoftTimeoutTailCost(softtimeout::TailCost::lookup(_indexEnv.getProperties())); set_global_filter_lower_limit(matching::GlobalFilterLowerLimit::lookup(_indexEnv.getProperties())); set_global_filter_upper_limit(matching::GlobalFilterUpperLimit::lookup(_indexEnv.getProperties())); + set_target_hits_max_adjustment_factor(matching::TargetHitsMaxAdjustmentFactor::lookup(_indexEnv.getProperties())); _mutateOnMatch._attribute = mutate::on_match::Attribute::lookup(_indexEnv.getProperties()); _mutateOnMatch._operation = mutate::on_match::Operation::lookup(_indexEnv.getProperties()); _mutateOnFirstPhase._attribute = mutate::on_first_phase::Attribute::lookup(_indexEnv.getProperties()); diff --git a/searchlib/src/vespa/searchlib/fef/ranksetup.h b/searchlib/src/vespa/searchlib/fef/ranksetup.h index 832b86d042a..72432c2ed8a 100644 --- a/searchlib/src/vespa/searchlib/fef/ranksetup.h +++ b/searchlib/src/vespa/searchlib/fef/ranksetup.h @@ -76,6 +76,7 @@ private: double _softTimeoutTailCost; double _global_filter_lower_limit; double _global_filter_upper_limit; + double _target_hits_max_adjustment_factor; MutateOperation _mutateOnMatch; MutateOperation _mutateOnFirstPhase; MutateOperation _mutateOnSecondPhase; @@ -393,6 +394,8 @@ public: double get_global_filter_lower_limit() const { return _global_filter_lower_limit; } void set_global_filter_upper_limit(double v) { _global_filter_upper_limit = v; } double get_global_filter_upper_limit() const { return _global_filter_upper_limit; } + void set_target_hits_max_adjustment_factor(double v) { _target_hits_max_adjustment_factor = v; } + double get_target_hits_max_adjustment_factor() const { return _target_hits_max_adjustment_factor; } /** * This method may be used to indicate that certain features diff --git a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp index 87ddb8b6edc..a70f387100b 100644 --- a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp @@ -43,6 +43,7 @@ NearestNeighborBlueprint::NearestNeighborBlueprint(const queryeval::FieldSpec& f double distance_threshold, double global_filter_lower_limit, double global_filter_upper_limit, + double target_hits_max_adjustment_factor, const vespalib::Doom& doom) : ComplexLeafBlueprint(field), _distance_calc(std::move(distance_calc)), @@ -55,6 +56,7 @@ NearestNeighborBlueprint::NearestNeighborBlueprint(const queryeval::FieldSpec& f _distance_threshold(std::numeric_limits<double>::max()), _global_filter_lower_limit(global_filter_lower_limit), _global_filter_upper_limit(global_filter_upper_limit), + _target_hits_max_adjustment_factor(target_hits_max_adjustment_factor), _distance_heap(target_hits), _found_hits(), _algorithm(Algorithm::EXACT), @@ -95,8 +97,10 @@ NearestNeighborBlueprint::set_global_filter(const GlobalFilter &global_filter, d } else { // post-filtering case // The goal is to expose 'targetHits' hits to first-phase ranking. // We try to achieve this by adjusting targetHits based on the estimated hit ratio of the query before post-filtering. + // However, this is bound by 'target-hits-max-adjustment-factor' to limit the cost of searching the HNSW index. if (estimated_hit_ratio > 0.0) { - _adjusted_target_hits = static_cast<double>(_target_hits) / estimated_hit_ratio; + _adjusted_target_hits = std::min(static_cast<double>(_target_hits) / estimated_hit_ratio, + static_cast<double>(_target_hits) * _target_hits_max_adjustment_factor); } } if (_algorithm != Algorithm::EXACT_FALLBACK) { @@ -133,7 +137,8 @@ NearestNeighborBlueprint::createLeafSearch(const search::fef::TermFieldMatchData default: ; } - return NearestNeighborIterator::create(strict, tfmd, *_distance_calc, + return NearestNeighborIterator::create(strict, tfmd, + std::make_unique<search::tensor::DistanceCalculator>(_attr_tensor, _query_tensor), _distance_heap, *_global_filter); } diff --git a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.h b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.h index f88cdd5adb1..174f0b23125 100644 --- a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.h +++ b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.h @@ -38,6 +38,7 @@ private: double _distance_threshold; double _global_filter_lower_limit; double _global_filter_upper_limit; + double _target_hits_max_adjustment_factor; mutable NearestNeighborDistanceHeap _distance_heap; std::vector<search::tensor::NearestNeighborIndex::Neighbor> _found_hits; Algorithm _algorithm; @@ -55,6 +56,7 @@ public: double distance_threshold, double global_filter_lower_limit, double global_filter_upper_limit, + double target_hits_max_adjustment_factor, const vespalib::Doom& doom); NearestNeighborBlueprint(const NearestNeighborBlueprint&) = delete; NearestNeighborBlueprint& operator=(const NearestNeighborBlueprint&) = delete; diff --git a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.cpp b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.cpp index 92c9a21db83..a71a8e6a49a 100644 --- a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.cpp @@ -23,9 +23,8 @@ template <bool strict, bool has_filter> class NearestNeighborImpl : public NearestNeighborIterator { public: - NearestNeighborImpl(Params params_in) - : NearestNeighborIterator(params_in), + : NearestNeighborIterator(std::move(params_in)), _lastScore(0.0) { } @@ -53,7 +52,7 @@ public: } void doUnpack(uint32_t docId) override { - double score = params().distance_calc.function().to_rawscore(_lastScore); + double score = params().distance_calc->function().to_rawscore(_lastScore); params().tfmd.setRawScore(docId, score); params().distanceHeap.used(_lastScore); } @@ -62,7 +61,7 @@ public: private: double computeDistance(uint32_t docId, double limit) { - return params().distance_calc.calc_with_limit(docId, limit); + return params().distance_calc->calc_with_limit(docId, limit); } double _lastScore; @@ -75,14 +74,14 @@ namespace { template <bool has_filter> std::unique_ptr<NearestNeighborIterator> -resolve_strict(bool strict, const NearestNeighborIterator::Params ¶ms) +resolve_strict(bool strict, NearestNeighborIterator::Params params) { if (strict) { using NNI = NearestNeighborImpl<true, has_filter>; - return std::make_unique<NNI>(params); + return std::make_unique<NNI>(std::move(params)); } else { using NNI = NearestNeighborImpl<false, has_filter>; - return std::make_unique<NNI>(params); + return std::make_unique<NNI>(std::move(params)); } } @@ -92,15 +91,15 @@ std::unique_ptr<NearestNeighborIterator> NearestNeighborIterator::create( bool strict, fef::TermFieldMatchData &tfmd, - const search::tensor::DistanceCalculator &distance_calc, + std::unique_ptr<search::tensor::DistanceCalculator> distance_calc, NearestNeighborDistanceHeap &distanceHeap, const GlobalFilter &filter) { - Params params(tfmd, distance_calc, distanceHeap, filter); + Params params(tfmd, std::move(distance_calc), distanceHeap, filter); if (filter.is_active()) { - return resolve_strict<true>(strict, params); + return resolve_strict<true>(strict, std::move(params)); } else { - return resolve_strict<false>(strict, params); + return resolve_strict<false>(strict, std::move(params)); } } diff --git a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.h b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.h index fe3f8d51d06..884f0f2f3eb 100644 --- a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.h +++ b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.h @@ -24,29 +24,29 @@ public: struct Params { fef::TermFieldMatchData &tfmd; - const search::tensor::DistanceCalculator &distance_calc; + std::unique_ptr<search::tensor::DistanceCalculator> distance_calc; NearestNeighborDistanceHeap &distanceHeap; const GlobalFilter &filter; Params(fef::TermFieldMatchData &tfmd_in, - const search::tensor::DistanceCalculator &distance_calc_in, + std::unique_ptr<search::tensor::DistanceCalculator> distance_calc_in, NearestNeighborDistanceHeap &distanceHeap_in, const GlobalFilter &filter_in) : tfmd(tfmd_in), - distance_calc(distance_calc_in), + distance_calc(std::move(distance_calc_in)), distanceHeap(distanceHeap_in), filter(filter_in) {} }; NearestNeighborIterator(Params params_in) - : _params(params_in) + : _params(std::move(params_in)) {} static std::unique_ptr<NearestNeighborIterator> create( bool strict, fef::TermFieldMatchData &tfmd, - const search::tensor::DistanceCalculator &distance_calc, + std::unique_ptr<search::tensor::DistanceCalculator> distance_calc, NearestNeighborDistanceHeap &distanceHeap, const GlobalFilter &filter); diff --git a/searchlib/src/vespa/searchlib/tensor/distance_calculator.cpp b/searchlib/src/vespa/searchlib/tensor/distance_calculator.cpp index 5759b4b74ea..f65c7103540 100644 --- a/searchlib/src/vespa/searchlib/tensor/distance_calculator.cpp +++ b/searchlib/src/vespa/searchlib/tensor/distance_calculator.cpp @@ -30,14 +30,6 @@ DistanceCalculator::DistanceCalculator(const tensor::ITensorAttribute& attr_tens assert(_dist_fun); } -DistanceCalculator::DistanceCalculator(const tensor::ITensorAttribute& attr_tensor, - BoundDistanceFunction::UP function_in) - : _attr_tensor(attr_tensor), - _query_tensor(nullptr), - _dist_fun(std::move(function_in)) -{ -} - DistanceCalculator::~DistanceCalculator() = default; namespace { diff --git a/searchlib/src/vespa/searchlib/tensor/distance_calculator.h b/searchlib/src/vespa/searchlib/tensor/distance_calculator.h index b65f4ff1868..f44bc0d33cf 100644 --- a/searchlib/src/vespa/searchlib/tensor/distance_calculator.h +++ b/searchlib/src/vespa/searchlib/tensor/distance_calculator.h @@ -29,12 +29,6 @@ public: DistanceCalculator(const tensor::ITensorAttribute& attr_tensor, const vespalib::eval::Value& query_tensor_in); - /** - * Only used by unit tests where ownership of query tensor and distance function is handled outside. - */ - DistanceCalculator(const tensor::ITensorAttribute& attr_tensor, - BoundDistanceFunction::UP function_in); - ~DistanceCalculator(); const tensor::ITensorAttribute& attribute_tensor() const { return _attr_tensor; } diff --git a/storage/src/tests/distributor/btree_bucket_database_test.cpp b/storage/src/tests/distributor/btree_bucket_database_test.cpp index 14d5a4142a8..40575cacfba 100644 --- a/storage/src/tests/distributor/btree_bucket_database_test.cpp +++ b/storage/src/tests/distributor/btree_bucket_database_test.cpp @@ -19,15 +19,15 @@ using document::BucketId; namespace { -BucketCopy BC(uint32_t node_idx, uint32_t state) { +BucketCopy BC(uint16_t node_idx, uint32_t state) { api::BucketInfo info(0x123, state, state); - return BucketCopy(0, node_idx, info); + return {0, node_idx, info}; } BucketInfo BI(uint32_t node_idx, uint32_t state) { BucketInfo bi; - bi.addNode(BC(node_idx, state), toVector<uint16_t>(0)); + bi.addNode(BC(node_idx, state), {0}); return bi; } diff --git a/storage/src/tests/distributor/bucketdatabasetest.cpp b/storage/src/tests/distributor/bucketdatabasetest.cpp index fcc64e0cccf..032b8ad8a9c 100644 --- a/storage/src/tests/distributor/bucketdatabasetest.cpp +++ b/storage/src/tests/distributor/bucketdatabasetest.cpp @@ -1,9 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "bucketdatabasetest.h" +#include <vespa/storage/storageutil/utils.h> #include <vespa/vespalib/util/benchmark_timer.h> #include <chrono> -#include <iomanip> #include <algorithm> namespace storage::distributor { @@ -16,21 +16,21 @@ void BucketDatabaseTest::SetUp() { namespace { -BucketCopy BC(uint32_t nodeIdx) { +BucketCopy BC(uint16_t nodeIdx) { return BucketCopy(0, nodeIdx, api::BucketInfo()); } -BucketInfo BI(uint32_t nodeIdx) { +BucketInfo BI(uint16_t nodeIdx) { BucketInfo bi; - bi.addNode(BC(nodeIdx), toVector<uint16_t>(0)); + bi.addNode(BC(nodeIdx), {0}); return bi; } -BucketInfo BI3(uint32_t node0, uint32_t node1, uint32_t node2) { +BucketInfo BI3(uint16_t node0, uint16_t node1, uint16_t node2) { BucketInfo bi; - bi.addNode(BC(node0), toVector<uint16_t>(node0, node1, node2)); - bi.addNode(BC(node1), toVector<uint16_t>(node0, node1, node2)); - bi.addNode(BC(node2), toVector<uint16_t>(node0, node1, node2)); + bi.addNode(BC(node0), {node0, node1, node2}); + bi.addNode(BC(node1), {node0, node1, node2}); + bi.addNode(BC(node2), {node0, node1, node2}); return bi; } diff --git a/storage/src/tests/distributor/bucketdatabasetest.h b/storage/src/tests/distributor/bucketdatabasetest.h index 33f914f8fd2..f24a62728d3 100644 --- a/storage/src/tests/distributor/bucketdatabasetest.h +++ b/storage/src/tests/distributor/bucketdatabasetest.h @@ -2,7 +2,6 @@ #pragma once #include <vespa/storage/bucketdb/bucketdatabase.h> -#include <vespa/storage/storageutil/utils.h> #include <vespa/vespalib/gtest/gtest.h> #include <functional> @@ -11,19 +10,14 @@ namespace storage::distributor { struct BucketDatabaseTest : public ::testing::TestWithParam<std::shared_ptr<BucketDatabase>> { void SetUp() override ; - std::string doFindParents(const std::vector<document::BucketId>& ids, - const document::BucketId& searchId); - std::string doFindAll(const std::vector<document::BucketId>& ids, - const document::BucketId& searchId); + std::string doFindParents(const std::vector<document::BucketId>& ids, const document::BucketId& searchId); + std::string doFindAll(const std::vector<document::BucketId>& ids, const document::BucketId& searchId); document::BucketId doCreate(const std::vector<document::BucketId>& ids, - uint32_t minBits, - const document::BucketId& wantedId); + uint32_t minBits, const document::BucketId& wantedId); BucketDatabase& db() noexcept { return *GetParam(); } - using UBoundFunc = std::function< - document::BucketId(const BucketDatabase&, - const document::BucketId&)>; + using UBoundFunc = std::function<document::BucketId(const BucketDatabase&, const document::BucketId&)>; void doTestUpperBound(const UBoundFunc& f); }; diff --git a/storage/src/tests/distributor/bucketstateoperationtest.cpp b/storage/src/tests/distributor/bucketstateoperationtest.cpp index 42ee4675e26..c9fab0b37e5 100644 --- a/storage/src/tests/distributor/bucketstateoperationtest.cpp +++ b/storage/src/tests/distributor/bucketstateoperationtest.cpp @@ -3,6 +3,7 @@ #include <tests/distributor/distributor_stripe_test_util.h> #include <vespa/storage/distributor/operations/idealstate/setbucketstateoperation.h> #include <vespa/storage/distributor/top_level_distributor.h> +#include <vespa/storage/storageutil/utils.h> #include <vespa/document/test/make_document_bucket.h> #include <vespa/vespalib/gtest/gtest.h> #include "dummy_cluster_context.h" diff --git a/storage/src/tests/distributor/distributor_bucket_space_test.cpp b/storage/src/tests/distributor/distributor_bucket_space_test.cpp index 3ea4c1ca3c2..00bc803e81c 100644 --- a/storage/src/tests/distributor/distributor_bucket_space_test.cpp +++ b/storage/src/tests/distributor/distributor_bucket_space_test.cpp @@ -100,10 +100,10 @@ DistributorBucketSpaceTest::CountVector DistributorBucketSpaceTest::count_service_layer_buckets(const std::vector<BucketId>& buckets) { CountVector result(3); - std::vector<uint16_t> ideal_nodes; for (auto& bucket : buckets) { const auto & ideal_nodes_bundle = bucket_space.get_ideal_service_layer_nodes_bundle(bucket); for (uint32_t i = 0; i < 3; ++i) { + IdealServiceLayerNodesBundle::ConstNodesRef ideal_nodes; switch (i) { case 0: ideal_nodes = ideal_nodes_bundle.available_nodes(); diff --git a/storage/src/tests/distributor/distributor_stripe_test_util.cpp b/storage/src/tests/distributor/distributor_stripe_test_util.cpp index 6ececa39583..5babde49380 100644 --- a/storage/src/tests/distributor/distributor_stripe_test_util.cpp +++ b/storage/src/tests/distributor/distributor_stripe_test_util.cpp @@ -10,6 +10,7 @@ #include <vespa/storage/distributor/distributormetricsset.h> #include <vespa/storage/distributor/ideal_state_total_metrics.h> #include <vespa/storage/distributor/node_supported_features_repo.h> +#include <vespa/storage/storageutil/utils.h> #include <vespa/vdslib/distribution/distribution.h> #include <vespa/vespalib/text/stringtokenizer.h> #include <vespa/vespalib/stllike/hash_map.hpp> @@ -225,7 +226,7 @@ DistributorStripeTestUtil::addIdealNodes(const lib::ClusterState& state, const d for (uint32_t i = 0; i < res.size(); ++i) { if (state.getNodeState(lib::Node(lib::NodeType::STORAGE, res[i])).getState() != lib::State::MAINTENANCE) { - entry->addNode(BucketCopy(0, res[i], api::BucketInfo(1,1,1)), toVector<uint16_t>(0)); + entry->addNode(BucketCopy(0, res[i], api::BucketInfo(1,1,1)), {0}); } } @@ -324,7 +325,7 @@ DistributorStripeTestUtil::insertBucketInfo(document::BucketId id, uint16_t node info2.setActive(); } BucketCopy copy(operation_context().generate_unique_timestamp(), node, info2); - entry->addNode(copy.setTrusted(trusted), toVector<uint16_t>(0)); + entry->addNode(copy.setTrusted(trusted), {0}); getBucketDatabase().update(entry); } diff --git a/storage/src/tests/distributor/distributor_stripe_test_util.h b/storage/src/tests/distributor/distributor_stripe_test_util.h index 9963b2c96b4..272301bf4a6 100644 --- a/storage/src/tests/distributor/distributor_stripe_test_util.h +++ b/storage/src/tests/distributor/distributor_stripe_test_util.h @@ -7,6 +7,7 @@ #include <tests/common/teststorageapp.h> #include <vespa/storage/common/hostreporter/hostinfo.h> #include <vespa/storage/distributor/stripe_host_info_notifier.h> +#include <vespa/storage/storageutil/utils.h> namespace storage { diff --git a/storage/src/tests/distributor/garbagecollectiontest.cpp b/storage/src/tests/distributor/garbagecollectiontest.cpp index c2f4836f4cb..9b5056f2066 100644 --- a/storage/src/tests/distributor/garbagecollectiontest.cpp +++ b/storage/src/tests/distributor/garbagecollectiontest.cpp @@ -71,8 +71,7 @@ struct GarbageCollectionOperationTest : Test, DistributorStripeTestUtil { std::shared_ptr<GarbageCollectionOperation> create_op() { auto op = std::make_shared<GarbageCollectionOperation>( - dummy_cluster_context, BucketAndNodes(makeDocumentBucket(_bucket_id), - toVector<uint16_t>(0, 1))); + dummy_cluster_context, BucketAndNodes(makeDocumentBucket(_bucket_id), {0, 1})); op->setIdealStateManager(&getIdealStateManager()); return op; } diff --git a/storage/src/tests/distributor/operationtargetresolvertest.cpp b/storage/src/tests/distributor/operationtargetresolvertest.cpp index 2d41b0f4d32..19ca81e933f 100644 --- a/storage/src/tests/distributor/operationtargetresolvertest.cpp +++ b/storage/src/tests/distributor/operationtargetresolvertest.cpp @@ -3,7 +3,6 @@ #include <tests/distributor/distributor_stripe_test_util.h> #include <vespa/config/helper/configgetter.h> #include <vespa/config/helper/configgetter.hpp> -#include <vespa/document/config/config-documenttypes.h> #include <vespa/document/repo/documenttyperepo.h> #include <vespa/document/test/make_bucket_space.h> #include <vespa/document/test/make_document_bucket.h> @@ -14,7 +13,6 @@ #include <vespa/storageapi/message/bucket.h> #include <vespa/storageapi/message/persistence.h> #include <vespa/vdslib/distribution/distribution.h> -#include <vespa/vdslib/distribution/idealnodecalculatorimpl.h> #include <vespa/vespalib/gtest/gtest.h> using document::BucketId; @@ -112,14 +110,10 @@ struct TestTargets { } // anonymous BucketInstanceList -OperationTargetResolverTest::getInstances(const BucketId& id, - bool stripToRedundancy) +OperationTargetResolverTest::getInstances(const BucketId& id, bool stripToRedundancy) { - lib::IdealNodeCalculatorImpl idealNodeCalc; auto &bucketSpaceRepo(operation_context().bucket_space_repo()); auto &distributorBucketSpace(bucketSpaceRepo.get(makeBucketSpace())); - idealNodeCalc.setDistribution(distributorBucketSpace.getDistribution()); - idealNodeCalc.setClusterState(distributorBucketSpace.getClusterState()); OperationTargetResolverImpl resolver( distributorBucketSpace, distributorBucketSpace.getBucketDatabase(), 16, distributorBucketSpace.getDistribution().getRedundancy(), @@ -142,24 +136,6 @@ TEST_F(OperationTargetResolverTest, simple) { .sendsTo(BucketId(16, 0), 0); } -TEST_F(OperationTargetResolverTest, multiple_nodes) { - setup_stripe(1, 2, "storage:2 distributor:1"); - - auto &bucketSpaceRepo(operation_context().bucket_space_repo()); - auto &distributorBucketSpace(bucketSpaceRepo.get(makeBucketSpace())); - for (int i = 0; i < 100; ++i) { - addNodesToBucketDB(BucketId(16, i), "0=0,1=0"); - - lib::IdealNodeCalculatorImpl idealNodeCalc; - idealNodeCalc.setDistribution(distributorBucketSpace.getDistribution()); - idealNodeCalc.setClusterState(distributorBucketSpace.getClusterState()); - lib::IdealNodeList idealNodes( - idealNodeCalc.getIdealStorageNodes(BucketId(16, i))); - uint16_t expectedNode = idealNodes[0].getIndex(); - MY_ASSERT_THAT(BucketId(32, i)).sendsTo(BucketId(16, i), expectedNode); - } -} - TEST_F(OperationTargetResolverTest, choose_ideal_state_when_many_copies) { setup_stripe(2, 4, "storage:4 distributor:1"); addNodesToBucketDB(BucketId(16, 0), "0=0,1=0,2=0,3=0"); // ideal nodes: 1, 3 diff --git a/storage/src/tests/distributor/simplemaintenancescannertest.cpp b/storage/src/tests/distributor/simplemaintenancescannertest.cpp index b5dc72d995b..3d3c58ba842 100644 --- a/storage/src/tests/distributor/simplemaintenancescannertest.cpp +++ b/storage/src/tests/distributor/simplemaintenancescannertest.cpp @@ -82,7 +82,7 @@ TEST_F(SimpleMaintenanceScannerTest, prioritize_single_bucket) { TEST_F(SimpleMaintenanceScannerTest, prioritize_single_bucket_alt_bucket_space) { document::BucketSpace bucketSpace(4); _bucketSpaceRepo->add(bucketSpace, std::make_unique<DistributorBucketSpace>()); - _scanner->reset(); + (void)_scanner->fetch_and_reset(); addBucketToDb(bucketSpace, 1); std::string expected("PrioritizedBucket(Bucket(BucketSpace(0x0000000000000004), BucketId(0x4000000000000001)), pri VERY_HIGH)\n"); @@ -148,7 +148,7 @@ TEST_F(SimpleMaintenanceScannerTest, reset) { ASSERT_TRUE(scanEntireDatabase(0)); EXPECT_EQ(expected, _priorityDb->toString()); - _scanner->reset(); + (void)_scanner->fetch_and_reset(); ASSERT_TRUE(scanEntireDatabase(3)); expected = "PrioritizedBucket(Bucket(BucketSpace(0x0000000000000001), BucketId(0x4000000000000001)), pri VERY_HIGH)\n" @@ -180,7 +180,7 @@ TEST_F(SimpleMaintenanceScannerTest, pending_maintenance_operation_statistics) { EXPECT_EQ(expected, stringifyGlobalPendingStats(stats)); } - _scanner->reset(); + (void)_scanner->fetch_and_reset(); { const auto & stats = _scanner->getPendingMaintenanceStats(); EXPECT_EQ(expectedEmpty, stringifyGlobalPendingStats(stats)); @@ -301,7 +301,7 @@ TEST_F(SimpleMaintenanceScannerTest, merge_pending_maintenance_stats) { TEST_F(SimpleMaintenanceScannerTest, empty_bucket_db_is_immediately_done_by_default) { auto res = _scanner->scanNext(); EXPECT_TRUE(res.isDone()); - _scanner->reset(); + (void)_scanner->fetch_and_reset(); res = _scanner->scanNext(); EXPECT_TRUE(res.isDone()); } diff --git a/storage/src/tests/distributor/statecheckerstest.cpp b/storage/src/tests/distributor/statecheckerstest.cpp index 4ca4d70a816..13c982f5a77 100644 --- a/storage/src/tests/distributor/statecheckerstest.cpp +++ b/storage/src/tests/distributor/statecheckerstest.cpp @@ -7,6 +7,7 @@ #include <vespa/document/test/make_document_bucket.h> #include <vespa/storage/distributor/top_level_bucket_db_updater.h> #include <vespa/storage/distributor/top_level_distributor.h> +#include <vespa/storage/distributor/activecopy.h> #include <vespa/storage/distributor/distributor_bucket_space.h> #include <vespa/storage/distributor/distributor_stripe.h> #include <vespa/storage/distributor/operations/idealstate/mergeoperation.h> @@ -1587,7 +1588,9 @@ TEST_F(StateCheckersTest, context_populates_ideal_state_containers) { StateChecker::Context c(node_context(), operation_context(), getDistributorBucketSpace(), statsTracker, makeDocumentBucket({17, 0})); - ASSERT_THAT(c.idealState(), ElementsAre(1, 3)); + ASSERT_EQ(2, c.idealState().size()); + ASSERT_EQ(1, c.idealState()[0]); + ASSERT_EQ(3, c.idealState()[1]); for (uint16_t node : c.idealState()) { ASSERT_TRUE(c.idealStateBundle.is_nonretired_or_maintenance(node)); } @@ -1736,4 +1739,9 @@ TEST_F(StateCheckersTest, stats_updates_for_maximum_time_since_gc_run) { EXPECT_EQ(runner.stats().max_observed_time_since_last_gc(), 1900s); } +TEST(ActiveCopyTest, control_size) { + EXPECT_EQ(12, sizeof(ActiveCopy)); + EXPECT_EQ(64, sizeof(IdealServiceLayerNodesBundle)); +} + } diff --git a/storage/src/tests/distributor/top_level_distributor_test_util.cpp b/storage/src/tests/distributor/top_level_distributor_test_util.cpp index 9859a6fb237..6bbe7a47da2 100644 --- a/storage/src/tests/distributor/top_level_distributor_test_util.cpp +++ b/storage/src/tests/distributor/top_level_distributor_test_util.cpp @@ -10,6 +10,7 @@ #include <vespa/storage/distributor/distributor_stripe_pool.h> #include <vespa/storage/distributor/distributor_stripe_thread.h> #include <vespa/storage/distributor/distributor_total_metrics.h> +#include <vespa/storage/storageutil/utils.h> #include <vespa/storage/common/bucket_stripe_utils.h> #include <vespa/vdslib/distribution/distribution.h> #include <vespa/vespalib/text/stringtokenizer.h> diff --git a/storage/src/tests/distributor/top_level_distributor_test_util.h b/storage/src/tests/distributor/top_level_distributor_test_util.h index cd5db7c8f80..51700848733 100644 --- a/storage/src/tests/distributor/top_level_distributor_test_util.h +++ b/storage/src/tests/distributor/top_level_distributor_test_util.h @@ -7,7 +7,6 @@ #include <tests/common/teststorageapp.h> #include <vespa/storage/common/hostreporter/hostinfo.h> #include <vespa/storage/frameworkimpl/component/distributorcomponentregisterimpl.h> -#include <vespa/storage/storageutil/utils.h> #include <vespa/storageapi/message/state.h> #include <vespa/storageframework/defaultimplementation/clock/fakeclock.h> diff --git a/storage/src/vespa/storage/common/distributorcomponent.h b/storage/src/vespa/storage/common/distributorcomponent.h index 06bb49a6090..6542bf2ddfe 100644 --- a/storage/src/vespa/storage/common/distributorcomponent.h +++ b/storage/src/vespa/storage/common/distributorcomponent.h @@ -34,13 +34,6 @@ namespace storage { -namespace bucketdb { - class DistrBucketDatabase; -} -namespace lib { - class IdealNodeCalculator; -} - using DistributorConfig = vespa::config::content::core::internal::InternalStorDistributormanagerType; using VisitorConfig = vespa::config::content::core::internal::InternalStorVisitordispatcherType; diff --git a/storage/src/vespa/storage/distributor/activecopy.cpp b/storage/src/vespa/storage/distributor/activecopy.cpp index 5d59d1a838f..4e3ef4f88ee 100644 --- a/storage/src/vespa/storage/distributor/activecopy.cpp +++ b/storage/src/vespa/storage/distributor/activecopy.cpp @@ -1,8 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "activecopy.h" - -#include <vespa/storage/storageutil/utils.h> #include <vespa/vdslib/distribution/distribution.h> #include <vespa/vespalib/stllike/asciistream.h> #include <algorithm> @@ -28,26 +26,9 @@ namespace storage::distributor { using IndexList = lib::Distribution::IndexList; -ActiveCopy::ActiveCopy(uint16_t node, const BucketDatabase::Entry& e, const std::vector<uint16_t>& idealState) - : _nodeIndex(node), - _ideal(0xffff) -{ - const BucketCopy* copy = e->getNode(node); - assert(copy != nullptr); - _doc_count = copy->getDocumentCount(); - _ready = copy->ready(); - _active = copy->active(); - for (uint32_t i=0; i<idealState.size(); ++i) { - if (idealState[i] == node) { - _ideal = i; - break; - } - } -} - vespalib::string ActiveCopy::getReason() const { - if (_ready && (_doc_count > 0) && (_ideal < 0xffff)) { + if (_ready && (_doc_count > 0) && valid_ideal()) { vespalib::asciistream ost; ost << "copy is ready, has " << _doc_count << " docs and ideal state priority " << _ideal; @@ -58,7 +39,7 @@ ActiveCopy::getReason() const { return ost.str(); } else if (_ready) { return "copy is ready"; - } else if ((_doc_count > 0) && (_ideal < 0xffff)) { + } else if ((_doc_count > 0) && valid_ideal()) { vespalib::asciistream ost; ost << "copy has " << _doc_count << " docs and ideal state priority " << _ideal; return ost.str(); @@ -68,7 +49,7 @@ ActiveCopy::getReason() const { return ost.str(); } else if (_active) { return "copy is already active"; - } else if (_ideal < 0xffff) { + } else if (valid_ideal()) { vespalib::asciistream ost; ost << "copy is ideal state priority " << _ideal; return ost.str(); @@ -86,7 +67,7 @@ operator<<(std::ostream& out, const ActiveCopy & e) { if (e._doc_count > 0) { out << ", doc_count " << e._doc_count; } - if (e._ideal < 0xffff) { + if (e.valid_ideal()) { out << ", ideal pri " << e._ideal; } out << ")"; @@ -95,26 +76,8 @@ operator<<(std::ostream& out, const ActiveCopy & e) { namespace { -struct ActiveStateOrder { - bool operator()(const ActiveCopy & e1, const ActiveCopy & e2) noexcept { - if (e1._ready != e2._ready) { - return e1._ready; - } - if (e1._doc_count != e2._doc_count) { - return e1._doc_count > e2._doc_count; - } - if (e1._ideal != e2._ideal) { - return e1._ideal < e2._ideal; - } - if (e1._active != e2._active) { - return e1._active; - } - return e1._nodeIndex < e2._nodeIndex; - } -}; - IndexList -buildValidNodeIndexList(BucketDatabase::Entry& e) { +buildValidNodeIndexList(const BucketDatabase::Entry& e) { IndexList result; result.reserve(e->getNodeCount()); for (uint32_t i=0, n=e->getNodeCount(); i < n; ++i) { @@ -126,22 +89,45 @@ buildValidNodeIndexList(BucketDatabase::Entry& e) { return result; } -std::vector<ActiveCopy> -buildNodeList(BucketDatabase::Entry& e,vespalib::ConstArrayRef<uint16_t> nodeIndexes, const std::vector<uint16_t>& idealState) +using SmallActiveCopyList = vespalib::SmallVector<ActiveCopy, 2>; +static_assert(sizeof(SmallActiveCopyList) == 40); + +SmallActiveCopyList +buildNodeList(const BucketDatabase::Entry& e,vespalib::ConstArrayRef<uint16_t> nodeIndexes, const IdealServiceLayerNodesBundle::Node2Index & idealState) { - std::vector<ActiveCopy> result; + SmallActiveCopyList result; result.reserve(nodeIndexes.size()); for (uint16_t nodeIndex : nodeIndexes) { - result.emplace_back(nodeIndex, e, idealState); + const BucketCopy *copy = e->getNode(nodeIndex); + assert(copy); + result.emplace_back(nodeIndex, *copy, idealState.lookup(nodeIndex)); } return result; } } +struct ActiveStateOrder { + bool operator()(const ActiveCopy & e1, const ActiveCopy & e2) noexcept { + if (e1._ready != e2._ready) { + return e1._ready; + } + if (e1._doc_count != e2._doc_count) { + return e1._doc_count > e2._doc_count; + } + if (e1._ideal != e2._ideal) { + return e1._ideal < e2._ideal; + } + if (e1._active != e2._active) { + return e1._active; + } + return e1.nodeIndex() < e2.nodeIndex(); + } +}; + ActiveList -ActiveCopy::calculate(const std::vector<uint16_t>& idealState, const lib::Distribution& distribution, - BucketDatabase::Entry& e, uint32_t max_activation_inhibited_out_of_sync_groups) +ActiveCopy::calculate(const Node2Index & idealState, const lib::Distribution& distribution, + const BucketDatabase::Entry& e, uint32_t max_activation_inhibited_out_of_sync_groups) { IndexList validNodesWithCopy = buildValidNodeIndexList(e); if (validNodesWithCopy.empty()) { @@ -161,7 +147,7 @@ ActiveCopy::calculate(const std::vector<uint16_t>& idealState, const lib::Distri : api::BucketInfo()); // Invalid by default uint32_t inhibited_groups = 0; for (const auto& group_nodes : groups) { - std::vector<ActiveCopy> entries = buildNodeList(e, group_nodes, idealState); + SmallActiveCopyList entries = buildNodeList(e, group_nodes, idealState); auto best = std::min_element(entries.begin(), entries.end(), ActiveStateOrder()); if ((groups.size() > 1) && (inhibited_groups < max_activation_inhibited_out_of_sync_groups) && @@ -179,24 +165,22 @@ ActiveCopy::calculate(const std::vector<uint16_t>& idealState, const lib::Distri } void -ActiveList::print(std::ostream& out, bool verbose, - const std::string& indent) const +ActiveList::print(std::ostream& out, bool verbose, const std::string& indent) const { out << "["; if (verbose) { for (size_t i=0; i<_v.size(); ++i) { - out << "\n" << indent << " " - << _v[i]._nodeIndex << " " << _v[i].getReason(); + out << "\n" << indent << " " << _v[i].nodeIndex() << " " << _v[i].getReason(); } if (!_v.empty()) { out << "\n" << indent; } } else { if (!_v.empty()) { - out << _v[0]._nodeIndex; + out << _v[0].nodeIndex(); } for (size_t i=1; i<_v.size(); ++i) { - out << " " << _v[i]._nodeIndex; + out << " " << _v[i].nodeIndex(); } } out << "]"; @@ -206,7 +190,7 @@ bool ActiveList::contains(uint16_t node) const noexcept { for (const auto& candidate : _v) { - if (node == candidate._nodeIndex) { + if (node == candidate.nodeIndex()) { return true; } } diff --git a/storage/src/vespa/storage/distributor/activecopy.h b/storage/src/vespa/storage/distributor/activecopy.h index 258fe3cdf16..a2de77306be 100644 --- a/storage/src/vespa/storage/distributor/activecopy.h +++ b/storage/src/vespa/storage/distributor/activecopy.h @@ -2,25 +2,43 @@ #pragma once +#include "ideal_service_layer_nodes_bundle.h" #include <vespa/storage/bucketdb/bucketdatabase.h> namespace storage::lib { class Distribution; } namespace storage::distributor { class ActiveList; +struct ActiveStateOrder; -struct ActiveCopy { - constexpr ActiveCopy() noexcept : _nodeIndex(-1), _ideal(-1), _doc_count(0), _ready(false), _active(false) { } - ActiveCopy(uint16_t node, const BucketDatabase::Entry& e, const std::vector<uint16_t>& idealState); +class ActiveCopy { + using Index = IdealServiceLayerNodesBundle::Index; + using Node2Index = IdealServiceLayerNodesBundle::Node2Index; +public: + constexpr ActiveCopy() noexcept + : _nodeIndex(Index::invalid()), + _ideal(Index::invalid()), + _doc_count(0), + _ready(false), + _active(false) + { } + ActiveCopy(uint16_t node, const BucketCopy & copy, uint16_t ideal) noexcept + : _nodeIndex(node), + _ideal(ideal), + _doc_count(copy.getDocumentCount()), + _ready(copy.ready()), + _active(copy.active()) + { } vespalib::string getReason() const; friend std::ostream& operator<<(std::ostream& out, const ActiveCopy& e); - static ActiveList calculate(const std::vector<uint16_t>& idealState, - const lib::Distribution&, - BucketDatabase::Entry&, - uint32_t max_activation_inhibited_out_of_sync_groups); - + static ActiveList calculate(const Node2Index & idealState, const lib::Distribution&, + const BucketDatabase::Entry&, uint32_t max_activation_inhibited_out_of_sync_groups); + uint16_t nodeIndex() const noexcept { return _nodeIndex; } +private: + friend ActiveStateOrder; + bool valid_ideal() const noexcept { return _ideal < Index::invalid(); } uint16_t _nodeIndex; uint16_t _ideal; uint32_t _doc_count; @@ -29,8 +47,6 @@ struct ActiveCopy { }; class ActiveList : public vespalib::Printable { - std::vector<ActiveCopy> _v; - public: ActiveList() {} ActiveList(std::vector<ActiveCopy>&& v) : _v(std::move(v)) { } @@ -41,6 +57,8 @@ public: [[nodiscard]] bool empty() const noexcept { return _v.empty(); } size_t size() const noexcept { return _v.size(); } void print(std::ostream&, bool verbose, const std::string& indent) const override; +private: + std::vector<ActiveCopy> _v; }; } diff --git a/storage/src/vespa/storage/distributor/distributor_bucket_space.cpp b/storage/src/vespa/storage/distributor/distributor_bucket_space.cpp index 299aaffb569..7ba9c67b156 100644 --- a/storage/src/vespa/storage/distributor/distributor_bucket_space.cpp +++ b/storage/src/vespa/storage/distributor/distributor_bucket_space.cpp @@ -121,9 +121,9 @@ setup_ideal_nodes_bundle(IdealServiceLayerNodesBundle& ideal_nodes_bundle, const lib::ClusterState& cluster_state, document::BucketId bucket) { - ideal_nodes_bundle.set_available_nodes(distribution.getIdealStorageNodes(cluster_state, bucket, up_states)); - ideal_nodes_bundle.set_available_nonretired_nodes(distribution.getIdealStorageNodes(cluster_state, bucket, nonretired_up_states)); - ideal_nodes_bundle.set_available_nonretired_or_maintenance_nodes(distribution.getIdealStorageNodes(cluster_state, bucket, nonretired_or_maintenance_up_states)); + ideal_nodes_bundle.set_nodes(distribution.getIdealStorageNodes(cluster_state, bucket, up_states), + distribution.getIdealStorageNodes(cluster_state, bucket, nonretired_up_states), + distribution.getIdealStorageNodes(cluster_state, bucket, nonretired_or_maintenance_up_states)); } /* diff --git a/storage/src/vespa/storage/distributor/distributor_stripe.cpp b/storage/src/vespa/storage/distributor/distributor_stripe.cpp index 37d81f45ac1..b686c6bc80c 100644 --- a/storage/src/vespa/storage/distributor/distributor_stripe.cpp +++ b/storage/src/vespa/storage/distributor/distributor_stripe.cpp @@ -314,7 +314,7 @@ DistributorStripe::enterRecoveryMode() { LOG(debug, "Entering recovery mode"); _schedulingMode = MaintenanceScheduler::RECOVERY_SCHEDULING_MODE; - _scanner->reset(); // Just drop accumulated stat on the floor. + (void)_scanner->fetch_and_reset(); // Just drop accumulated stats on the floor. // We enter recovery mode due to cluster state or distribution config changes. // Until we have completed a new DB scan round, we don't know the state of our // newly owned buckets and must not report stats for these out to the cluster @@ -643,7 +643,7 @@ DistributorStripe::updateInternalMetricsForCompletedScan() _bucketDBMetricUpdater.completeRound(); _bucketDbStats = _bucketDBMetricUpdater.getLastCompleteStats(); - _maintenanceStats = _scanner->reset(); + _maintenanceStats = _scanner->fetch_and_reset(); auto new_space_stats = toBucketSpacesStats(_maintenanceStats.perNodeStats); if (merge_no_longer_pending_edge(_bucketSpacesStats, new_space_stats)) { _must_send_updated_host_info = true; diff --git a/storage/src/vespa/storage/distributor/distributor_stripe_component.cpp b/storage/src/vespa/storage/distributor/distributor_stripe_component.cpp index 16cc887096f..47b89b2dd19 100644 --- a/storage/src/vespa/storage/distributor/distributor_stripe_component.cpp +++ b/storage/src/vespa/storage/distributor/distributor_stripe_component.cpp @@ -5,6 +5,7 @@ #include "distributor_bucket_space.h" #include "pendingmessagetracker.h" #include "storage_node_up_states.h" +#include <vespa/storage/storageutil/utils.h> #include <vespa/storageframework/generic/clock/clock.h> #include <vespa/document/select/parser.h> #include <vespa/vdslib/state/cluster_state_bundle.h> @@ -53,18 +54,19 @@ class UpdateBucketDatabaseProcessor : public BucketDatabase::EntryUpdateProcesso const std::vector<BucketCopy>& _changed_nodes; std::vector<uint16_t> _ideal_nodes; bool _reset_trusted; + using ConstNodesRef = IdealServiceLayerNodesBundle::ConstNodesRef; public: - UpdateBucketDatabaseProcessor(const framework::Clock& clock, const std::vector<BucketCopy>& changed_nodes, std::vector<uint16_t> ideal_nodes, bool reset_trusted); + UpdateBucketDatabaseProcessor(const framework::Clock& clock, const std::vector<BucketCopy>& changed_nodes, ConstNodesRef ideal_nodes, bool reset_trusted); ~UpdateBucketDatabaseProcessor() override; BucketDatabase::Entry create_entry(const document::BucketId& bucket) const override; bool process_entry(BucketDatabase::Entry &entry) const override; }; -UpdateBucketDatabaseProcessor::UpdateBucketDatabaseProcessor(const framework::Clock& clock, const std::vector<BucketCopy>& changed_nodes, std::vector<uint16_t> ideal_nodes, bool reset_trusted) +UpdateBucketDatabaseProcessor::UpdateBucketDatabaseProcessor(const framework::Clock& clock, const std::vector<BucketCopy>& changed_nodes, ConstNodesRef ideal_nodes, bool reset_trusted) : BucketDatabase::EntryUpdateProcessor(), _clock(clock), _changed_nodes(changed_nodes), - _ideal_nodes(std::move(ideal_nodes)), + _ideal_nodes(ideal_nodes.cbegin(), ideal_nodes.cend()), _reset_trusted(reset_trusted) { } @@ -244,4 +246,14 @@ DistributorStripeComponent::parse_selection(const vespalib::string& selection) c return parser.parse(selection); } +void +DistributorStripeComponent::update_bucket_database(const document::Bucket& bucket, const BucketCopy& changed_node, uint32_t update_flags) { + update_bucket_database(bucket, toVector<BucketCopy>(changed_node),update_flags); +} + +void +DistributorStripeComponent::remove_node_from_bucket_database(const document::Bucket& bucket, uint16_t node_index) { + remove_nodes_from_bucket_database(bucket, toVector<uint16_t>(node_index)); +} + } diff --git a/storage/src/vespa/storage/distributor/distributor_stripe_component.h b/storage/src/vespa/storage/distributor/distributor_stripe_component.h index 8bf507f3fac..8fd439992f7 100644 --- a/storage/src/vespa/storage/distributor/distributor_stripe_component.h +++ b/storage/src/vespa/storage/distributor/distributor_stripe_component.h @@ -8,7 +8,6 @@ #include "operationowner.h" #include "statechecker.h" #include <vespa/storage/common/distributorcomponent.h> -#include <vespa/storage/storageutil/utils.h> #include <vespa/storageapi/messageapi/storagecommand.h> #include <vespa/storageapi/buckets/bucketinfo.h> @@ -68,10 +67,7 @@ public: /** * Simple API for the common case of modifying a single node. */ - void update_bucket_database(const document::Bucket& bucket, const BucketCopy& changed_node, uint32_t update_flags) override { - update_bucket_database(bucket, toVector<BucketCopy>(changed_node),update_flags); - } - + void update_bucket_database(const document::Bucket& bucket, const BucketCopy& changed_node, uint32_t update_flags) override; /** * Adds the given copies to the bucket database. */ @@ -82,9 +78,7 @@ public: * If the resulting bucket is empty afterwards, removes the entire * bucket entry from the bucket database. */ - void remove_node_from_bucket_database(const document::Bucket& bucket, uint16_t node_index) override { - remove_nodes_from_bucket_database(bucket, toVector<uint16_t>(node_index)); - } + void remove_node_from_bucket_database(const document::Bucket& bucket, uint16_t node_index) override; /** * Removes the given bucket copies from the bucket database. diff --git a/storage/src/vespa/storage/distributor/ideal_service_layer_nodes_bundle.cpp b/storage/src/vespa/storage/distributor/ideal_service_layer_nodes_bundle.cpp index cc4eedd2a35..1ce5e5c589f 100644 --- a/storage/src/vespa/storage/distributor/ideal_service_layer_nodes_bundle.cpp +++ b/storage/src/vespa/storage/distributor/ideal_service_layer_nodes_bundle.cpp @@ -1,30 +1,60 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "ideal_service_layer_nodes_bundle.h" -#include <vespa/vdslib/distribution/idealnodecalculator.h> -#include <vespa/vespalib/stllike/hash_set_insert.hpp> - +#include <vespa/vespalib/stllike/hash_map.hpp> namespace storage::distributor { -IdealServiceLayerNodesBundle::IdealServiceLayerNodesBundle() noexcept - : _available_nodes(), - _available_nonretired_nodes(), - _available_nonretired_or_maintenance_nodes(), - _unordered_nonretired_or_maintenance_nodes() -{ +namespace { +constexpr size_t BUILD_HASH_LIMIT = 32; } +struct IdealServiceLayerNodesBundle::LookupMap : public vespalib::hash_map<uint16_t, Index> { + using Parent = vespalib::hash_map<uint16_t, Index>; + using Parent::Parent; +}; + +IdealServiceLayerNodesBundle::IdealServiceLayerNodesBundle() noexcept = default; +IdealServiceLayerNodesBundle::IdealServiceLayerNodesBundle(IdealServiceLayerNodesBundle &&) noexcept = default; +IdealServiceLayerNodesBundle::~IdealServiceLayerNodesBundle() = default; + void -IdealServiceLayerNodesBundle::set_available_nonretired_or_maintenance_nodes(std::vector<uint16_t> available_nonretired_or_maintenance_nodes) { - _available_nonretired_or_maintenance_nodes = std::move(available_nonretired_or_maintenance_nodes); - _unordered_nonretired_or_maintenance_nodes.clear(); - _unordered_nonretired_or_maintenance_nodes.insert(_available_nonretired_or_maintenance_nodes.begin(), - _available_nonretired_or_maintenance_nodes.end()); +IdealServiceLayerNodesBundle::set_nodes(ConstNodesRef nodes, + ConstNodesRef nonretired_nodes, + ConstNodesRef nonretired_or_maintenance_nodes) +{ + _nodes.clear(); + _nodes.reserve(nodes.size() + nonretired_nodes.size() + nonretired_or_maintenance_nodes.size()); + std::for_each(nodes.cbegin(), nodes.cend(), [this](uint16_t n) { _nodes.emplace_back(n); }); + _available_sz = nodes.size(); + std::for_each(nonretired_nodes.cbegin(), nonretired_nodes.cend(), [this](uint16_t n) { _nodes.emplace_back(n); }); + _nonretired_sz = nonretired_nodes.size(); + std::for_each(nonretired_or_maintenance_nodes.cbegin(), nonretired_or_maintenance_nodes.cend(), [this](uint16_t n) { _nodes.emplace_back(n); }); + + if (nonretired_or_maintenance_nodes.size() > BUILD_HASH_LIMIT) { + _nonretired_or_maintenance_node_2_index = std::make_unique<LookupMap>(nonretired_or_maintenance_nodes.size()); + for (uint16_t i(0); i < nonretired_or_maintenance_nodes.size(); i++) { + _nonretired_or_maintenance_node_2_index->insert(std::make_pair(nonretired_or_maintenance_nodes[i], Index(i))); + } + } } -IdealServiceLayerNodesBundle::IdealServiceLayerNodesBundle(IdealServiceLayerNodesBundle &&) noexcept = default; +IdealServiceLayerNodesBundle::Index +IdealServiceLayerNodesBundle::ConstNodesRef2Index::lookup(uint16_t node) const noexcept { + for (uint16_t i(0); i < _idealState.size(); i++) { + if (node == _idealState[i]) return Index(i); + } + return Index::invalid(); +} -IdealServiceLayerNodesBundle::~IdealServiceLayerNodesBundle() = default; +IdealServiceLayerNodesBundle::Index +IdealServiceLayerNodesBundle::nonretired_or_maintenance_index(uint16_t node) const noexcept { + if (_nonretired_or_maintenance_node_2_index) { + const auto found = _nonretired_or_maintenance_node_2_index->find(node); + return (found != _nonretired_or_maintenance_node_2_index->end()) ? found->second : Index::invalid(); + } else { + return ConstNodesRef2Index(available_nonretired_or_maintenance_nodes()).lookup(node); + } +} } diff --git a/storage/src/vespa/storage/distributor/ideal_service_layer_nodes_bundle.h b/storage/src/vespa/storage/distributor/ideal_service_layer_nodes_bundle.h index 9577ec09208..1fce5bf0813 100644 --- a/storage/src/vespa/storage/distributor/ideal_service_layer_nodes_bundle.h +++ b/storage/src/vespa/storage/distributor/ideal_service_layer_nodes_bundle.h @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/vespalib/stllike/hash_set.h> +#include <vespa/vespalib/util/small_vector.h> namespace storage::distributor { @@ -9,30 +9,63 @@ namespace storage::distributor { * Bundle of ideal service layer nodes for a bucket. */ class IdealServiceLayerNodesBundle { - std::vector<uint16_t> _available_nodes; - std::vector<uint16_t> _available_nonretired_nodes; - std::vector<uint16_t> _available_nonretired_or_maintenance_nodes; - vespalib::hash_set<uint16_t> _unordered_nonretired_or_maintenance_nodes; public: + using ConstNodesRef = vespalib::ConstArrayRef<uint16_t>; + class Index { + public: + constexpr explicit Index(uint16_t index) noexcept : _index(index) {} + constexpr bool valid() const noexcept { + return _index < MAX_INDEX; + } + constexpr operator uint16_t () const noexcept { return _index; } + static constexpr Index invalid() noexcept { return Index(MAX_INDEX); } + private: + static constexpr uint16_t MAX_INDEX = 0xffff; + uint16_t _index; + }; + struct Node2Index { + virtual ~Node2Index() = default; + virtual Index lookup(uint16_t node) const noexcept = 0; + }; + class NonRetiredOrMaintenance2Index final : public Node2Index { + public: + NonRetiredOrMaintenance2Index(const IdealServiceLayerNodesBundle & idealState) noexcept : _idealState(idealState) {} + Index lookup(uint16_t node) const noexcept override { + return _idealState.nonretired_or_maintenance_index(node); + } + private: + const IdealServiceLayerNodesBundle & _idealState; + }; + class ConstNodesRef2Index final : public Node2Index { + public: + ConstNodesRef2Index(ConstNodesRef idealState) noexcept : _idealState(idealState) {} + Index lookup(uint16_t node) const noexcept override; + private: + ConstNodesRef _idealState; + }; IdealServiceLayerNodesBundle() noexcept; IdealServiceLayerNodesBundle(IdealServiceLayerNodesBundle &&) noexcept; ~IdealServiceLayerNodesBundle(); - void set_available_nodes(std::vector<uint16_t> available_nodes) { - _available_nodes = std::move(available_nodes); - } - void set_available_nonretired_nodes(std::vector<uint16_t> available_nonretired_nodes) { - _available_nonretired_nodes = std::move(available_nonretired_nodes); - } - void set_available_nonretired_or_maintenance_nodes(std::vector<uint16_t> available_nonretired_or_maintenance_nodes); - const std::vector<uint16_t> & available_nodes() const noexcept { return _available_nodes; } - const std::vector<uint16_t> & available_nonretired_nodes() const noexcept { return _available_nonretired_nodes; } - const std::vector<uint16_t> & available_nonretired_or_maintenance_nodes() const noexcept { - return _available_nonretired_or_maintenance_nodes; + void set_nodes(ConstNodesRef nodes, ConstNodesRef nonretired_nodes, ConstNodesRef nonretired_or_maintenance_nodes); + ConstNodesRef available_nodes() const noexcept { return {_nodes.data(), _available_sz}; } + ConstNodesRef available_nonretired_nodes() const noexcept { return {_nodes.data() + _available_sz, _nonretired_sz}; } + ConstNodesRef available_nonretired_or_maintenance_nodes() const noexcept { + uint16_t offset = _available_sz + _nonretired_sz; + return {_nodes.data() + offset, _nodes.size() - offset}; } bool is_nonretired_or_maintenance(uint16_t node) const noexcept { - return _unordered_nonretired_or_maintenance_nodes.contains(node); + return nonretired_or_maintenance_index(node) != Index::invalid(); } + NonRetiredOrMaintenance2Index nonretired_or_maintenance_to_index() const noexcept { return {*this}; } + ConstNodesRef2Index available_to_index() const noexcept { return {available_nodes()}; } +private: + struct LookupMap; + Index nonretired_or_maintenance_index(uint16_t node) const noexcept; + vespalib::SmallVector<uint16_t,16> _nodes; + std::unique_ptr<LookupMap> _nonretired_or_maintenance_node_2_index; + uint16_t _available_sz; + uint16_t _nonretired_sz; }; } diff --git a/storage/src/vespa/storage/distributor/idealstatemetricsset.cpp b/storage/src/vespa/storage/distributor/idealstatemetricsset.cpp index d50b2004bf2..ea345176dd0 100644 --- a/storage/src/vespa/storage/distributor/idealstatemetricsset.cpp +++ b/storage/src/vespa/storage/distributor/idealstatemetricsset.cpp @@ -134,7 +134,7 @@ IdealStateMetricSet::IdealStateMetricSet() IdealStateMetricSet::~IdealStateMetricSet() = default; -void IdealStateMetricSet::setPendingOperations(vespalib::ConstArrayRef<uint64_t> newMetrics) { +void IdealStateMetricSet::setPendingOperations(std::span<uint64_t, IdealStateOperation::OPERATION_COUNT> newMetrics) { for (uint32_t i = 0; i < IdealStateOperation::OPERATION_COUNT; i++) { operations[i]->pending.set(newMetrics[i]); } diff --git a/storage/src/vespa/storage/distributor/idealstatemetricsset.h b/storage/src/vespa/storage/distributor/idealstatemetricsset.h index 0bbc13d061a..e51e58ba3a4 100644 --- a/storage/src/vespa/storage/distributor/idealstatemetricsset.h +++ b/storage/src/vespa/storage/distributor/idealstatemetricsset.h @@ -5,7 +5,7 @@ #include <vespa/metrics/valuemetric.h> #include <vespa/metrics/countmetric.h> #include <vespa/storage/distributor/operations/idealstate/idealstateoperation.h> -#include <vespa/vespalib/util/arrayref.h> +#include <span> namespace storage::distributor { @@ -62,7 +62,7 @@ public: IdealStateMetricSet(); ~IdealStateMetricSet() override; - void setPendingOperations(vespalib::ConstArrayRef<uint64_t> newMetrics); + void setPendingOperations(std::span<uint64_t, IdealStateOperation::OPERATION_COUNT> newMetrics); }; } // storage::distributor diff --git a/storage/src/vespa/storage/distributor/maintenance/simplemaintenancescanner.cpp b/storage/src/vespa/storage/distributor/maintenance/simplemaintenancescanner.cpp index e0c1abaaffa..86399c1b620 100644 --- a/storage/src/vespa/storage/distributor/maintenance/simplemaintenancescanner.cpp +++ b/storage/src/vespa/storage/distributor/maintenance/simplemaintenancescanner.cpp @@ -49,7 +49,7 @@ SimpleMaintenanceScanner::PendingMaintenanceStats & SimpleMaintenanceScanner::PendingMaintenanceStats::operator = (PendingMaintenanceStats &&) noexcept = default; SimpleMaintenanceScanner::PendingMaintenanceStats -SimpleMaintenanceScanner::PendingMaintenanceStats::reset() { +SimpleMaintenanceScanner::PendingMaintenanceStats::fetch_and_reset() { PendingMaintenanceStats prev = std::move(*this); global = GlobalMaintenanceStats(); perNodeStats.reset(prev.perNodeStats.numNodes()); @@ -78,11 +78,11 @@ SimpleMaintenanceScanner::scanNext() } SimpleMaintenanceScanner::PendingMaintenanceStats -SimpleMaintenanceScanner::reset() +SimpleMaintenanceScanner::fetch_and_reset() { _bucketCursor = document::BucketId(); _bucketSpaceItr = _bucketSpaceRepo.begin(); - return _pendingMaintenance.reset(); + return _pendingMaintenance.fetch_and_reset(); } void diff --git a/storage/src/vespa/storage/distributor/maintenance/simplemaintenancescanner.h b/storage/src/vespa/storage/distributor/maintenance/simplemaintenancescanner.h index 35b022c7af7..3d1a57a6422 100644 --- a/storage/src/vespa/storage/distributor/maintenance/simplemaintenancescanner.h +++ b/storage/src/vespa/storage/distributor/maintenance/simplemaintenancescanner.h @@ -29,7 +29,7 @@ public: PendingMaintenanceStats(PendingMaintenanceStats &&) noexcept; PendingMaintenanceStats &operator = (PendingMaintenanceStats &&) noexcept; ~PendingMaintenanceStats(); - PendingMaintenanceStats reset(); + [[nodiscard]] PendingMaintenanceStats fetch_and_reset(); GlobalMaintenanceStats global; NodeMaintenanceStatsTracker perNodeStats; @@ -53,7 +53,7 @@ public: ~SimpleMaintenanceScanner() override; ScanResult scanNext() override; - PendingMaintenanceStats reset(); + [[nodiscard]] PendingMaintenanceStats fetch_and_reset(); // TODO: move out into own interface! void prioritizeBucket(const document::Bucket &id); diff --git a/storage/src/vespa/storage/distributor/messagetracker.cpp b/storage/src/vespa/storage/distributor/messagetracker.cpp index 28fbaad4619..842238aa24c 100644 --- a/storage/src/vespa/storage/distributor/messagetracker.cpp +++ b/storage/src/vespa/storage/distributor/messagetracker.cpp @@ -20,7 +20,7 @@ MessageTracker::~MessageTracker() = default; void MessageTracker::flushQueue(MessageSender& sender) { - _sentMessages.resize(_commandQueue.size()); + _sentMessages.resize(_sentMessages.size() + _commandQueue.size()); for (const auto & toSend : _commandQueue) { toSend._msg->setAddress(api::StorageMessageAddress::create(_cluster_ctx.cluster_name_ptr(), lib::NodeType::STORAGE, toSend._target)); _sentMessages[toSend._msg->getMsgId()] = toSend._target; diff --git a/storage/src/vespa/storage/distributor/operations/external/putoperation.cpp b/storage/src/vespa/storage/distributor/operations/external/putoperation.cpp index 86ea9a559f5..854e7d15f82 100644 --- a/storage/src/vespa/storage/distributor/operations/external/putoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/external/putoperation.cpp @@ -10,7 +10,6 @@ #include <vespa/storage/distributor/storage_node_up_states.h> #include <vespa/storageapi/message/persistence.h> #include <vespa/vdslib/distribution/distribution.h> -#include <vespa/vdslib/distribution/idealnodecalculatorimpl.h> #include <vespa/vdslib/state/clusterstate.h> #include <algorithm> @@ -67,13 +66,11 @@ PutOperation::insertDatabaseEntryAndScheduleCreateBucket(const OperationTargetLi assert(!multipleBuckets); (void) multipleBuckets; BucketDatabase::Entry entry(_bucket_space.getBucketDatabase().get(lastBucket)); - const std::vector<uint16_t> & idealState = _bucket_space.get_ideal_service_layer_nodes_bundle( - lastBucket).available_nodes(); - active = ActiveCopy::calculate(idealState, _bucket_space.getDistribution(), entry, + active = ActiveCopy::calculate(_bucket_space.get_ideal_service_layer_nodes_bundle(lastBucket).available_to_index(), _bucket_space.getDistribution(), entry, _op_ctx.distributor_config().max_activation_inhibited_out_of_sync_groups()); LOG(debug, "Active copies for bucket %s: %s", entry.getBucketId().toString().c_str(), active.toString().c_str()); for (uint32_t i=0; i<active.size(); ++i) { - BucketCopy copy(*entry->getNode(active[i]._nodeIndex)); + BucketCopy copy(*entry->getNode(active[i].nodeIndex())); copy.setActive(true); entry->updateNode(copy); } diff --git a/storage/src/vespa/storage/distributor/operationtargetresolver.h b/storage/src/vespa/storage/distributor/operationtargetresolver.h index 5e3c4a73f66..2de477d03e5 100644 --- a/storage/src/vespa/storage/distributor/operationtargetresolver.h +++ b/storage/src/vespa/storage/distributor/operationtargetresolver.h @@ -15,23 +15,23 @@ namespace storage::distributor { class OperationTarget : public vespalib::AsciiPrintable { document::Bucket _bucket; - lib::Node _node; - bool _newCopy; + lib::Node _node; + bool _newCopy; public: - OperationTarget() : _newCopy(true) {} - OperationTarget(const document::Bucket& bucket, const lib::Node& node, bool newCopy) + OperationTarget() noexcept : _newCopy(true) {} + OperationTarget(const document::Bucket& bucket, const lib::Node& node, bool newCopy) noexcept : _bucket(bucket), _node(node), _newCopy(newCopy) {} - document::BucketId getBucketId() const { return _bucket.getBucketId(); } - document::Bucket getBucket() const { return _bucket; } - const lib::Node& getNode() const { return _node; } - bool isNewCopy() const { return _newCopy; } + document::BucketId getBucketId() const noexcept { return _bucket.getBucketId(); } + document::Bucket getBucket() const noexcept { return _bucket; } + const lib::Node& getNode() const noexcept { return _node; } + bool isNewCopy() const noexcept { return _newCopy; } - bool operator==(const OperationTarget& o) const { + bool operator==(const OperationTarget& o) const noexcept { return (_bucket == o._bucket && _node == o._node && _newCopy == o._newCopy); } - bool operator!=(const OperationTarget& o) const { + bool operator!=(const OperationTarget& o) const noexcept { return !(operator==(o)); } @@ -40,13 +40,13 @@ public: class OperationTargetList : public std::vector<OperationTarget> { public: - bool hasAnyNewCopies() const { + bool hasAnyNewCopies() const noexcept { for (size_t i=0; i<size(); ++i) { if (operator[](i).isNewCopy()) return true; } return false; } - bool hasAnyExistingCopies() const { + bool hasAnyExistingCopies() const noexcept { for (size_t i=0; i<size(); ++i) { if (!operator[](i).isNewCopy()) return true; } @@ -63,8 +63,7 @@ public: PUT }; - virtual OperationTargetList getTargets(OperationType type, - const document::BucketId& id) = 0; + virtual OperationTargetList getTargets(OperationType type, const document::BucketId& id) = 0; }; } diff --git a/storage/src/vespa/storage/distributor/operationtargetresolverimpl.cpp b/storage/src/vespa/storage/distributor/operationtargetresolverimpl.cpp index 736d8c692e3..eb08cf51f43 100644 --- a/storage/src/vespa/storage/distributor/operationtargetresolverimpl.cpp +++ b/storage/src/vespa/storage/distributor/operationtargetresolverimpl.cpp @@ -9,22 +9,8 @@ namespace storage::distributor { -namespace { - -lib::IdealNodeList -make_node_list(const std::vector<uint16_t>& nodes) -{ - lib::IdealNodeList list; - for (auto node : nodes) { - list.push_back(lib::Node(lib::NodeType::STORAGE, node)); - } - return list; -} - -} - BucketInstance::BucketInstance(const document::BucketId& id, const api::BucketInfo& info, lib::Node node, - uint16_t idealLocationPriority, bool trusted, bool exist) + uint16_t idealLocationPriority, bool trusted, bool exist) noexcept : _bucket(id), _info(info), _node(node), _idealLocationPriority(idealLocationPriority), _trusted(trusted), _exist(exist) { @@ -44,19 +30,19 @@ BucketInstance::print(vespalib::asciistream& out, const PrintProperties&) const bool BucketInstanceList::contains(lib::Node node) const { - for (uint32_t i=0; i<_instances.size(); ++i) { - if (_instances[i]._node == node) return true; + for (const auto & instance : _instances) { + if (instance._node == node) return true; } return false; } void -BucketInstanceList::add(const BucketDatabase::Entry& e, const lib::IdealNodeList& idealState) +BucketInstanceList::add(const BucketDatabase::Entry& e, const IdealServiceLayerNodesBundle::Node2Index & idealState) { for (uint32_t i = 0; i < e.getBucketInfo().getNodeCount(); ++i) { const BucketCopy& copy(e.getBucketInfo().getNodeRef(i)); lib::Node node(lib::NodeType::STORAGE, copy.getNode()); - _instances.emplace_back(e.getBucketId(), copy.getBucketInfo(), node, idealState.indexOf(node), copy.trusted()); + _instances.emplace_back(e.getBucketId(), copy.getBucketInfo(), node, idealState.lookup(copy.getNode()), copy.trusted(), true); } } @@ -66,8 +52,8 @@ BucketInstanceList::populate(const document::BucketId& specificId, const Distrib std::vector<BucketDatabase::Entry> entries; db.getParents(specificId, entries); for (const auto & entry : entries) { - lib::IdealNodeList idealNodes(make_node_list(distributor_bucket_space.get_ideal_service_layer_nodes_bundle(entry.getBucketId()).available_nonretired_or_maintenance_nodes())); - add(entry, idealNodes); + auto node2Index = distributor_bucket_space.get_ideal_service_layer_nodes_bundle(entry.getBucketId()).nonretired_or_maintenance_to_index(); + add(entry, node2Index); } } @@ -96,7 +82,7 @@ BucketInstanceList::limitToRedundancyCopies(uint16_t redundancy) document::BucketId BucketInstanceList::leastSpecificLeafBucketInSubtree(const document::BucketId& candidateId, const document::BucketId& mostSpecificId, - const BucketDatabase& db) const + const BucketDatabase& db) { assert(candidateId.contains(mostSpecificId)); document::BucketId treeNode = candidateId; @@ -110,18 +96,17 @@ BucketInstanceList::leastSpecificLeafBucketInSubtree(const document::BucketId& c } void -BucketInstanceList::extendToEnoughCopies(const DistributorBucketSpace& distributor_bucket_space, - const BucketDatabase& db, - const document::BucketId& targetIfNonPreExisting, - const document::BucketId& mostSpecificId) +BucketInstanceList::extendToEnoughCopies(const DistributorBucketSpace& distributor_bucket_space, const BucketDatabase& db, + const document::BucketId& targetIfNonPreExisting, const document::BucketId& mostSpecificId) { document::BucketId newTarget(_instances.empty() ? targetIfNonPreExisting : _instances[0]._bucket); newTarget = leastSpecificLeafBucketInSubtree(newTarget, mostSpecificId, db); - lib::IdealNodeList idealNodes(make_node_list(distributor_bucket_space.get_ideal_service_layer_nodes_bundle(newTarget).available_nonretired_nodes())); + const auto & idealNodes = distributor_bucket_space.get_ideal_service_layer_nodes_bundle(newTarget).available_nonretired_nodes(); for (uint32_t i=0; i<idealNodes.size(); ++i) { - if (!contains(idealNodes[i])) { - _instances.emplace_back(newTarget, api::BucketInfo(), idealNodes[i], i, false, false); + lib::Node node(lib::NodeType::STORAGE, idealNodes[i]); + if (!contains(node)) { + _instances.emplace_back(newTarget, api::BucketInfo(), node, i, false, false); } } } @@ -131,7 +116,7 @@ BucketInstanceList::createTargets(document::BucketSpace bucketSpace) { OperationTargetList result; for (const auto& bi : _instances) { - result.push_back(OperationTarget(document::Bucket(bucketSpace, bi._bucket), bi._node, !bi._exist)); + result.emplace_back(document::Bucket(bucketSpace, bi._bucket), bi._node, !bi._exist); } return result; } diff --git a/storage/src/vespa/storage/distributor/operationtargetresolverimpl.h b/storage/src/vespa/storage/distributor/operationtargetresolverimpl.h index 0caeee466e0..b76388da9bc 100644 --- a/storage/src/vespa/storage/distributor/operationtargetresolverimpl.h +++ b/storage/src/vespa/storage/distributor/operationtargetresolverimpl.h @@ -3,8 +3,8 @@ #pragma once #include "operationtargetresolver.h" +#include "ideal_service_layer_nodes_bundle.h" #include <vespa/storage/bucketdb/bucketdatabase.h> -#include <vespa/vdslib/distribution/idealnodecalculator.h> #include <algorithm> namespace storage::distributor { @@ -19,11 +19,11 @@ struct BucketInstance : public vespalib::AsciiPrintable { bool _trusted; bool _exist; - BucketInstance() : _idealLocationPriority(0xffff), - _trusted(false), _exist(false) {} + BucketInstance() noexcept + : _idealLocationPriority(0xffff), _trusted(false), _exist(false) {} BucketInstance(const document::BucketId& id, const api::BucketInfo& info, lib::Node node, uint16_t idealLocationPriority, bool trusted, - bool exist = true); + bool exist) noexcept; void print(vespalib::asciistream& out, const PrintProperties&) const override; }; @@ -42,10 +42,10 @@ class BucketInstanceList : public vespalib::AsciiPrintable { * Postconditions: * <return value>.contains(mostSpecificId) */ - document::BucketId leastSpecificLeafBucketInSubtree( - const document::BucketId& candidateId, - const document::BucketId& mostSpecificId, - const BucketDatabase& db) const; + static document::BucketId + leastSpecificLeafBucketInSubtree(const document::BucketId& candidateId, + const document::BucketId& mostSpecificId, + const BucketDatabase& db); public: void add(const BucketInstance& instance) { _instances.push_back(instance); } @@ -65,7 +65,7 @@ public: const document::BucketId& mostSpecificId); void populate(const document::BucketId&, const DistributorBucketSpace&, BucketDatabase&); - void add(const BucketDatabase::Entry& e, const lib::IdealNodeList& idealState); + void add(const BucketDatabase::Entry& e, const IdealServiceLayerNodesBundle::Node2Index & idealState); template <typename Order> void sort(const Order& order) { @@ -79,9 +79,9 @@ public: class OperationTargetResolverImpl : public OperationTargetResolver { const DistributorBucketSpace& _distributor_bucket_space; - BucketDatabase& _bucketDatabase; - uint32_t _minUsedBucketBits; - uint16_t _redundancy; + BucketDatabase& _bucketDatabase; + uint32_t _minUsedBucketBits; + uint16_t _redundancy; document::BucketSpace _bucketSpace; public: @@ -97,8 +97,7 @@ public: _bucketSpace(bucketSpace) {} - BucketInstanceList getAllInstances(OperationType type, - const document::BucketId& id); + BucketInstanceList getAllInstances(OperationType type, const document::BucketId& id); BucketInstanceList getInstances(OperationType type, const document::BucketId& id) { BucketInstanceList result(getAllInstances(type, id)); result.limitToRedundancyCopies(_redundancy); diff --git a/storage/src/vespa/storage/distributor/statechecker.h b/storage/src/vespa/storage/distributor/statechecker.h index 25918e7a047..d120b5e62d7 100644 --- a/storage/src/vespa/storage/distributor/statechecker.h +++ b/storage/src/vespa/storage/distributor/statechecker.h @@ -77,7 +77,9 @@ public: const bool merges_inhibited_in_bucket_space; const BucketDatabase::Entry& getSiblingEntry() const noexcept { return siblingEntry; } - const std::vector<uint16_t> & idealState() const noexcept { return idealStateBundle.available_nonretired_or_maintenance_nodes(); } + IdealServiceLayerNodesBundle::ConstNodesRef idealState() const noexcept { + return idealStateBundle.available_nonretired_or_maintenance_nodes(); + } document::Bucket getBucket() const noexcept { return bucket; } document::BucketId getBucketId() const noexcept { return bucket.getBucketId(); } diff --git a/storage/src/vespa/storage/distributor/statecheckers.cpp b/storage/src/vespa/storage/distributor/statecheckers.cpp index 43766225155..2aef2d17f54 100644 --- a/storage/src/vespa/storage/distributor/statecheckers.cpp +++ b/storage/src/vespa/storage/distributor/statecheckers.cpp @@ -145,8 +145,10 @@ JoinBucketsStateChecker::isFirstSibling(const document::BucketId& bucketId) namespace { +using ConstNodesRef = IdealServiceLayerNodesBundle::ConstNodesRef; + bool -equalNodeSet(const std::vector<uint16_t>& idealState, const BucketDatabase::Entry& dbEntry) +equalNodeSet(ConstNodesRef idealState, const BucketDatabase::Entry& dbEntry) { if (idealState.size() != dbEntry->getNodeCount()) { return false; @@ -187,6 +189,42 @@ inconsistentJoinIsAllowed(const StateChecker::Context& context) && bucketAndSiblingReplicaLocationsEqualIdealState(context)); } +bool +isInconsistentlySplit(const StateChecker::Context& c) +{ + return (c.entries.size() > 1); +} + +// We don't want to invoke joins on buckets that have more replicas than +// required. This is in particular because joins cause ideal states to change +// for the target buckets and trigger merges. Since the removal of the non- +// ideal replicas is done by the DeleteBuckets state-checker, it will become +// preempted by potential follow-up joins unless we explicitly avoid these. +bool +contextBucketHasTooManyReplicas(const StateChecker::Context& c) +{ + return (c.entry->getNodeCount() > c.distribution.getRedundancy()); +} + +bool +bucketAtDistributionBitLimit(const document::BucketId& bucket, const StateChecker::Context& c) +{ + return (bucket.getUsedBits() <= std::max(uint32_t(c.systemState.getDistributionBitCount()), + c.distributorConfig.getMinimalBucketSplit())); +} + +bool +legalBucketSplitLevel(const document::BucketId& bucket, const StateChecker::Context& c) +{ + return bucket.getUsedBits() >= c.distributorConfig.getMinimalBucketSplit(); +} + +bool +bucketHasMultipleChildren(const document::BucketId& bucket, const StateChecker::Context& c) +{ + return c.db.childCount(bucket) > 1; +} + } // anon ns bool @@ -246,28 +284,6 @@ JoinBucketsStateChecker::singleBucketJoinIsEnabled(const Context& c) return c.distributorConfig.getEnableJoinForSiblingLessBuckets(); } -namespace { - -// We don't want to invoke joins on buckets that have more replicas than -// required. This is in particular because joins cause ideal states to change -// for the target buckets and trigger merges. Since the removal of the non- -// ideal replicas is done by the DeleteBuckets state-checker, it will become -// preempted by potential follow-up joins unless we explicitly avoid these. -bool -contextBucketHasTooManyReplicas(const StateChecker::Context& c) -{ - return (c.entry->getNodeCount() > c.distribution.getRedundancy()); -} - -bool -bucketAtDistributionBitLimit(const document::BucketId& bucket, const StateChecker::Context& c) -{ - return (bucket.getUsedBits() <= std::max(uint32_t(c.systemState.getDistributionBitCount()), - c.distributorConfig.getMinimalBucketSplit())); -} - -} - bool JoinBucketsStateChecker::shouldJoin(const Context& c) { @@ -361,22 +377,6 @@ JoinBucketsStateChecker::smallEnoughToJoin(const Context& c) return true; } -namespace { - -bool -legalBucketSplitLevel(const document::BucketId& bucket, const StateChecker::Context& c) -{ - return bucket.getUsedBits() >= c.distributorConfig.getMinimalBucketSplit(); -} - -bool -bucketHasMultipleChildren(const document::BucketId& bucket, const StateChecker::Context& c) -{ - return c.db.childCount(bucket) > 1; -} - -} - document::Bucket JoinBucketsStateChecker::computeJoinBucket(const Context& c) { @@ -482,16 +482,6 @@ SplitInconsistentStateChecker::getReason(const document::BucketId& bucketId, con return reason.str(); } -namespace { - -bool -isInconsistentlySplit(const StateChecker::Context& c) -{ - return (c.entries.size() > 1); -} - -} - StateChecker::Result SplitInconsistentStateChecker::check(Context& c) const { @@ -513,7 +503,8 @@ SplitInconsistentStateChecker::check(Context& c) const namespace { -bool containsMaintenanceNode(const std::vector<uint16_t>& ideal, const StateChecker::Context& c) +bool +containsMaintenanceNode(ConstNodesRef ideal, const StateChecker::Context& c) { for (uint16_t n : ideal) { if (c.systemState.getNodeState(lib::Node(lib::NodeType::STORAGE, n)).getState() == lib::State::MAINTENANCE) { @@ -523,7 +514,8 @@ bool containsMaintenanceNode(const std::vector<uint16_t>& ideal, const StateChec return false; } -bool ideal_node_is_unavailable_in_pending_state(const StateChecker::Context& c) { +bool +ideal_node_is_unavailable_in_pending_state(const StateChecker::Context& c) { if (!c.pending_cluster_state) { return false; } @@ -536,7 +528,7 @@ bool ideal_node_is_unavailable_in_pending_state(const StateChecker::Context& c) } bool -consistentApartFromEmptyBucketsInNonIdealLocationAndInvalidEntries(const std::vector<uint16_t>& idealNodes, const BucketInfo& entry) +consistentApartFromEmptyBucketsInNonIdealLocationAndInvalidEntries(ConstNodesRef idealNodes, const BucketInfo& entry) { api::BucketInfo info; for (uint32_t i=0, n=entry.getNodeCount(); i<n; ++i) { @@ -820,7 +812,7 @@ DeleteExtraCopiesStateChecker::bucketHasNoData(const Context& c) bool DeleteExtraCopiesStateChecker::copyIsInIdealState(const BucketCopy& cp, const Context& c) { - return hasItem(c.idealState(), cp.getNode()); + return c.idealStateBundle.is_nonretired_or_maintenance(cp.getNode()); } bool @@ -940,7 +932,7 @@ bool BucketStateStateChecker::shouldSkipActivationDueToMaintenance(const ActiveList& activeNodes, const Context& c) { for (uint32_t i = 0; i < activeNodes.size(); ++i) { - const auto node_index = activeNodes[i]._nodeIndex; + const auto node_index = activeNodes[i].nodeIndex(); const BucketCopy* cp(c.entry->getNode(node_index)); if (!cp || cp->active()) { continue; @@ -978,7 +970,8 @@ BucketStateStateChecker::check(Context& c) const return Result::noMaintenanceNeeded(); } - ActiveList activeNodes = ActiveCopy::calculate(c.idealState(), c.distribution, c.entry, + ActiveList activeNodes = ActiveCopy::calculate(c.idealStateBundle.nonretired_or_maintenance_to_index(), + c.distribution, c.entry, c.distributorConfig.max_activation_inhibited_out_of_sync_groups()); if (activeNodes.empty()) { return Result::noMaintenanceNeeded(); @@ -990,12 +983,12 @@ BucketStateStateChecker::check(Context& c) const vespalib::asciistream reason; std::vector<uint16_t> operationNodes; for (uint32_t i=0; i<activeNodes.size(); ++i) { - const BucketCopy* cp = c.entry->getNode(activeNodes[i]._nodeIndex); + const BucketCopy* cp = c.entry->getNode(activeNodes[i].nodeIndex()); if (cp == nullptr || cp->active()) { continue; } - operationNodes.push_back(activeNodes[i]._nodeIndex); - reason << "[Setting node " << activeNodes[i]._nodeIndex << " as active: " << activeNodes[i].getReason() << "]"; + operationNodes.push_back(activeNodes[i].nodeIndex()); + reason << "[Setting node " << activeNodes[i].nodeIndex() << " as active: " << activeNodes[i].getReason() << "]"; } // Deactivate all copies that are currently marked as active. @@ -1006,7 +999,7 @@ BucketStateStateChecker::check(Context& c) const } bool shouldBeActive = false; for (uint32_t j=0; j<activeNodes.size(); ++j) { - if (activeNodes[j]._nodeIndex == cp.getNode()) { + if (activeNodes[j].nodeIndex() == cp.getNode()) { shouldBeActive = true; } } @@ -1022,7 +1015,7 @@ BucketStateStateChecker::check(Context& c) const std::vector<uint16_t> activeNodeIndexes; for (uint32_t i=0; i<activeNodes.size(); ++i) { - activeNodeIndexes.push_back(activeNodes[i]._nodeIndex); + activeNodeIndexes.push_back(activeNodes[i].nodeIndex()); } auto op = std::make_unique<SetBucketStateOperation>(c.node_ctx, BucketAndNodes(c.getBucket(), operationNodes), activeNodeIndexes); diff --git a/storage/src/vespa/storage/persistence/persistenceutil.h b/storage/src/vespa/storage/persistence/persistenceutil.h index c3fcb68ddc8..4bd0222bb9e 100644 --- a/storage/src/vespa/storage/persistence/persistenceutil.h +++ b/storage/src/vespa/storage/persistence/persistenceutil.h @@ -10,7 +10,6 @@ #include <vespa/persistence/spi/result.h> #include <vespa/persistence/spi/context.h> #include <vespa/vespalib/io/fileutil.h> -#include <vespa/storage/storageutil/utils.h> namespace storage::api { class StorageMessage; diff --git a/storage/src/vespa/storage/storageutil/utils.h b/storage/src/vespa/storage/storageutil/utils.h index debb7e71ace..3d3f5b85d71 100644 --- a/storage/src/vespa/storage/storageutil/utils.h +++ b/storage/src/vespa/storage/storageutil/utils.h @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vector> +#include <vespa/vespalib/util/arrayref.h> #include <sstream> namespace storage { @@ -10,50 +10,55 @@ namespace storage { * Creates a vector of the given type with one entry in it. */ template<class A> -std::vector<A> toVector(A entry) { +std::vector<A> +toVector(A entry) { std::vector<A> entries; entries.push_back(entry); return entries; -}; +} /** * Creates a vector of the given type with two entries in it. */ template<class A> -std::vector<A> toVector(A entry, A entry2) { +std::vector<A> +toVector(A entry, A entry2) { std::vector<A> entries; entries.push_back(entry); entries.push_back(entry2); return entries; -}; +} /** * Creates a vector of the given type with two entries in it. */ template<class A> -std::vector<A> toVector(A entry, A entry2, A entry3) { +std::vector<A> +toVector(A entry, A entry2, A entry3) { std::vector<A> entries; entries.push_back(entry); entries.push_back(entry2); entries.push_back(entry3); return entries; -}; +} /** * Creates a vector of the given type with two entries in it. */ template<class A> -std::vector<A> toVector(A entry, A entry2, A entry3, A entry4) { +std::vector<A> +toVector(A entry, A entry2, A entry3, A entry4) { std::vector<A> entries; entries.push_back(entry); entries.push_back(entry2); entries.push_back(entry3); entries.push_back(entry4); return entries; -}; +} template<class A> -std::string dumpVector(const std::vector<A>& vec) { +std::string +dumpVector(const std::vector<A>& vec) { std::ostringstream ost; for (uint32_t i = 0; i < vec.size(); ++i) { if (!ost.str().empty()) { @@ -65,27 +70,5 @@ std::string dumpVector(const std::vector<A>& vec) { return ost.str(); } -template<class A> -bool hasItem(const std::vector<A>& vec, A entry) { - for (uint32_t i = 0; i < vec.size(); ++i) { - if (vec[i] == entry) { - return true; - } - } - - return false; -} - -template<typename T> -struct ConfigReader : public T::Subscriber, public T -{ - T& config; // Alter to inherit T to simplify but kept this for compatability - - ConfigReader(const std::string& configId) : config(*this) { - T::subscribe(configId, *this); - } - void configure(const T& c) { config = c; } -}; - } diff --git a/vdslib/src/tests/distribution/CMakeLists.txt b/vdslib/src/tests/distribution/CMakeLists.txt index c4ae8b0291c..3f3be1e1cad 100644 --- a/vdslib/src/tests/distribution/CMakeLists.txt +++ b/vdslib/src/tests/distribution/CMakeLists.txt @@ -3,7 +3,6 @@ vespa_add_library(vdslib_testdistribution SOURCES distributiontest.cpp grouptest.cpp - idealnodecalculatorimpltest.cpp DEPENDS vdslib GTest::GTest diff --git a/vdslib/src/tests/distribution/distributiontest.cpp b/vdslib/src/tests/distribution/distributiontest.cpp index ec7c05fa7a2..ce07711a069 100644 --- a/vdslib/src/tests/distribution/distributiontest.cpp +++ b/vdslib/src/tests/distribution/distributiontest.cpp @@ -5,7 +5,6 @@ #include <vespa/config/subscription/configuri.h> #include <vespa/fastos/file.h> #include <vespa/vdslib/distribution/distribution.h> -#include <vespa/vdslib/distribution/idealnodecalculator.h> #include <vespa/vdslib/state/clusterstate.h> #include <vespa/vdslib/state/random.h> #include <vespa/vespalib/data/slime/slime.h> @@ -84,6 +83,51 @@ TEST(DistributionTest, test_verify_java_distributions) namespace { +/** +* A list of ideal nodes, sorted in preferred order. Wraps a vector to hide +* unneeded details, and make it easily printable. +*/ +class IdealNodeList : public document::Printable { +public: + IdealNodeList() noexcept; + ~IdealNodeList(); + + void push_back(const Node& node) { + _idealNodes.push_back(node); + } + + const Node& operator[](uint32_t i) const noexcept { return _idealNodes[i]; } + uint32_t size() const noexcept { return _idealNodes.size(); } + bool contains(const Node& n) const noexcept { + return indexOf(n) != 0xffff; + } + uint16_t indexOf(const Node& n) const noexcept { + for (uint16_t i=0; i<_idealNodes.size(); ++i) { + if (n == _idealNodes[i]) return i; + } + return 0xffff; + } + + void print(std::ostream& out, bool, const std::string &) const override; +private: + std::vector<Node> _idealNodes; +}; + +IdealNodeList::IdealNodeList() noexcept = default; +IdealNodeList::~IdealNodeList() = default; + +void +IdealNodeList::print(std::ostream& out, bool , const std::string &) const +{ + out << "["; + for (uint32_t i=0; i<_idealNodes.size(); ++i) { + if (i != 0) out << ", "; + out << _idealNodes[i]; + } + out << "]"; +} + + struct ExpectedResult { ExpectedResult() { } ExpectedResult(const ExpectedResult &) = default; diff --git a/vdslib/src/tests/distribution/idealnodecalculatorimpltest.cpp b/vdslib/src/tests/distribution/idealnodecalculatorimpltest.cpp deleted file mode 100644 index 4159491097c..00000000000 --- a/vdslib/src/tests/distribution/idealnodecalculatorimpltest.cpp +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/config-stor-distribution.h> -#include <vespa/vdslib/distribution/idealnodecalculatorimpl.h> -#include <vespa/vdslib/distribution/distribution.h> -#include <vespa/vdslib/state/clusterstate.h> -#include <vespa/vespalib/gtest/gtest.h> - -namespace storage::lib { - -/** - * Class is just a wrapper for distribution, so little needs to be tested. Just - * that: - * - * - get ideal nodes calls gets propagated correctly. - * - Changes in distribution/cluster state is picked up. - */ - -TEST(IdealNodeCalculatorImplTest, test_normal_usage) -{ - ClusterState state("storage:10"); - Distribution distr(Distribution::getDefaultDistributionConfig(3, 10)); - IdealNodeCalculatorImpl impl; - IdealNodeCalculatorConfigurable& configurable(impl); - IdealNodeCalculator& calc(impl); - configurable.setDistribution(distr); - configurable.setClusterState(state); - - std::string expected("[storage.8, storage.9, storage.6]"); - EXPECT_EQ( - expected, - calc.getIdealStorageNodes(document::BucketId(16, 5)).toString()); -} - -} diff --git a/vdslib/src/vespa/vdslib/distribution/CMakeLists.txt b/vdslib/src/vespa/vdslib/distribution/CMakeLists.txt index 0d9342291e8..58ec94eec9c 100644 --- a/vdslib/src/vespa/vdslib/distribution/CMakeLists.txt +++ b/vdslib/src/vespa/vdslib/distribution/CMakeLists.txt @@ -4,7 +4,6 @@ vespa_add_library(vdslib_distribution OBJECT distribution.cpp distribution_config_util.cpp group.cpp - idealnodecalculatorimpl.cpp redundancygroupdistribution.cpp DEPENDS ) diff --git a/vdslib/src/vespa/vdslib/distribution/idealnodecalculator.h b/vdslib/src/vespa/vdslib/distribution/idealnodecalculator.h deleted file mode 100644 index 4eb8f7e04ae..00000000000 --- a/vdslib/src/vespa/vdslib/distribution/idealnodecalculator.h +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * An interface to implement for a calculator calcuting ideal state. It should - * be easy to wrap this calculator in a cache. Thus options that seldom change, - * are taken in as set parameters, such that existing cache can be invalidated. - */ -#pragma once - -#include <vespa/document/bucket/bucketid.h> -#include <vespa/document/util/printable.h> -#include <vespa/vdslib/state/node.h> -#include <vector> -#include <memory> - -namespace storage::lib { - -class Distribution; -class ClusterState; - -/** - * A list of ideal nodes, sorted in preferred order. Wraps a vector to hide - * unneeded details, and make it easily printable. - */ -class IdealNodeList : public document::Printable { -public: - IdealNodeList() noexcept; - ~IdealNodeList(); - - void push_back(const Node& node) { - _idealNodes.push_back(node); - } - - const Node& operator[](uint32_t i) const noexcept { return _idealNodes[i]; } - uint32_t size() const noexcept { return _idealNodes.size(); } - bool contains(const Node& n) const noexcept { - return indexOf(n) != 0xffff; - } - uint16_t indexOf(const Node& n) const noexcept { - for (uint16_t i=0; i<_idealNodes.size(); ++i) { - if (n == _idealNodes[i]) return i; - } - return 0xffff; - } - - void print(std::ostream& out, bool, const std::string &) const override; -private: - std::vector<Node> _idealNodes; -}; - -/** - * Simple interface to use for those who needs to calculate ideal nodes. - */ -class IdealNodeCalculator { -public: - using SP = std::shared_ptr<IdealNodeCalculator>; - enum UpStates { - UpInit, - UpInitMaintenance, - UP_STATE_COUNT - }; - - virtual ~IdealNodeCalculator() = default; - - virtual IdealNodeList getIdealNodes(const NodeType&, const document::BucketId&, UpStates upStates = UpInit) const = 0; - - // Wrapper functions to make prettier call if nodetype is given. - IdealNodeList getIdealDistributorNodes(const document::BucketId& bucket, UpStates upStates = UpInit) const { - return getIdealNodes(NodeType::DISTRIBUTOR, bucket, upStates); - } - IdealNodeList getIdealStorageNodes(const document::BucketId& bucket, UpStates upStates = UpInit) const { - return getIdealNodes(NodeType::STORAGE, bucket, upStates); - } -}; - - -/** - * More complex interface that provides a way to alter needed settings not - * provided in the function call itself. - */ -class IdealNodeCalculatorConfigurable : public IdealNodeCalculator -{ -public: - using SP = std::shared_ptr<IdealNodeCalculatorConfigurable>; - - virtual void setDistribution(const Distribution&) = 0; - virtual void setClusterState(const ClusterState&) = 0; -}; - -} diff --git a/vdslib/src/vespa/vdslib/distribution/idealnodecalculatorimpl.cpp b/vdslib/src/vespa/vdslib/distribution/idealnodecalculatorimpl.cpp deleted file mode 100644 index 86123f47d6f..00000000000 --- a/vdslib/src/vespa/vdslib/distribution/idealnodecalculatorimpl.cpp +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "idealnodecalculatorimpl.h" -#include "distribution.h" -#include <vespa/vespalib/util/exceptions.h> -#include <ostream> -#include <cassert> - -namespace storage::lib { - -IdealNodeList::IdealNodeList() noexcept = default; -IdealNodeList::~IdealNodeList() = default; - -void -IdealNodeList::print(std::ostream& out, bool , const std::string &) const -{ - out << "["; - for (uint32_t i=0; i<_idealNodes.size(); ++i) { - if (i != 0) out << ", "; - out << _idealNodes[i]; - } - out << "]"; -} - -IdealNodeCalculatorImpl::IdealNodeCalculatorImpl() - : _distribution(0), - _clusterState(0) -{ - initUpStateMapping(); -} - -IdealNodeCalculatorImpl::~IdealNodeCalculatorImpl() = default; - -void -IdealNodeCalculatorImpl::setDistribution(const Distribution& d) { - _distribution = &d; -} -void -IdealNodeCalculatorImpl::setClusterState(const ClusterState& cs) { - _clusterState = &cs; -} - -IdealNodeList -IdealNodeCalculatorImpl::getIdealNodes(const NodeType& nodeType, - const document::BucketId& bucket, - UpStates upStates) const -{ - assert(_clusterState != 0); - assert(_distribution != 0); - std::vector<uint16_t> nodes; - _distribution->getIdealNodes(nodeType, *_clusterState, bucket, nodes, _upStates[upStates]); - IdealNodeList list; - for (uint32_t i=0; i<nodes.size(); ++i) { - list.push_back(Node(nodeType, nodes[i])); - } - return list; -} - -void -IdealNodeCalculatorImpl::initUpStateMapping() { - _upStates.clear(); - _upStates.resize(UP_STATE_COUNT); - _upStates[UpInit] = "ui"; - _upStates[UpInitMaintenance] = "uim"; - for (uint32_t i=0; i<_upStates.size(); ++i) { - if (_upStates[i] == 0) { - throw vespalib::IllegalStateException("Failed to initialize up state. Code likely not updated " - "after another upstate was added.", VESPA_STRLOC); - } - } -} - -} diff --git a/vdslib/src/vespa/vdslib/distribution/idealnodecalculatorimpl.h b/vdslib/src/vespa/vdslib/distribution/idealnodecalculatorimpl.h deleted file mode 100644 index 9b36f1094fd..00000000000 --- a/vdslib/src/vespa/vdslib/distribution/idealnodecalculatorimpl.h +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * A cache for an ideal nodes implementation. Making it cheap for localized - * access, regardless of real implementation. - */ -#pragma once - -#include "idealnodecalculator.h" - -namespace storage::lib { - -class IdealNodeCalculatorImpl : public IdealNodeCalculatorConfigurable { - std::vector<const char*> _upStates; - const Distribution* _distribution; - const ClusterState* _clusterState; - -public: - IdealNodeCalculatorImpl(); - ~IdealNodeCalculatorImpl(); - - void setDistribution(const Distribution& d) override; - void setClusterState(const ClusterState& cs) override; - - IdealNodeList getIdealNodes(const NodeType& nodeType, - const document::BucketId& bucket, - UpStates upStates) const override; -private: - void initUpStateMapping(); -}; - -} diff --git a/vespa-dependencies-enforcer/allowed-maven-dependencies.txt b/vespa-dependencies-enforcer/allowed-maven-dependencies.txt index cb2806b66d8..581ccd1d317 100644 --- a/vespa-dependencies-enforcer/allowed-maven-dependencies.txt +++ b/vespa-dependencies-enforcer/allowed-maven-dependencies.txt @@ -37,7 +37,7 @@ com.google.http-client:google-http-client-apache-v2:1.43.3 com.google.http-client:google-http-client-gson:1.42.3 com.google.inject:guice:4.2.3:no_aop com.google.j2objc:j2objc-annotations:2.8 -com.google.protobuf:protobuf-java:3.21.7 +com.google.protobuf:protobuf-java:3.24.0 com.ibm.icu:icu4j:70.1 com.intellij:annotations:9.0.4 com.microsoft.onnxruntime:onnxruntime:1.13.1 diff --git a/vespalib/src/tests/guard/guard_test.cpp b/vespalib/src/tests/guard/guard_test.cpp index 9e5e7e55cc6..c61c4874eff 100644 --- a/vespalib/src/tests/guard/guard_test.cpp +++ b/vespalib/src/tests/guard/guard_test.cpp @@ -7,20 +7,7 @@ using namespace vespalib; -class Test : public TestApp -{ -public: - void testFilePointer(); - void testFileDescriptor(); - void testDirPointer(); - void testValueGuard(); - void testMaxValueGuard(); - void testCounterGuard(); - int Main() override; -}; - -void -Test::testFilePointer() +TEST("testFilePointer") { { FilePointer file(fopen("bogus", "r")); @@ -72,8 +59,7 @@ Test::testFilePointer() } } -void -Test::testFileDescriptor() +TEST("testFileDescriptor") { { FileDescriptor file(open("bogus", O_RDONLY)); @@ -126,124 +112,7 @@ Test::testFileDescriptor() } } -void -Test::testDirPointer() -{ - { - DirPointer dir(opendir("bogus")); - EXPECT_TRUE(!dir.valid()); - } - { - DirPointer dir(opendir(TEST_PATH("").c_str())); - EXPECT_TRUE(dir.valid()); - - dirent *de; - bool foundGuardCpp = false; - while ((de = readdir(dir)) != NULL) { - if (strcmp(de->d_name, "guard_test.cpp") == 0) { - foundGuardCpp = true; - } - } - EXPECT_TRUE(foundGuardCpp); - } - { - DIR *dp = NULL; - { - DirPointer dir(opendir(".")); - EXPECT_TRUE(dir.valid()); - dp = dir; - } - EXPECT_TRUE(dp != NULL); - // EXPECT_TRUE(readdir(dp) == NULL); - } - { - DirPointer dir(opendir(".")); - EXPECT_TRUE(dir.valid()); - dir.reset(opendir(".")); - EXPECT_TRUE(dir.valid()); - - DIR *ref = dir.dp(); - DIR *dp = dir.release(); - EXPECT_TRUE(dp != NULL); - EXPECT_TRUE(dp == ref); - EXPECT_TRUE(!dir.valid()); - EXPECT_TRUE(dir.dp() == NULL); - closedir(dp); - } -} - -void -Test::testValueGuard() -{ - int value = 10; - { - ValueGuard<int> guard(value); - value = 20; - EXPECT_TRUE(value == 20); - } - EXPECT_TRUE(value == 10); - { - ValueGuard<int> guard(value, 50); - value = 20; - EXPECT_TRUE(value == 20); - } - EXPECT_TRUE(value == 50); - { - ValueGuard<int> guard(value); - value = 20; - guard.update(100); - EXPECT_TRUE(value == 20); - } - EXPECT_TRUE(value == 100); - { - ValueGuard<int> guard(value); - value = 20; - guard.dismiss(); - EXPECT_TRUE(value == 20); - } - EXPECT_TRUE(value == 20); -} - -void -Test::testMaxValueGuard() -{ - int value = 10; - { - MaxValueGuard<int> guard(value); - value = 20; - EXPECT_TRUE(value == 20); - } - EXPECT_TRUE(value == 10); - { - MaxValueGuard<int> guard(value); - value = 5; - EXPECT_TRUE(value == 5); - } - EXPECT_TRUE(value == 5); - { - MaxValueGuard<int> guard(value, 50); - value = 100; - EXPECT_TRUE(value == 100); - } - EXPECT_TRUE(value == 50); - { - MaxValueGuard<int> guard(value); - value = 200; - guard.update(100); - EXPECT_TRUE(value == 200); - } - EXPECT_TRUE(value == 100); - { - MaxValueGuard<int> guard(value); - value = 200; - guard.dismiss(); - EXPECT_TRUE(value == 200); - } - EXPECT_TRUE(value == 200); -} - -void -Test::testCounterGuard() +TEST("testCounterGuard") { int cnt = 10; { @@ -254,17 +123,4 @@ Test::testCounterGuard() EXPECT_TRUE(cnt == 10); } -int -Test::Main() -{ - TEST_INIT("guard_test"); - testFilePointer(); - testFileDescriptor(); - testDirPointer(); - testValueGuard(); - testMaxValueGuard(); - testCounterGuard(); - TEST_DONE(); -} - -TEST_APPHOOK(Test) +TEST_MAIN() { TEST_RUN_ALL(); }
\ No newline at end of file diff --git a/vespalib/src/tests/io/fileutil/fileutiltest.cpp b/vespalib/src/tests/io/fileutil/fileutiltest.cpp index 0948d18304e..93803c1fe9e 100644 --- a/vespalib/src/tests/io/fileutil/fileutiltest.cpp +++ b/vespalib/src/tests/io/fileutil/fileutiltest.cpp @@ -102,15 +102,6 @@ TEST("require that vespalib::File::open works") ASSERT_TRUE(fileExists("mydir/myfile")); f.unlink(); } - // Opening with direct IO support works. - { - File f("mydir/myfile"); - f.open(File::CREATE | File::DIRECTIO, false); - ASSERT_TRUE(fileExists("mydir/myfile")); - if (!f.isOpenWithDirectIO()) { - std::cerr << "This platform does not support direct IO\n"; - } - } // Opening plain file works { File f("myfile"); @@ -126,16 +117,6 @@ TEST("require that vespalib::File::open works") //std::cerr << e.what() << "\n"; EXPECT_EQUAL(IoException::ILLEGAL_PATH, e.getType()); } - // Test opening already open file - { - std::unique_ptr<File> f(new File("myfile")); - f->open(File::CREATE, false); - f->closeFileWhenDestructed(false); - File f2(f->getFileDescriptor(), "myfile"); - f.reset(); - ASSERT_TRUE(f2.isOpen()); - f2.write(" ", 1, 0); - } // Test reopening file in same object { File f("myfile"); @@ -161,29 +142,6 @@ TEST("require that vespalib::File::isOpen works") ASSERT_TRUE(!f.isOpen()); } -TEST("require that vespalib::File::stat works") -{ - std::filesystem::remove(std::filesystem::path("myfile")); - std::filesystem::remove_all(std::filesystem::path("mydir")); - EXPECT_EQUAL(false, fileExists("myfile")); - EXPECT_EQUAL(false, fileExists("mydir")); - std::filesystem::create_directory(std::filesystem::path("mydir")); - File f("myfile"); - f.open(File::CREATE, false); - f.write("foobar", 6, 0); - - FileInfo info = f.stat(); - EXPECT_EQUAL(6, info._size); - EXPECT_EQUAL(true, info._plainfile); - EXPECT_EQUAL(false, info._directory); - - EXPECT_EQUAL(6, f.getFileSize()); - f.close(); - - EXPECT_EQUAL(true, fileExists("myfile")); - EXPECT_EQUAL(true, fileExists("mydir")); -} - TEST("require that vespalib::File::resize works") { std::filesystem::remove(std::filesystem::path("myfile")); @@ -204,47 +162,6 @@ TEST("require that vespalib::File::resize works") EXPECT_EQUAL(std::string("foo"), std::string(&vec[0], 3)); } -TEST("require that copy constructor and assignment for vespalib::File works") -{ - // Copy file not opened. - { - File f("myfile"); - File f2(f); - EXPECT_EQUAL(f.getFilename(), f2.getFilename()); - } - // Copy file opened - { - File f("myfile"); - f.open(File::CREATE); - File f2(f); - EXPECT_EQUAL(f.getFilename(), f2.getFilename()); - ASSERT_TRUE(f2.isOpen()); - ASSERT_TRUE(!f.isOpen()); - } - // Assign file opened to another file opened - { - File f("myfile"); - f.open(File::CREATE); - int fd = f.getFileDescriptor(); - File f2("targetfile"); - f2.open(File::CREATE); - f = f2; - EXPECT_EQUAL(std::string("targetfile"), f2.getFilename()); - EXPECT_EQUAL(f.getFilename(), f2.getFilename()); - ASSERT_TRUE(!f2.isOpen()); - ASSERT_TRUE(f.isOpen()); - try{ - File f3(fd, "myfile"); - f3.closeFileWhenDestructed(false); // Already closed - f3.write("foo", 3, 0); - TEST_FATAL("This file descriptor should have been closed"); - } catch (IoException& e) { - //std::cerr << e.what() << "\n"; - EXPECT_EQUAL(IoException::INTERNAL_FAILURE, e.getType()); - } - } -} - TEST("require that we can read all data written to file") { // Write text into a file. diff --git a/vespalib/src/vespa/vespalib/io/fileutil.cpp b/vespalib/src/vespa/vespalib/io/fileutil.cpp index 6c169ab8d98..cb478f0f225 100644 --- a/vespalib/src/vespa/vespalib/io/fileutil.cpp +++ b/vespalib/src/vespa/vespalib/io/fileutil.cpp @@ -5,7 +5,6 @@ #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/util/size_literals.h> #include <vespa/vespalib/util/stringfmt.h> -#include <ostream> #include <cassert> #include <filesystem> #include <dirent.h> @@ -16,35 +15,34 @@ #include <vespa/log/log.h> LOG_SETUP(".vespalib.io.fileutil"); +namespace fs = std::filesystem; + namespace vespalib { namespace { - FileInfo::UP - processStat(struct stat& filestats, bool result, stringref path) { - FileInfo::UP resval; - if (result) { - resval.reset(new FileInfo); - resval->_plainfile = S_ISREG(filestats.st_mode); - resval->_directory = S_ISDIR(filestats.st_mode); - resval->_symlink = S_ISLNK(filestats.st_mode); - resval->_size = filestats.st_size; - } else if (errno != ENOENT) { - asciistream ost; - ost << "An IO error occured while statting '" << path << "'. " - << "errno(" << errno << "): " << getErrorString(errno); - throw IoException(ost.str(), IoException::getErrorType(errno), - VESPA_STRLOC); - } - LOG(debug, "stat(%s): Existed? %s, Plain file? %s, Directory? %s, " - "Size: %" PRIu64, - string(path).c_str(), - resval.get() ? "true" : "false", - resval.get() && resval->_plainfile ? "true" : "false", - resval.get() && resval->_directory ? "true" : "false", - resval.get() ? resval->_size : 0); - return resval; +FileInfo::UP +processStat(struct stat& filestats, bool result, stringref path) { + FileInfo::UP resval; + if (result) { + resval = std::make_unique<FileInfo>(); + resval->_plainfile = S_ISREG(filestats.st_mode); + resval->_directory = S_ISDIR(filestats.st_mode); + resval->_size = filestats.st_size; + } else if (errno != ENOENT) { + asciistream ost; + ost << "An IO error occured while statting '" << path << "'. " + << "errno(" << errno << "): " << getErrorString(errno); + throw IoException(ost.str(), IoException::getErrorType(errno), VESPA_STRLOC); } + LOG(debug, "stat(%s): Existed? %s, Plain file? %s, Directory? %s, Size: %" PRIu64, + string(path).c_str(), + resval.get() ? "true" : "false", + resval.get() && resval->_plainfile ? "true" : "false", + resval.get() && resval->_directory ? "true" : "false", + resval.get() ? resval->_size : 0); + return resval; +} string safeStrerror(int errnum) @@ -54,167 +52,61 @@ safeStrerror(int errnum) } -bool -FileInfo::operator==(const FileInfo& fi) const -{ - return (_size == fi._size && _plainfile == fi._plainfile - && _directory == fi._directory); -} - -std::ostream& -operator<<(std::ostream& out, const FileInfo& info) -{ - out << "FileInfo(size: " << info._size; - if (info._plainfile) out << ", plain file"; - if (info._directory) out << ", directory"; - out << ")"; - return out; -} - File::File(stringref filename) : _fd(-1), - _flags(0), - _filename(filename), - _close(true), - _fileReads(0), - _fileWrites(0) -{ -} - -File::File(int fileDescriptor, stringref filename) - : _fd(fileDescriptor), - _flags(0), - _filename(filename), - _close(true), - _fileReads(0), - _fileWrites(0) -{ -} + _filename(filename) +{ } File::~File() { - if (_close && _fd != -1) close(); -} - -File::File(File& f) - : _fd(f._fd), - _flags(f._flags), - _filename(f._filename), - _close(f._close), - _fileReads(f._fileReads), - _fileWrites(f._fileWrites) -{ - f._fd = -1; - f._flags = 0; - f._close = true; - f._fileReads = 0; - f._fileWrites = 0; -} - -File& -File::operator=(File& f) -{ - if (_close && _fd != -1) close(); - _fd = f._fd; - _flags = f._flags; - _filename = f._filename; - _close = f._close; - _fileReads = f._fileReads; - _fileWrites = f._fileWrites; - f._fd = -1; - f._flags = 0; - f._close = true; - f._fileReads = 0; - f._fileWrites = 0; - return *this; -} - -void -File::setFilename(stringref filename) -{ - if (_filename == filename) return; - if (_close && _fd != -1) close(); - _filename = filename; - _fd = -1; - _flags = 0; - _close = true; + if (_fd != -1) close(); } namespace { - int openAndCreateDirsIfMissing(const string & filename, int flags, - bool createDirsIfMissing) +int openAndCreateDirsIfMissing(const string & filename, int flags, bool createDirsIfMissing) +{ + int fd = ::open(filename.c_str(), flags, 0644); + if (fd < 0 && errno == ENOENT && ((flags & O_CREAT) != 0) + && createDirsIfMissing) { - int fd = ::open(filename.c_str(), flags, 0644); - if (fd < 0 && errno == ENOENT && ((flags & O_CREAT) != 0) - && createDirsIfMissing) - { - auto pos = filename.rfind('/'); - if (pos != string::npos) { - string path(filename.substr(0, pos)); - std::filesystem::create_directories(std::filesystem::path(path)); - LOG(spam, "open(%s, %d): Retrying open after creating parent " - "directories.", filename.c_str(), flags); - fd = ::open(filename.c_str(), flags, 0644); - } + auto pos = filename.rfind('/'); + if (pos != string::npos) { + string path(filename.substr(0, pos)); + fs::create_directories(fs::path(path)); + LOG(spam, "open(%s, %d): Retrying open after creating parent directories.", filename.c_str(), flags); + fd = ::open(filename.c_str(), flags, 0644); } - return fd; } + return fd; +} } void File::open(int flags, bool autoCreateDirectories) { if ((flags & File::READONLY) != 0) { if ((flags & File::CREATE) != 0) { - throw IllegalArgumentException( - "Cannot use READONLY and CREATE options at the same time", - VESPA_STRLOC); + throw IllegalArgumentException("Cannot use READONLY and CREATE options at the same time", VESPA_STRLOC); } if ((flags & File::TRUNC) != 0) { - throw IllegalArgumentException( - "Cannot use READONLY and TRUNC options at the same time", - VESPA_STRLOC); + throw IllegalArgumentException("Cannot use READONLY and TRUNC options at the same time", VESPA_STRLOC); } if (autoCreateDirectories) { - throw IllegalArgumentException( - "No point in auto-creating directories on read only access", - VESPA_STRLOC); + throw IllegalArgumentException("No point in auto-creating directories on read only access", VESPA_STRLOC); } } int openflags = ((flags & File::READONLY) != 0 ? O_RDONLY : O_RDWR) | ((flags & File::CREATE) != 0 ? O_CREAT : 0) -#ifdef __linux__ - | ((flags & File::DIRECTIO) != 0 ? O_DIRECT : 0) -#endif | ((flags & File::TRUNC) != 0 ? O_TRUNC: 0); int fd = openAndCreateDirsIfMissing(_filename, openflags, autoCreateDirectories); -#ifdef __linux__ - if (fd < 0 && ((flags & File::DIRECTIO) != 0)) { - openflags = (openflags ^ O_DIRECT); - flags = (flags ^ DIRECTIO); - LOG(debug, "open(%s, %d): Retrying without direct IO due to failure " - "opening with errno(%d): %s", - _filename.c_str(), flags, errno, safeStrerror(errno).c_str()); - fd = openAndCreateDirsIfMissing(_filename, openflags, autoCreateDirectories); - } -#endif if (fd < 0) { asciistream ost; - ost << "open(" << _filename << ", 0x" - << hex << flags << dec << "): Failed, errno(" << errno - << "): " << safeStrerror(errno); + ost << "open(" << _filename << ", 0x" << hex << flags << dec + << "): Failed, errno(" << errno << "): " << safeStrerror(errno); throw IoException(ost.str(), IoException::getErrorType(errno), VESPA_STRLOC); } - _flags = flags; - if (_close && _fd != -1) close(); + if (_fd != -1) close(); _fd = fd; - LOG(debug, "open(%s, %d). File opened with file descriptor %d.", - _filename.c_str(), flags, fd); -} - -void -File::closeFileWhenDestructed(bool closeOnDestruct) -{ - _close = closeOnDestruct; + LOG(debug, "open(%s, %d). File opened with file descriptor %d.", _filename.c_str(), flags, fd); } FileInfo @@ -226,13 +118,11 @@ File::stat() const result = processStat(filestats, fstat(_fd, &filestats) == 0, _filename); assert(result.get()); // The file must exist in a file instance } else { - result = processStat(filestats, - ::stat(_filename.c_str(), &filestats) == 0, - _filename); + result = processStat(filestats, ::stat(_filename.c_str(), &filestats) == 0, _filename); // If the file does not exist yet, act like it does. It will // probably be created when opened. - if (result.get() == 0) { - result.reset(new FileInfo()); + if ( ! result) { + result = std::make_unique<FileInfo>(); result->_size = 0; result->_directory = false; result->_plainfile = true; @@ -246,69 +136,32 @@ File::resize(off_t size) { if (ftruncate(_fd, size) != 0) { asciistream ost; - ost << "resize(" << _filename << ", " << size << "): Failed, errno(" - << errno << "): " << safeStrerror(errno); + ost << "resize(" << _filename << ", " << size << "): Failed, errno(" << errno << "): " << safeStrerror(errno); throw IoException(ost.str(), IoException::getErrorType(errno), VESPA_STRLOC); } - LOG(debug, "resize(%s): Resized to %" PRIu64 " bytes.", - _filename.c_str(), size); -} - -void -File::verifyDirectIO(uint64_t buf, size_t bufsize, off_t offset) const -{ - if (offset % 512 != 0) { - LOG(error, - "Access to file %s failed because offset %" PRIu64 " wasn't 512-byte " - "aligned. Buffer memory address was %" PRIx64 ", length %zu", - _filename.c_str(), static_cast<uint64_t>(offset), buf, bufsize); - assert(false); - } - if (buf % 512 != 0) { - LOG(error, - "Access to file %s failed because buffer memory address %" PRIx64 " " - "wasn't 512-byte aligned. Offset was %" PRIu64 ", length %zu", - _filename.c_str(), buf, static_cast<uint64_t>(offset), bufsize); - assert(false); - } - if (bufsize % 512 != 0) { - LOG(error, - "Access to file %s failed because buffer size %zu wasn't 512-byte " - "aligned. Buffer memory address was %" PRIx64 ", offset %" PRIu64, - _filename.c_str(), bufsize, buf, static_cast<uint64_t>(offset)); - assert(false); - } + LOG(debug, "resize(%s): Resized to %" PRIu64 " bytes.", _filename.c_str(), size); } off_t File::write(const void *buf, size_t bufsize, off_t offset) { - ++_fileWrites; size_t left = bufsize; - LOG(debug, "write(%s): Writing %zu bytes at offset %" PRIu64 ".", - _filename.c_str(), bufsize, offset); - - if (_flags & DIRECTIO) { - verifyDirectIO((uint64_t)buf, bufsize, offset); - } + LOG(debug, "write(%s): Writing %zu bytes at offset %" PRIu64 ".", _filename.c_str(), bufsize, offset); while (left > 0) { ssize_t written = ::pwrite(_fd, buf, left, offset); if (written > 0) { - LOG(spam, "write(%s): Wrote %zd bytes at offset %" PRIu64 ".", - _filename.c_str(), written, offset); + LOG(spam, "write(%s): Wrote %zd bytes at offset %" PRIu64 ".", _filename.c_str(), written, offset); left -= written; buf = ((const char*) buf) + written; offset += written; } else if (written == 0) { - LOG(spam, "write(%s): Wrote %zd bytes at offset %" PRIu64 ".", - _filename.c_str(), written, offset); + LOG(spam, "write(%s): Wrote %zd bytes at offset %" PRIu64 ".", _filename.c_str(), written, offset); assert(false); // Can this happen? } else if (errno != EINTR && errno != EAGAIN) { asciistream ost; - ost << "write(" << _fd << ", " << buf - << ", " << left << ", " << offset << "), Failed, errno(" - << errno << "): " << safeStrerror(errno); + ost << "write(" << _fd << ", " << buf << ", " << left << ", " << offset + << "), Failed, errno(" << errno << "): " << safeStrerror(errno); throw IoException(ost.str(), IoException::getErrorType(errno), VESPA_STRLOC); } } @@ -318,37 +171,23 @@ File::write(const void *buf, size_t bufsize, off_t offset) size_t File::read(void *buf, size_t bufsize, off_t offset) const { - ++_fileReads; size_t remaining = bufsize; - LOG(debug, "read(%s): Reading %zu bytes from offset %" PRIu64 ".", - _filename.c_str(), bufsize, offset); - - if (_flags & DIRECTIO) { - verifyDirectIO((uint64_t)buf, bufsize, offset); - } + LOG(debug, "read(%s): Reading %zu bytes from offset %" PRIu64 ".", _filename.c_str(), bufsize, offset); while (remaining > 0) { ssize_t bytesread = ::pread(_fd, buf, remaining, offset); if (bytesread > 0) { - LOG(spam, "read(%s): Read %zd bytes from offset %" PRIu64 ".", - _filename.c_str(), bytesread, offset); + LOG(spam, "read(%s): Read %zd bytes from offset %" PRIu64 ".", _filename.c_str(), bytesread, offset); remaining -= bytesread; buf = ((char*) buf) + bytesread; offset += bytesread; - if (((_flags & DIRECTIO) != 0) && ((bytesread % 512) != 0) && (offset == getFileSize())) { - LOG(spam, "read(%s): Found EOF. Directio read to unaligned file end at offset %" PRIu64 ".", - _filename.c_str(), offset); - break; - } } else if (bytesread == 0) { // EOF - LOG(spam, "read(%s): Found EOF. Zero bytes read from offset %" PRIu64 ".", - _filename.c_str(), offset); + LOG(spam, "read(%s): Found EOF. Zero bytes read from offset %" PRIu64 ".", _filename.c_str(), offset); break; } else if (errno != EINTR && errno != EAGAIN) { asciistream ost; ost << "read(" << _fd << ", " << buf << ", " << remaining << ", " - << offset << "): Failed, errno(" << errno << "): " - << safeStrerror(errno); + << offset << "): Failed, errno(" << errno << "): " << safeStrerror(errno); throw IoException(ost.str(), IoException::getErrorType(errno), VESPA_STRLOC); } } @@ -433,13 +272,7 @@ bool File::unlink() { close(); - return std::filesystem::remove(std::filesystem::path(_filename)); -} - -namespace { - - uint32_t diskAlignmentSize = 4_Ki; - + return fs::remove(fs::path(_filename)); } DirectoryList @@ -465,16 +298,6 @@ listDirectory(const string & path) return result; } -MallocAutoPtr -getAlignedBuffer(size_t size) -{ - void *ptr; - int result = posix_memalign(&ptr, diskAlignmentSize, size); - assert(result == 0); - (void)result; - return MallocAutoPtr(ptr); -} - string dirname(stringref name) { size_t found = name.rfind('/'); @@ -517,8 +340,7 @@ getOpenErrorString(const int osError, stringref filename) { asciistream os; string dirName(dirname(filename)); - os << "error=" << osError << "(\"" << - getErrorString(osError) << "\") fileStat"; + os << "error=" << osError << "(\"" << getErrorString(osError) << "\") fileStat"; addStat(os, filename); os << " dirStat"; addStat(os, dirName); diff --git a/vespalib/src/vespa/vespalib/io/fileutil.h b/vespalib/src/vespa/vespalib/io/fileutil.h index 4de36daa85f..148317a7edf 100644 --- a/vespalib/src/vespa/vespalib/io/fileutil.h +++ b/vespalib/src/vespa/vespalib/io/fileutil.h @@ -43,14 +43,10 @@ struct FileInfo { bool _plainfile; bool _directory; - bool _symlink; off_t _size; - bool operator==(const FileInfo&) const; }; -std::ostream& operator<<(std::ostream&, const FileInfo&); - /** * @brief A File instance is used to access a single open file. * @@ -61,74 +57,44 @@ std::ostream& operator<<(std::ostream&, const FileInfo&); */ class File { private: - int _fd; - int _flags; - vespalib::string _filename; - bool _close; - mutable int _fileReads; // Tracks number of file reads done on this file - mutable int _fileWrites; // Tracks number of file writes done in this file + int _fd; + string _filename; + void sync(); /** - * Verify that direct I/O alignment preconditions hold. Triggers assertion - * failure on violations. + * Get information about the current file. If file is opened, file descriptor + * will be used for stat. If file is not open, and the file does not exist + * yet, you will get fileinfo describing an empty file. */ - void verifyDirectIO(uint64_t buf, size_t bufsize, off_t offset) const; - + FileInfo stat() const; public: using UP = std::unique_ptr<File>; /** * If failing to open file using direct IO it will retry using cached IO. */ - enum Flag { READONLY = 1, CREATE = 2, DIRECTIO = 4, TRUNC = 8 }; + enum Flag { READONLY = 1, CREATE = 2, TRUNC = 8 }; /** Create a file instance, without opening the file. */ - File(vespalib::stringref filename); - - /** Create a file instance of an already open file. */ - File(int fileDescriptor, vespalib::stringref filename); - - /** Copying a file instance, moves any open file descriptor. */ - File(File& f); - File& operator=(File& f); + File(stringref filename); /** Closes the file if not instructed to do otherwise. */ - virtual ~File(); + ~File(); - /** - * Make this instance point at another file. - * Closes the old file it it was open. - */ - void setFilename(vespalib::stringref filename); + const string& getFilename() const { return _filename; } - const vespalib::string& getFilename() const { return _filename; } - - virtual void open(int flags, bool autoCreateDirectories = false); + void open(int flags, bool autoCreateDirectories = false); bool isOpen() const { return (_fd != -1); } - bool isOpenWithDirectIO() const { return ((_flags & DIRECTIO) != 0); } - - /** - * Whether or not file should be closed when this instance is destructed. - * By default it will be closed. - */ - void closeFileWhenDestructed(bool close); - virtual int getFileDescriptor() const { return _fd; } - - /** - * Get information about the current file. If file is opened, file descriptor - * will be used for stat. If file is not open, and the file does not exist - * yet, you will get fileinfo describing an empty file. - */ - virtual FileInfo stat() const; + int getFileDescriptor() const { return _fd; } /** * Get the filesize of a file, specified by a file descriptor. * * @throw IoException If we failed to stat the file. */ - virtual off_t getFileSize() const { return stat()._size; } + off_t getFileSize() const { return stat()._size; } /** * Resize the currently open file to a given size, @@ -138,7 +104,7 @@ public: * @param size new size of file * @throw IoException If we failed to resize the file. */ - virtual void resize(off_t size); + void resize(off_t size); /** * Writes data to file. @@ -152,7 +118,7 @@ public: * @throw IoException If we failed to write to the file. * @return Always return bufsize. */ - virtual off_t write(const void *buf, size_t bufsize, off_t offset); + off_t write(const void *buf, size_t bufsize, off_t offset); /** * Read characters from a file. @@ -167,7 +133,7 @@ public: * @return The number of bytes actually read. If less than * bufsize, this indicates that EOF was reached. */ - virtual size_t read(void *buf, size_t bufsize, off_t offset) const; + size_t read(void *buf, size_t bufsize, off_t offset) const; /** * Read the file into a string. @@ -177,7 +143,7 @@ public: * @throw IoException If we failed to read from file. * @return The content of the file. */ - vespalib::string readAll() const; + string readAll() const; /** * Read a file into a string. @@ -188,7 +154,7 @@ public: * @throw IoException If we failed to read from file. * @return The content of the file. */ - static vespalib::string readAll(vespalib::stringref path); + static string readAll(stringref path); /** * Sync file or directory. @@ -198,24 +164,17 @@ public: * * @throw IoException If we failed to sync the file. */ - static void sync(vespalib::stringref path); - - virtual void sync(); - virtual bool close(); - virtual bool unlink(); + static void sync(stringref path); - int getFileReadCount() const { return _fileReads; } - int getFileWriteCount() const { return _fileWrites; } + bool close(); + bool unlink(); }; /** * List the contents of the given directory. */ -using DirectoryList = std::vector<vespalib::string>; -extern DirectoryList listDirectory(const vespalib::string & path); - -extern MallocAutoPtr getAlignedBuffer(size_t size); - +using DirectoryList = std::vector<string>; +extern DirectoryList listDirectory(const string & path); string dirname(stringref name); string getOpenErrorString(const int osError, stringref name); diff --git a/vespalib/src/vespa/vespalib/stllike/hash_map.cpp b/vespalib/src/vespa/vespalib/stllike/hash_map.cpp index abb88fe674f..50a3d73fe12 100644 --- a/vespalib/src/vespa/vespalib/stllike/hash_map.cpp +++ b/vespalib/src/vespa/vespalib/stllike/hash_map.cpp @@ -16,6 +16,7 @@ VESPALIB_HASH_MAP_INSTANTIATE(vespalib::string, double); VESPALIB_HASH_MAP_INSTANTIATE(int64_t, int32_t); VESPALIB_HASH_MAP_INSTANTIATE(int64_t, uint32_t); VESPALIB_HASH_MAP_INSTANTIATE(int32_t, uint32_t); +VESPALIB_HASH_MAP_INSTANTIATE(uint16_t, uint16_t); VESPALIB_HASH_MAP_INSTANTIATE(uint16_t, uint32_t); VESPALIB_HASH_MAP_INSTANTIATE(uint32_t, int32_t); VESPALIB_HASH_MAP_INSTANTIATE(uint32_t, uint32_t); diff --git a/vespalib/src/vespa/vespalib/stllike/string.hpp b/vespalib/src/vespa/vespalib/stllike/string.hpp index e0144ab6f85..3438c6b641a 100644 --- a/vespalib/src/vespa/vespalib/stllike/string.hpp +++ b/vespalib/src/vespa/vespalib/stllike/string.hpp @@ -17,8 +17,10 @@ void small_string<StackSize>::_reserveBytes(size_type newBufferSize) noexcept { if (isAllocated()) { _buf = (char *) realloc(_buf, newBufferSize); + assert(_buf); } else { char *tmp = (char *) malloc(newBufferSize); + assert(tmp); memcpy(tmp, _stack, _sz); tmp[_sz] = '\0'; _buf = tmp; @@ -96,6 +98,7 @@ void small_string<StackSize>::init_slower(const void *s) noexcept { _bufferSize = _sz+1; _buf = (char *) malloc(_bufferSize); + assert(_buf); memcpy(_buf, s, _sz); _buf[_sz] = '\0'; } @@ -105,6 +108,7 @@ void small_string<StackSize>::appendAlloc(const void * s, size_type addSz) noexc { size_type newBufferSize = roundUp2inN(_sz+addSz+1); char * buf = (char *) malloc(newBufferSize); + assert(buf); memcpy(buf, buffer(), _sz); if (isAllocated()) { free(_buf); diff --git a/vespalib/src/vespa/vespalib/util/guard.h b/vespalib/src/vespa/vespalib/util/guard.h index 32237a59d9a..efd7b8345c9 100644 --- a/vespalib/src/vespa/vespalib/util/guard.h +++ b/vespalib/src/vespa/vespalib/util/guard.h @@ -2,8 +2,7 @@ #pragma once -#include <stdio.h> -#include <dirent.h> +#include <cstdio> #include <unistd.h> namespace vespalib { @@ -19,43 +18,43 @@ class FilePointer { private: FILE *_fp; - FilePointer(const FilePointer &); - FilePointer &operator=(const FilePointer &); public: /** * @brief Create a FilePointer from a FILE pointer. * * @param file the underlying FILE pointer **/ - explicit FilePointer(FILE *file = NULL) : _fp(file) {} + explicit FilePointer(FILE *file = nullptr) noexcept : _fp(file) {} + FilePointer(const FilePointer &) = delete; + FilePointer &operator=(const FilePointer &) = delete; /** * @brief Close the file if it is still open. **/ ~FilePointer() { reset(); } /** - * @brief Check whether we have a FILE pointer (not NULL) + * @brief Check whether we have a FILE pointer (not nullptr) * * @return true if we have an underlying FILE pointer **/ - bool valid() const { return (_fp != NULL); } + bool valid() const noexcept { return (_fp != nullptr); } /** * @brief Obtain the internal FILE pointer * * @return internal FILE pointer **/ - FILE *fp() const { return _fp; } + FILE *fp() const noexcept { return _fp; } /** * @brief Implicit cast to obtain internal FILE pointer * * @return internal FILE pointer **/ - operator FILE*() { return _fp; } + operator FILE*() noexcept { return _fp; } /** * @brief Take ownership of a new FILE pointer. * * The previously owned FILE pointer is closed, if present. **/ - void reset(FILE *file = NULL) { + void reset(FILE *file = nullptr) { if (valid()) { fclose(_fp); } @@ -68,81 +67,13 @@ public: * * @return the released FILE pointer **/ - FILE *release() { + FILE *release() noexcept { FILE *tmp = _fp; - _fp = NULL; + _fp = nullptr; return tmp; } }; - -/** - * @brief A DirPointer wraps a bald DIR pointer inside a guarding object. - * - * The underlying directory is closed when the DirPointer object is - * destructed. - **/ -class DirPointer -{ -private: - DIR *_dp; - DirPointer(const DirPointer &); - DirPointer &operator=(const DirPointer &); -public: - /** - * @brief Create a DirPointer from a DIR pointer. - * - * @param dir the underlying DIR pointer - **/ - explicit DirPointer(DIR *dir = NULL) : _dp(dir) {} - /** - * Close the directory if it is still open. - **/ - ~DirPointer() { reset(); } - /** - * @brief Check whether we have a DIR pointer (not NULL) - * - * @return true if we have an underlying DIR pointer - **/ - bool valid() const { return (_dp != NULL); } - /** - * @brief Obtain the internal DIR pointer - * - * @return internal DIR pointer - **/ - DIR *dp() const { return _dp; } - /** - * @brief Implicit cast to obtain internal DIR pointer - * - * @return internal DIR pointer - **/ - operator DIR*() { return _dp; } - /** - * @brief Take ownership of a new DIR pointer. - * - * The previously owned DIR pointer is closed, if present. - **/ - void reset(DIR *dir = NULL) { - if (valid()) { - closedir(_dp); - } - _dp = dir; - } - /** - * @brief Release ownership of the current DIR pointer. - * - * The directory will no longer be closed by the destructor. - * - * @return the released DIR pointer - **/ - DIR *release() { - DIR *tmp = _dp; - _dp = NULL; - return tmp; - } -}; - - /** * @brief A FileDescriptor wraps a file descriptor inside a guarding object. * @@ -153,15 +84,15 @@ class FileDescriptor { private: int _fd; - FileDescriptor(const FileDescriptor &); - FileDescriptor &operator=(const FileDescriptor &); public: /** * @brief Create a FileDescriptor from a file descriptor. * * @param file the underlying file descriptor **/ - explicit FileDescriptor(int file = -1) : _fd(file) {} + explicit FileDescriptor(int file = -1) noexcept : _fd(file) {} + FileDescriptor(const FileDescriptor &) = delete; + FileDescriptor &operator=(const FileDescriptor &) = delete; /** * @brief Close the file if it is still open. **/ @@ -171,13 +102,13 @@ public: * * @return true if we have an underlying file descriptor **/ - bool valid() const { return (_fd >= 0); } + bool valid() const noexcept { return (_fd >= 0); } /** * @brief Obtain the internal file descriptor * * @return internal file descriptor **/ - int fd() const { return _fd; } + int fd() const noexcept { return _fd; } /** * @brief Take ownership of a new file descriptor. * @@ -196,7 +127,7 @@ public: * * @return the released file descriptor **/ - int release() { + int release() noexcept { int tmp = _fd; _fd = -1; return tmp; @@ -216,161 +147,20 @@ class CounterGuard { private: int &_cnt; - CounterGuard(const CounterGuard &); - CounterGuard &operator=(const CounterGuard &); public: /** * @brief Increase the value * * @param cnt a reference to the value that will be modified **/ - explicit CounterGuard(int &cnt) : _cnt(cnt) { ++cnt; } + explicit CounterGuard(int &cnt) noexcept : _cnt(cnt) { ++cnt; } + CounterGuard(const CounterGuard &) = delete; + CounterGuard &operator=(const CounterGuard &) = delete; /** * @brief Decrease the value **/ ~CounterGuard() { --_cnt; } }; - -/** - * @brief A ValueGuard is used to set a variable to a specific value - * when the ValueGuard is destructed. - * - * This can be used to revert a variable if an exception is thrown. - * However, you must remember to dismiss the guard if you don't want - * it to set the value when it goes out of scope. - **/ -template<typename T> -class ValueGuard -{ -private: - bool _active; - T &_ref; - T _value; - - ValueGuard(const ValueGuard &); - ValueGuard &operator=(const ValueGuard &); -public: - /** - * @brief Create a ValueGuard for the given variable. - * - * The variable will be reverted to its original value in the destructor. - * - * @param ref the variable that will be modified - **/ - explicit ValueGuard(T &ref) : _active(true), _ref(ref), _value(ref) {} - /** - * @brief Create a ValueGuard for the given variable. - * - * The variable will be set to the given value in the destructor. - * - * @param ref the variable that will be modified - * @param val the value it will be set to - **/ - ValueGuard(T &ref, const T &val) : _active(true), _ref(ref), _value(val) {} - /** - * @brief Reset the variable. - * - * Set the variable to the value defined in the constructor or the - * update method. If dismiss has been invoked, the variable is not - * modified. - **/ - ~ValueGuard() { - if (_active) { - _ref = _value; - } - } - /** - * @brief Dismiss this guard. - * - * When a guard has been dismissed, the destructor will not modify - * the variable. The dismiss method is typically used to indicate - * that everything went ok, and that we no longer need to protect - * the variable from exceptions. - **/ - void dismiss() { _active = false; } - /// @brief See dismiss - void deactivate() { dismiss(); } - /** - * @brief Update the value the variable will be set to in the - * destructor. - * - * This can be used to set revert points during execution. - **/ - void update(const T &val) { _value = val; } - void operator=(const T& val) { update(val); } -}; - - -/** - * @brief A MaxValueGuard is used to enfore an upper bound on the - * value of a variable when the MaxValueGuard is destructed. - * - * This can be used to revert a variable if an exception is thrown. - * However, you must remember to dismiss the guard if you don't want - * it to set the value when it goes out of scope. - **/ -template<typename T> -class MaxValueGuard { - bool _active; - T &_ref; - T _value; - - MaxValueGuard(const MaxValueGuard &); - MaxValueGuard &operator=(const MaxValueGuard &); -public: - /** - * @brief Create a MaxValueGuard for the given variable. - * - * The variable will be reverted back to its original value in the - * destructor if it has increased. - * - * @param ref the variable that will be modified - **/ - explicit MaxValueGuard(T &ref) : _active(true), _ref(ref), _value(ref) {} - /** - * @brief Create a ValueGuard for the given variable. - * - * The given upper bound will be enforced in the destructor. - * - * @param ref the variable that will be modified - * @param val upper bound for the variable - **/ - MaxValueGuard(T& ref, const T& val) : _active(true), _ref(ref), _value(val) {} - /** - * @brief Enforce the upper bound. - * - * If the current value of the variable is greater than the upper - * bound, it is set to the upper bound as defined in the - * constructor or the update method. If dismiss has been invoked, - * the variable is not modified. - **/ - ~MaxValueGuard() { - if (_active && _ref > _value) { - _ref = _value; - } - } - /** - * @brief Dismiss this guard. - * - * When a guard is dismissed, the destructor will not modify the - * variable. The dismiss method is typically used to indicate that - * everything went ok, and that we no longer need to protect the - * variable from exceptions. - **/ - void dismiss() { _active = false; } - /// @brief See dismiss - void deactivate() { dismiss(); } - /** - * @brief Update the upper bound that will be enforced in the - * destructor. - * - * This can be used to set revert points during execution. - **/ - void update(const T &val) { _value = val; } - /// @brief See update. - void operator=(const T& val) { update(val); } -}; - } // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/util/mmap_file_allocator.cpp b/vespalib/src/vespa/vespalib/util/mmap_file_allocator.cpp index 51a639a3c4e..f711d3d8685 100644 --- a/vespalib/src/vespa/vespalib/util/mmap_file_allocator.cpp +++ b/vespalib/src/vespa/vespalib/util/mmap_file_allocator.cpp @@ -11,6 +11,7 @@ #include <filesystem> using vespalib::make_string_short::fmt; +namespace fs = std::filesystem; namespace vespalib::alloc { @@ -21,7 +22,7 @@ MmapFileAllocator::MmapFileAllocator(const vespalib::string& dir_name) _allocations(), _freelist() { - std::filesystem::create_directories(std::filesystem::path(_dir_name)); + fs::create_directories(fs::path(_dir_name)); _file.open(O_RDWR | O_CREAT | O_TRUNC, false); } @@ -30,7 +31,7 @@ MmapFileAllocator::~MmapFileAllocator() assert(_allocations.empty()); _file.close(); _file.unlink(); - std::filesystem::remove_all(std::filesystem::path(_dir_name)); + fs::remove_all(fs::path(_dir_name)); } uint64_t |