diff options
author | Valerij Fredriksen <valerijf@oath.com> | 2017-11-02 15:57:23 +0100 |
---|---|---|
committer | Valerij Fredriksen <valerijf@oath.com> | 2017-11-02 15:57:23 +0100 |
commit | 1e1b84fbf754e5e90f815a5c2f83ca79865505d5 (patch) | |
tree | 325b6a8c11f98878199e080446774012e52b1b7d | |
parent | 1721b453c41d030b7892d86f92740b577a7aaf77 (diff) | |
parent | 25a2dc312d6b1b8e8dbefce9e06f9bd4619bd176 (diff) |
Merge branch 'master' into freva/instance-validator
# Conflicts:
# athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/AthenzInstanceProviderService.java
# athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/AthenzInstanceProviderServiceTest.java
74 files changed, 919 insertions, 465 deletions
diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/AthenzInstanceProviderService.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/AthenzInstanceProviderService.java index 89ea4301398..06f8d347b78 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/AthenzInstanceProviderService.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/AthenzInstanceProviderService.java @@ -6,16 +6,17 @@ import com.yahoo.component.AbstractComponent; import com.yahoo.config.model.api.SuperModelProvider; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; +import com.yahoo.jdisc.http.SecretStore; import com.yahoo.log.LogLevel; import com.yahoo.vespa.hosted.athenz.instanceproviderservice.config.AthenzProviderServiceConfig; import com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl.AthenzCertificateClient; import com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl.CertificateClient; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl.FileBackedKeyProvider; import com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl.IdentityDocumentGenerator; import com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl.IdentityDocumentServlet; import com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl.InstanceConfirmationServlet; import com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl.InstanceValidator; import com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl.KeyProvider; +import com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl.SecretStoreKeyProvider; import com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl.StatusServlet; import com.yahoo.vespa.hosted.provision.NodeRepository; import org.eclipse.jetty.server.Server; @@ -49,9 +50,9 @@ public class AthenzInstanceProviderService extends AbstractComponent { @Inject public AthenzInstanceProviderService(AthenzProviderServiceConfig config, SuperModelProvider superModelProvider, - NodeRepository nodeRepository, Zone zone) { - this(config, new FileBackedKeyProvider(config.keyPathPrefix()), Executors.newSingleThreadScheduledExecutor(), - superModelProvider, nodeRepository, zone, new AthenzCertificateClient(config), createSslContextFactory()); + NodeRepository nodeRepository, Zone zone, SecretStore secretStore) { + this(config, new SecretStoreKeyProvider(secretStore, getZoneConfig(config, zone).secretName()), Executors.newSingleThreadScheduledExecutor(), + superModelProvider, nodeRepository, zone, new AthenzCertificateClient(config, getZoneConfig(config, zone)), createSslContextFactory()); } private AthenzInstanceProviderService(AthenzProviderServiceConfig config, @@ -115,6 +116,11 @@ public class AthenzInstanceProviderService extends AbstractComponent { } + private static AthenzProviderServiceConfig.Zones getZoneConfig(AthenzProviderServiceConfig config, Zone zone) { + String key = zone.environment().value() + "." + zone.region().value(); + return config.zones(key); + } + static SslContextFactory createSslContextFactory() { try { SslContextFactory sslContextFactory = new SslContextFactory(); @@ -137,22 +143,25 @@ public class AthenzInstanceProviderService extends AbstractComponent { private final SslContextFactory sslContextFactory; private final KeyProvider keyProvider; private final AthenzProviderServiceConfig config; + private final AthenzProviderServiceConfig.Zones zoneConfig; AthenzCertificateUpdater(CertificateClient certificateClient, SslContextFactory sslContextFactory, KeyProvider keyProvider, - AthenzProviderServiceConfig config) { + AthenzProviderServiceConfig config, + AthenzProviderServiceConfig.Zones zoneConfig) { this.certificateClient = certificateClient; this.sslContextFactory = sslContextFactory; this.keyProvider = keyProvider; this.config = config; + this.zoneConfig = zoneConfig; } @Override public void run() { try { log.log(LogLevel.INFO, "Updating Athenz certificate through ZTS"); - PrivateKey privateKey = keyProvider.getPrivateKey(config.keyVersion()); + PrivateKey privateKey = keyProvider.getPrivateKey(zoneConfig.secretVersion()); X509Certificate certificate = certificateClient.updateCertificate(privateKey, EXPIRY_TIME); String dummyPassword = "athenz"; diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/impl/AthenzCertificateClient.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/impl/AthenzCertificateClient.java index 031133ade19..dab1581f580 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/impl/AthenzCertificateClient.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/impl/AthenzCertificateClient.java @@ -21,24 +21,26 @@ public class AthenzCertificateClient implements CertificateClient { private final AthenzProviderServiceConfig config; private final AthenzPrincipalAuthority authority; + private final AthenzProviderServiceConfig.Zones zoneConfig; - public AthenzCertificateClient(AthenzProviderServiceConfig config) { + public AthenzCertificateClient(AthenzProviderServiceConfig config, AthenzProviderServiceConfig.Zones zoneConfig) { this.config = config; this.authority = new AthenzPrincipalAuthority(config.athenzPrincipalHeaderName()); + this.zoneConfig = zoneConfig; } @Override public X509Certificate updateCertificate(PrivateKey privateKey, TemporalAmount expiryTime) { SimpleServiceIdentityProvider identityProvider = new SimpleServiceIdentityProvider( - authority, config.domain(), config.serviceName(), - privateKey, Integer.toString(config.keyVersion()), TimeUnit.MINUTES.toSeconds(10)); + authority, zoneConfig.domain(), zoneConfig.serviceName(), + privateKey, Integer.toString(zoneConfig.secretVersion()), TimeUnit.MINUTES.toSeconds(10)); ZTSClient ztsClient = new ZTSClient( - config.ztsUrl(), config.domain(), config.serviceName(), identityProvider); + config.ztsUrl(), zoneConfig.domain(), zoneConfig.serviceName(), identityProvider); InstanceRefreshRequest req = ZTSClient.generateInstanceRefreshRequest( - config.domain(), config.serviceName(), privateKey, + zoneConfig.domain(), zoneConfig.serviceName(), privateKey, config.certDnsSuffix(), (int)expiryTime.get(ChronoUnit.SECONDS)); - String pemEncoded = ztsClient.postInstanceRefreshRequest(config.domain(), config.serviceName(), req) + String pemEncoded = ztsClient.postInstanceRefreshRequest(zoneConfig.domain(), zoneConfig.serviceName(), req) .getCertificate(); return Crypto.loadX509Certificate(pemEncoded); } diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/impl/IdentityDocumentGenerator.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/impl/IdentityDocumentGenerator.java index 0e8ca0017f4..55acf0b796c 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/impl/IdentityDocumentGenerator.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/impl/IdentityDocumentGenerator.java @@ -27,15 +27,18 @@ public class IdentityDocumentGenerator { private final String providerService; private final String ztsUrl; private final String providerDomain; + private final int signingSecretVersion; - public IdentityDocumentGenerator(AthenzProviderServiceConfig config, NodeRepository nodeRepository, Zone zone, KeyProvider keyProvider) { + public IdentityDocumentGenerator(AthenzProviderServiceConfig config, AthenzProviderServiceConfig.Zones zoneConfig, + NodeRepository nodeRepository, Zone zone, KeyProvider keyProvider) { this.nodeRepository = nodeRepository; this.zone = zone; this.keyProvider = keyProvider; this.dnsSuffix = config.certDnsSuffix(); - this.providerService = config.serviceName(); + this.providerService = zoneConfig.serviceName(); this.ztsUrl = config.ztsUrl(); - this.providerDomain = config.domain(); + this.providerDomain = zoneConfig.domain(); + this.signingSecretVersion = zoneConfig.secretVersion(); } public String generateSignedIdentityDocument(String hostname) { @@ -49,7 +52,7 @@ public class IdentityDocumentGenerator { Signature sigGenerator = Signature.getInstance("SHA512withRSA"); // TODO: Get the correct version 0 ok for now - PrivateKey privateKey = keyProvider.getPrivateKey(0); + PrivateKey privateKey = keyProvider.getPrivateKey(signingSecretVersion); sigGenerator.initSign(privateKey); sigGenerator.update(encodedIdentityDocument.getBytes()); String signature = Base64.getEncoder().encodeToString(sigGenerator.sign()); diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/impl/SecretStoreKeyProvider.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/impl/SecretStoreKeyProvider.java new file mode 100644 index 00000000000..93abda1f9ea --- /dev/null +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/impl/SecretStoreKeyProvider.java @@ -0,0 +1,56 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl; + +import com.yahoo.athenz.auth.util.Crypto; +import com.yahoo.jdisc.http.SecretStore; + +import java.security.KeyPair; +import java.security.PrivateKey; +import java.security.PublicKey; +import java.util.HashMap; +import java.util.Map; + +/** + * @author mortent + */ +public class SecretStoreKeyProvider implements KeyProvider { + + private final SecretStore secretStore; + private final String secretName; + private final Map<Integer, KeyPair> secrets; + + + public SecretStoreKeyProvider(SecretStore secretStore, String secretName) { + this.secretStore = secretStore; + this.secretName = secretName; + this.secrets = new HashMap<>(); + } + + @Override + public PrivateKey getPrivateKey(int version) { + return getKeyPair(version).getPrivate(); + } + + @Override + public PublicKey getPublicKey(int version) { + return getKeyPair(version).getPublic(); + } + + private KeyPair getKeyPair(int version) { + synchronized (secrets) { + KeyPair keyPair = secrets.get(version); + if (keyPair == null) { + keyPair = readKeyPair(version); + secrets.put(version, keyPair); + } + return keyPair; + } + } + + // TODO: Consider moving to cryptoutils + private KeyPair readKeyPair(int version) { + PrivateKey privateKey = Crypto.loadPrivateKey(secretStore.getSecret(secretName, version)); + PublicKey publicKey = Crypto.extractPublicKey(privateKey); + return new KeyPair(publicKey, privateKey); + } +} diff --git a/athenz-identity-provider-service/src/main/resources/configdefinitions/athenz-provider-service.def b/athenz-identity-provider-service/src/main/resources/configdefinitions/athenz-provider-service.def index 7e9c19cb86a..4aad9a4eae2 100644 --- a/athenz-identity-provider-service/src/main/resources/configdefinitions/athenz-provider-service.def +++ b/athenz-identity-provider-service/src/main/resources/configdefinitions/athenz-provider-service.def @@ -2,28 +2,28 @@ namespace=vespa.hosted.athenz.instanceproviderservice.config # Athenz domain -domain string +zones{}.domain string # Athenz service name -serviceName string +zones{}.serviceName string -# Current key version -keyVersion int default=0 +# Secret name of private Key +zones{}.secretName string -# HTTPS port for Athenz Provider Service endpoint -port int default=8443 +# Secret version +zones{}.secretVersion int -# File name prefix for private and public key. Component assumes suffix .[priv|pub].<version>. -keyPathPrefix string +# HTTPS port for Athenz Provider Service endpoint +port int default=8443 # InstanceConfirmation API path -apiPath string default="/athenz/v1/provider" +apiPath string default="/athenz/v1/provider" # Athenz principal authority header name -athenzPrincipalHeaderName string default="Athenz-Principal-Auth" +athenzPrincipalHeaderName string default="Athenz-Principal-Auth" # Athenz ZTS server url -ztsUrl string +ztsUrl string # Certificate DNS suffix -certDnsSuffix string +certDnsSuffix string diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/AthenzInstanceProviderServiceTest.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/AthenzInstanceProviderServiceTest.java index a3cc82cd917..6a74d9ce3ad 100644 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/AthenzInstanceProviderServiceTest.java +++ b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/AthenzInstanceProviderServiceTest.java @@ -123,16 +123,19 @@ public class AthenzInstanceProviderServiceTest { public static AthenzProviderServiceConfig getAthenzProviderConfig(String domain, String service, String dnsSuffix) { return new AthenzProviderServiceConfig( new AthenzProviderServiceConfig.Builder() - .domain(domain) - .serviceName(service) + .zones(ImmutableMap.of(zone.environment().value() + "." + zone.region().value(), zoneConfig)) .port(PORT) - .keyPathPrefix("dummy-path") .certDnsSuffix(dnsSuffix) .ztsUrl("localhost/zts") .athenzPrincipalHeaderName("Athenz-Principal-Auth") .apiPath("")); } + + private AthenzProviderServiceConfig.Zones getZoneConfig(AthenzProviderServiceConfig config, Zone zone) { + return config.zones(zone.environment().value() + "." + zone.region().value()); + } + private static boolean getStatus(HttpClient client) { try { HttpResponse response = client.execute(new HttpGet("https://localhost:" + PORT + "/status.html")); @@ -226,17 +229,20 @@ public class AthenzInstanceProviderServiceTest { private final KeyPair keyPair; private final AthenzProviderServiceConfig config; + private final AthenzProviderServiceConfig.Zones zoneConfig; - private SelfSignedCertificateClient(KeyPair keyPair, AthenzProviderServiceConfig config) { + private SelfSignedCertificateClient(KeyPair keyPair, AthenzProviderServiceConfig config, + AthenzProviderServiceConfig.Zones zoneConfig) { this.keyPair = keyPair; this.config = config; + this.zoneConfig = zoneConfig; } @Override public X509Certificate updateCertificate(PrivateKey privateKey, TemporalAmount expiryTime) { try { ContentSigner contentSigner = new JcaContentSignerBuilder("SHA512WithRSA").build(keyPair.getPrivate()); - X500Name dnName = new X500Name("CN=" + config.domain() + "." + config.serviceName()); + X500Name dnName = new X500Name("CN=" + zoneConfig.domain() + "." + zoneConfig.serviceName()); Calendar calendar = Calendar.getInstance(); calendar.add(Calendar.HOUR, 1); Date endDate = calendar.getTime(); diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/Model.java b/config-model-api/src/main/java/com/yahoo/config/model/api/Model.java index f8f749ef070..3fec8550623 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/Model.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/Model.java @@ -7,7 +7,6 @@ import com.yahoo.vespa.config.ConfigPayload; import com.yahoo.vespa.config.buildergen.ConfigDefinition; import java.time.Instant; -import java.util.Optional; import java.util.Set; import java.util.Collection; @@ -57,21 +56,11 @@ public interface Model { default void reloadDeployFileDistributor(FileDistribution fileDistribution) { } /** - * Get the provisioning info for this model. + * Gets the allocated hosts for this model. * * @return {@link AllocatedHosts} instance, if available. - * @deprecated use allocatedHosts */ - @Deprecated - // TODO: Remove this (and the implementation below) when no version older than 6.143 is deployed anywhere - default Optional<AllocatedHosts> getProvisionInfo() { - return Optional.of(allocatedHosts()); - } - - @SuppressWarnings("deprecation") - default AllocatedHosts allocatedHosts() { - return getProvisionInfo().get(); - } + AllocatedHosts allocatedHosts(); /** * Returns whether this application allows serving config request for a different version. diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ReservedDocumentTypeNameValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ReservedDocumentTypeNameValidator.java new file mode 100644 index 00000000000..9f081f7083b --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ReservedDocumentTypeNameValidator.java @@ -0,0 +1,42 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.content; + +import com.yahoo.documentmodel.NewDocumentType; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +public class ReservedDocumentTypeNameValidator { + + public static final List<String> ORDERED_RESERVED_NAMES = Collections.unmodifiableList( + Arrays.asList("and", "false", "id", "not", "null", "or", "true")); + public static final Set<String> RESERVED_NAMES = Collections.unmodifiableSet(new HashSet<>(ORDERED_RESERVED_NAMES)); + + public void validate(Map<String, NewDocumentType> documentDefinitions) { + List<String> conflictingNames = documentDefinitions.keySet().stream() + .filter(this::isReservedName) + .collect(Collectors.toList()); + if (!conflictingNames.isEmpty()) { + throw new IllegalArgumentException(makeReservedNameMessage(conflictingNames)); + } + } + + private boolean isReservedName(String name) { + return RESERVED_NAMES.contains(name.toLowerCase()); + } + + private static String asQuotedListString(List<String> list) { + return list.stream().map(s -> String.format("'%s'", s)).collect(Collectors.joining(", ")); + } + + private static String makeReservedNameMessage(List<String> conflictingNames) { + return String.format("The following document types conflict with reserved keyword names: %s. Reserved keywords are %s", + asQuotedListString(conflictingNames), asQuotedListString(ORDERED_RESERVED_NAMES)); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index b4f8889690e..c7755d3de5a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -617,6 +617,7 @@ public class ContentCluster extends AbstractConfigProducer implements StorDistri throw new IllegalArgumentException("In indexed content cluster '" + search.getClusterName() + "': Using multi-level dispatch setup is not supported when using hierarchical distribution."); } } + new ReservedDocumentTypeNameValidator().validate(documentDefinitions); new GlobalDistributionValidator().validate(documentDefinitions, globallyDistributedDocuments, redundancy); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ClusterTest.java index 2a3dbe002e6..0c41b8ecc0b 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ClusterTest.java @@ -25,7 +25,9 @@ import com.yahoo.vespa.model.content.utils.ContentClusterUtils; import com.yahoo.vespa.model.content.utils.SearchDefinitionBuilder; import com.yahoo.vespa.model.test.utils.ApplicationPackageUtils; import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.ExpectedException; import java.util.List; @@ -38,6 +40,8 @@ public class ClusterTest extends ContentBaseTest { private final static String HOSTS = "<admin version='2.0'><adminserver hostalias='mockhost' /></admin>"; + @Rule + public ExpectedException expectedException = ExpectedException.none(); ContentCluster parse(String xml) { xml = HOSTS + xml; @@ -787,6 +791,25 @@ public class ClusterTest extends ContentBaseTest { assertTrue(cluster.getSearch().getSearchNodes().get(0).getPreShutdownCommand().isPresent()); } + @Test + public void reserved_document_name_throws_exception() { + expectedException.expect(IllegalArgumentException.class); + expectedException.expectMessage("The following document types conflict with reserved keyword names: 'true'."); + + String xml = "<content version=\"1.0\" id=\"storage\">" + + " <redundancy>1</redundancy>" + + " <documents>" + + " <document type=\"true\" mode=\"index\"/>" + + " </documents>" + + " <group>" + + " <node distribution-key=\"0\" hostalias=\"mockhost\"/>" + + " </group>" + + "</content>"; + + List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("true"); + new VespaModelCreatorWithMockPkg(null, xml, sds).create(); + } + private ContentCluster createWithZone(String clusterXml, Zone zone) throws Exception { DeployState.Builder deployStateBuilder = new DeployState.Builder().properties(new DeployProperties.Builder() .hostedVespa(true) diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ReservedDocumentTypeNameValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ReservedDocumentTypeNameValidatorTest.java new file mode 100644 index 00000000000..0ad5fb3b0bd --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ReservedDocumentTypeNameValidatorTest.java @@ -0,0 +1,57 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.content; + +import com.yahoo.documentmodel.NewDocumentType; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.function.Function; +import java.util.stream.Collectors; + +public class ReservedDocumentTypeNameValidatorTest { + + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + private static Map<String, NewDocumentType> asDocTypeMapping(List<String> typeNames) { + return typeNames.stream().collect(Collectors.toMap(Function.identity(), n -> new NewDocumentType(new NewDocumentType.Name(n)))); + } + + @Test + public void exception_thrown_on_reserved_names() { + expectedException.expect(IllegalArgumentException.class); + expectedException.expectMessage("The following document types conflict with reserved keyword names: " + + "'and', 'false', 'id', 'not', 'null', 'or', 'true'. " + + "Reserved keywords are 'and', 'false', 'id', 'not', 'null', 'or', 'true'"); + + // Ensure ordering is consistent for testing + Map<String, NewDocumentType> orderedDocTypes = new TreeMap<>(asDocTypeMapping(ReservedDocumentTypeNameValidator.ORDERED_RESERVED_NAMES)); + + ReservedDocumentTypeNameValidator validator = new ReservedDocumentTypeNameValidator(); + validator.validate(orderedDocTypes); + } + + @Test + public void exception_is_not_thrown_on_unreserved_name() { + ReservedDocumentTypeNameValidator validator = new ReservedDocumentTypeNameValidator(); + validator.validate(asDocTypeMapping(Collections.singletonList("foo"))); + } + + @Test + public void validation_is_case_insensitive() { + expectedException.expect(IllegalArgumentException.class); + expectedException.expectMessage("The following document types conflict with reserved keyword names: " + + "'NULL', 'True', 'anD'."); + + ReservedDocumentTypeNameValidator validator = new ReservedDocumentTypeNameValidator(); + Map<String, NewDocumentType> orderedDocTypes = new TreeMap<>(asDocTypeMapping(Arrays.asList("NULL", "True", "anD"))); + validator.validate(orderedDocTypes); + } + +} diff --git a/container-accesslogging/src/main/java/com/yahoo/container/logging/AccessLog.java b/container-accesslogging/src/main/java/com/yahoo/container/logging/AccessLog.java index 4356843ef3a..c22345bad79 100644 --- a/container-accesslogging/src/main/java/com/yahoo/container/logging/AccessLog.java +++ b/container-accesslogging/src/main/java/com/yahoo/container/logging/AccessLog.java @@ -10,6 +10,7 @@ import java.net.URI; /** * Logs to all the configured access logs. + * * @author tonytv */ public class AccessLog { @@ -25,7 +26,7 @@ public class AccessLog { return new AccessLog(new ComponentRegistry<>()); } - public void log(final AccessLogEntry accessLogEntry) { + public void log(AccessLogEntry accessLogEntry) { for (AccessLogInterface log: implementers.allComponents()) { log.log(accessLogEntry); } diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java index 802a7ce778c..670c48bb339 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java @@ -26,7 +26,6 @@ import java.util.logging.Logger; * @author hmusum * @author Steinar Knutsen * @author bratseth - * @since 5.1 */ public abstract class ThreadedHttpRequestHandler extends ThreadedRequestHandler { diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/metric/MetricUpdater.java b/container-disc/src/main/java/com/yahoo/container/jdisc/metric/MetricUpdater.java index 7f5ac58054e..9dbba6d351b 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/metric/MetricUpdater.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/metric/MetricUpdater.java @@ -10,14 +10,17 @@ import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.time.Duration; +import java.time.temporal.ChronoUnit; +import java.time.temporal.TemporalAmount; import java.util.Timer; import java.util.TimerTask; /** * Uses a timer to emit metrics - * + * + * @author bjorncs * @author vegardh - * @since 5.17 * */ public class MetricUpdater extends AbstractComponent { @@ -31,38 +34,74 @@ public class MetricUpdater extends AbstractComponent { private static final String MEMORY_MAPPINGS_COUNT = "jdisc.memory_mappings"; private static final String OPEN_FILE_DESCRIPTORS = "jdisc.open_file_descriptors"; - private final Metric metric; - private final ActiveContainerMetrics activeContainerMetrics; - private final Timer timer = new Timer(); - long freeMemory = -1; - long totalMemory = -1; + private final Scheduler scheduler; @Inject public MetricUpdater(Metric metric, ActiveContainerMetrics activeContainerMetrics) { - this(metric, activeContainerMetrics, 10*1000); + this(new TimerScheduler(), metric, activeContainerMetrics); } - - public MetricUpdater(Metric metric, ActiveContainerMetrics activeContainerMetrics, long delayMillis) { - this.metric = metric; - this.activeContainerMetrics = activeContainerMetrics; - timer.schedule(new UpdaterTask(), delayMillis, delayMillis); + + MetricUpdater(Scheduler scheduler, Metric metric, ActiveContainerMetrics activeContainerMetrics) { + this.scheduler = scheduler; + scheduler.schedule(new UpdaterTask(metric, activeContainerMetrics), Duration.ofSeconds(10)); } - + @Override public void deconstruct() { - if (timer!=null) timer.cancel(); + scheduler.cancel(); + } + + // Note: Linux-specific + private static long count_mappings() { + long count = 0; + try { + Path p = Paths.get("/proc/self/maps"); + if (!p.toFile().exists()) return 0; // E.g. MacOS + byte[] data = Files.readAllBytes(p); + for (byte b : data) { + if (b == '\n') { + ++count; + } + } + } catch (Exception e) { + System.err.println("Could not read /proc/self/maps: " + e); + } + return count; } + // Note: Linux-specific + + private static long count_open_files() { + long count = 0; + try { + Path p = Paths.get("/proc/self/fd"); + if (!p.toFile().exists()) return 0; // E.g. MacOS + try (DirectoryStream<Path> stream = Files.newDirectoryStream(p)) { + for (Path entry : stream) { + ++count; + } + } + } catch (Exception e) { + System.err.println("Could not read /proc/self/fd: " + e); + } + return count; + } + + private static class UpdaterTask implements Runnable { - // For testing - long getFreeMemory() { return freeMemory; } - long getTotalMemory() { return totalMemory; } + private final Runtime runtime = Runtime.getRuntime(); + private final Metric metric; + private final ActiveContainerMetrics activeContainerMetrics; + + public UpdaterTask(Metric metric, ActiveContainerMetrics activeContainerMetrics) { + this.metric = metric; + this.activeContainerMetrics = activeContainerMetrics; + } - private class UpdaterTask extends TimerTask { @SuppressWarnings("deprecation") @Override public void run() { - freeMemory = Runtime.getRuntime().freeMemory(); - totalMemory = Runtime.getRuntime().totalMemory(); + long freeMemory = runtime.freeMemory(); + long totalMemory = runtime.totalMemory(); long usedMemory = totalMemory - freeMemory; metric.set(DEPRECATED_FREE_MEMORY_BYTES, freeMemory, null); metric.set(DEPRECATED_USED_MEMORY_BYTES, usedMemory, null); @@ -75,38 +114,32 @@ public class MetricUpdater extends AbstractComponent { activeContainerMetrics.emitMetrics(metric); } - // Note: Linux-specific - private long count_mappings() { - long count = 0; - try { - Path p = Paths.get("/proc/self/maps"); - byte[] data = Files.readAllBytes(p); - for (byte b : data) { - if (b == '\n') { - ++count; - } + } + + private static class TimerScheduler implements Scheduler { + + private final Timer timer = new Timer(); + + @Override + public void schedule(Runnable runnable, TemporalAmount frequency) { + long frequencyMillis = frequency.get(ChronoUnit.MILLIS); + timer.schedule(new TimerTask() { + @Override + public void run() { + runnable.run(); } - } catch (Exception e) { - System.err.println("Could not read /proc/self/maps: " + e); - } - return count; + }, frequencyMillis, frequencyMillis) ; } - // Note: Linux-specific - private long count_open_files() { - long count = 0; - try { - Path p = Paths.get("/proc/self/fd"); - try (DirectoryStream<Path> stream = Files.newDirectoryStream(p)) { - for (Path entry : stream) { - ++count; - } - } - } catch (Exception e) { - System.err.println("Could not read /proc/self/fd: " + e); - } - return count; + @Override + public void cancel() { + timer.cancel(); } } + + interface Scheduler { + void schedule(Runnable runnable, TemporalAmount frequency); + void cancel(); + } } diff --git a/container-disc/src/test/java/com/yahoo/container/jdisc/metric/MetricUpdaterTest.java b/container-disc/src/test/java/com/yahoo/container/jdisc/metric/MetricUpdaterTest.java index 51e156a3494..c165757ecb1 100644 --- a/container-disc/src/test/java/com/yahoo/container/jdisc/metric/MetricUpdaterTest.java +++ b/container-disc/src/test/java/com/yahoo/container/jdisc/metric/MetricUpdaterTest.java @@ -2,34 +2,38 @@ package com.yahoo.container.jdisc.metric; import com.yahoo.jdisc.Metric; -import com.yahoo.jdisc.application.MetricConsumer; import com.yahoo.jdisc.statistics.ActiveContainerMetrics; import org.junit.Test; -import org.mockito.Mockito; -import static org.junit.Assert.assertTrue; +import java.time.temporal.TemporalAmount; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +/** + * @author bjorncs + */ public class MetricUpdaterTest { @Test - public void testFreeMemory() throws InterruptedException { - MetricConsumer consumer = Mockito.mock(MetricConsumer.class); - MetricProvider provider = MetricProviders.newInstance(consumer); + public void metrics_are_updated_in_scheduler_cycle() throws InterruptedException { + Metric metric = mock(Metric.class); + ActiveContainerMetrics activeContainerMetrics = mock(ActiveContainerMetrics.class); + new MetricUpdater(new MockScheduler(), metric, activeContainerMetrics); + verify(activeContainerMetrics, times(1)).emitMetrics(any()); + verify(metric, times(8)).set(anyString(), any(), any()); + } - Metric metric = provider.get(); - MetricUpdater updater = new MetricUpdater(metric, Mockito.mock(ActiveContainerMetrics.class), 10); - long start = System.currentTimeMillis(); - boolean updated = false; - while (System.currentTimeMillis() - start < 60000 && !updated) { - Thread.sleep(10); - if (memoryMetricsUpdated(updater)) { - updated = true; - } + private static class MockScheduler implements MetricUpdater.Scheduler { + @Override + public void schedule(Runnable runnable, TemporalAmount frequency) { + runnable.run(); } - assertTrue(memoryMetricsUpdated(updater)); + @Override + public void cancel() {} } - private boolean memoryMetricsUpdated(MetricUpdater updater) { - return updater.getFreeMemory()>0 && updater.getTotalMemory()>0; - } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 12ebfa625ac..26debf3083f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -337,11 +337,16 @@ public class ApplicationController { store(application); // store missing information even if we fail deployment below } - // Ensure that the deploying change is tested - if (! canDeployDirectlyTo(zone, options) && - ! application.deploymentJobs().isDeployableTo(zone.environment(), application.deploying())) - throw new IllegalArgumentException("Rejecting deployment of " + application + " to " + zone + - " as " + application.deploying().get() + " is not tested"); + if ( ! canDeployDirectlyTo(zone, options)) { // validate automated deployment + if (!application.deploymentJobs().isDeployableTo(zone.environment(), application.deploying())) + throw new IllegalArgumentException("Rejecting deployment of " + application + " to " + zone + + " as " + application.deploying().get() + " is not tested"); + Deployment existingDeployment = application.deployments().get(zone); + if (existingDeployment != null && existingDeployment.version().isAfter(version)) + throw new IllegalArgumentException("Rejecting deployment of " + application + " to " + zone + + " as the requested version " + version + " is older than" + + " the current version " + existingDeployment.version()); + } // Carry out deployment DeploymentId deploymentId = new DeploymentId(applicationId, zone); @@ -356,7 +361,8 @@ public class ApplicationController { // Use info from previous deployments is available Deployment previousDeployment = application.deployments().getOrDefault(zone, new Deployment(zone, revision, version, clock.instant())); Deployment newDeployment = new Deployment(zone, revision, version, clock.instant(), - previousDeployment.clusterUtils(), previousDeployment.clusterInfo(), previousDeployment.metrics()); + previousDeployment.clusterUtils(), + previousDeployment.clusterInfo(), previousDeployment.metrics()); application = application.with(newDeployment); store(application); @@ -623,7 +629,7 @@ public class ApplicationController { /** Returns whether a direct deployment to given zone is allowed */ private static boolean canDeployDirectlyTo(Zone zone, DeployOptions options) { - return !options.screwdriverBuildJob.isPresent() || + return ! options.screwdriverBuildJob.isPresent() || options.screwdriverBuildJob.get().screwdriverId == null || zone.environment().isManuallyDeployed(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java index a0d038c0e8d..710d2ad6492 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java @@ -11,6 +11,7 @@ import com.yahoo.vespa.hosted.controller.ApplicationController; import java.time.Instant; import java.util.Comparator; import java.util.List; +import java.util.Optional; import java.util.stream.Stream; /** @@ -47,21 +48,35 @@ public class ApplicationList { // ----------------------------------- Filters - /** Returns the subset of applications which is currently upgrading to the given version */ + /** Returns the subset of applications which are currently upgrading (to any version) */ + public ApplicationList upgrading() { + return listOf(list.stream().filter(application -> isUpgrading(application))); + } + + /** Returns the subset of applications which are currently upgrading to the given version */ public ApplicationList upgradingTo(Version version) { return listOf(list.stream().filter(application -> isUpgradingTo(version, application))); } - /** Returns the subset of applications which is currently upgrading to a version lower than the given version */ + /** Returns the subset of applications which are currently upgrading to a version lower than the given version */ public ApplicationList upgradingToLowerThan(Version version) { return listOf(list.stream().filter(application -> isUpgradingToLowerThan(version, application))); } - /** Returns the subset of applications which is currently not upgrading to the given version */ + /** Returns the subset of applications which are currently not upgrading to the given version */ public ApplicationList notUpgradingTo(Version version) { return listOf(list.stream().filter(application -> ! isUpgradingTo(version, application))); } + /** + * Returns the subset of applications which are currently not upgrading to the given version, + * or returns all if no version is specified + */ + public ApplicationList notUpgradingTo(Optional<Version> version) { + if ( ! version.isPresent()) return this; + return notUpgradingTo(version.get()); + } + /** Returns the subset of applications which is currently not deploying a new application revision */ public ApplicationList notDeployingApplication() { return listOf(list.stream().filter(application -> ! isDeployingApplicationChange(application))); @@ -155,7 +170,13 @@ public class ApplicationList { } // ----------------------------------- Internal helpers - + + private static boolean isUpgrading(Application application) { + if ( ! (application.deploying().isPresent()) ) return false; + if ( ! (application.deploying().get() instanceof Change.VersionChange) ) return false; + return true; + } + private static boolean isUpgradingTo(Version version, Application application) { if ( ! (application.deploying().isPresent()) ) return false; if ( ! (application.deploying().get() instanceof Change.VersionChange) ) return false; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index 131d89fd650..0a3e6a1beb8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -1,8 +1,10 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.deployment; +import com.yahoo.component.Version; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.curator.Lock; @@ -12,6 +14,7 @@ import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.LockedApplication; import com.yahoo.vespa.hosted.controller.application.ApplicationList; import com.yahoo.vespa.hosted.controller.application.Change; +import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobReport; @@ -155,22 +158,28 @@ public class DeploymentTrigger { * newer (different) than the one last completed successfully in next */ private boolean changesAvailable(Application application, JobStatus previous, JobStatus next) { - if ( ! previous.lastSuccess().isPresent()) return false; - if ( ! application.deploying().isPresent()) return false; Change change = application.deploying().get(); - if (change instanceof Change.VersionChange && // the last completed is out of date - don't continue with it - ! ((Change.VersionChange)change).version().equals(previous.lastSuccess().get().version())) - return false; + + if ( ! previous.lastSuccess().isPresent() && + ! productionJobHasSucceededFor(previous, change)) return false; + + if (change instanceof Change.VersionChange) { + Version targetVersion = ((Change.VersionChange)change).version(); + if ( ! (targetVersion.equals(previous.lastSuccess().get().version())) ) + return false; // version is outdated + if (isOnNewerVersionInProductionThan(targetVersion, application, next.type())) + return false; // Don't downgrade + } if (next == null) return true; if ( ! next.lastSuccess().isPresent()) return true; - + JobStatus.JobRun previousSuccess = previous.lastSuccess().get(); JobStatus.JobRun nextSuccess = next.lastSuccess().get(); if (previousSuccess.revision().isPresent() && ! previousSuccess.revision().get().equals(nextSuccess.revision().get())) return true; - if (! previousSuccess.version().equals(nextSuccess.version())) + if ( ! previousSuccess.version().equals(nextSuccess.version())) return true; return false; } @@ -396,6 +405,10 @@ public class DeploymentTrigger { if ( ! deploysTo(application, jobType)) return false; // Ignore applications that are not associated with a project if ( ! application.deploymentJobs().projectId().isPresent()) return false; + if (application.deploying().isPresent() && application.deploying().get() instanceof Change.VersionChange) { + Version targetVersion = ((Change.VersionChange)application.deploying().get()).version(); + if (isOnNewerVersionInProductionThan(targetVersion, application, jobType)) return false; // Don't downgrade + } return true; } @@ -404,6 +417,40 @@ public class DeploymentTrigger { return application.deploymentJobs().jobStatus().entrySet().stream() .anyMatch(entry -> entry.getKey().isProduction() && entry.getValue().isRunning(jobTimeoutLimit())); } + + /** + * When upgrading it is ok to trigger the next job even if the previous failed if the previous has earlier succeeded + * on the version we are currently upgrading to + */ + private boolean productionJobHasSucceededFor(JobStatus jobStatus, Change change) { + if ( ! (change instanceof Change.VersionChange) ) return false; + if ( ! isProduction(jobStatus.type())) return false; + Optional<JobStatus.JobRun> lastSuccess = jobStatus.lastSuccess(); + if ( ! lastSuccess.isPresent()) return false; + return lastSuccess.get().version().equals(((Change.VersionChange)change).version()); + } + + /** + * Returns whether the current deployed version in the zone given by the job + * is newer than the given version. This may be the case even if the production job + * in question failed, if the failure happens after deployment. + * In that case we should never deploy an earlier version as that may potentially + * downgrade production nodes which we are not guaranteed to support. + */ + private boolean isOnNewerVersionInProductionThan(Version version, Application application, JobType job) { + if ( ! isProduction(job)) return false; + Optional<Zone> zone = job.zone(controller.system()); + if ( ! zone.isPresent()) return false; + Deployment existingDeployment = application.deployments().get(zone.get()); + if (existingDeployment == null) return false; + return existingDeployment.version().isAfter(version); + } + + private boolean isProduction(JobType job) { + Optional<Zone> zone = job.zone(controller.system()); + if ( ! zone.isPresent()) return false; // arbitrary + return zone.get().environment() == Environment.prod; + } private boolean acceptNewRevisionNow(LockedApplication application) { if ( ! application.deploying().isPresent()) return true; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java index 44b15053a8c..36b87e4cead 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java @@ -12,6 +12,9 @@ import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import com.yahoo.yolean.Exceptions; import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; import java.util.logging.Level; import java.util.logging.Logger; @@ -39,29 +42,36 @@ public class Upgrader extends Maintainer { */ @Override public void maintain() { - VespaVersion target = controller().versionStatus().version(controller().systemVersion()); - if (target == null) return; // we don't have information about the current system version at this time - - switch (target.confidence()) { - case broken: - ApplicationList toCancel = applications().upgradingTo(target.versionNumber()) - .without(UpgradePolicy.canary); - if (toCancel.isEmpty()) break; - log.info("Version " + target.versionNumber() + " is broken, cancelling upgrades of non-canaries"); - cancelUpgradesOf(toCancel); - break; - case low: - upgrade(applications().with(UpgradePolicy.canary), target.versionNumber()); - break; - case normal: - upgrade(applications().with(UpgradePolicy.defaultPolicy), target.versionNumber()); - break; - case high: - upgrade(applications().with(UpgradePolicy.conservative), target.versionNumber()); - break; - default: - throw new IllegalArgumentException("Unknown version confidence " + target.confidence()); - } + ApplicationList applications = applications(); + + // Determine target versions for each upgrade policy + Optional<Version> canaryTarget = controller().versionStatus().systemVersion().map(VespaVersion::versionNumber); + Optional<Version> defaultTarget = newestVersionWithConfidence(VespaVersion.Confidence.normal); + Optional<Version> conservativeTarget = newestVersionWithConfidence(VespaVersion.Confidence.high); + + // Cancel any upgrades to the wrong targets + cancelUpgradesOf(applications.with(UpgradePolicy.canary).upgrading().notUpgradingTo(canaryTarget)); + cancelUpgradesOf(applications.with(UpgradePolicy.defaultPolicy).upgrading().notUpgradingTo(defaultTarget)); + cancelUpgradesOf(applications.with(UpgradePolicy.conservative).upgrading().notUpgradingTo(conservativeTarget)); + + // Schedule the right upgrades + canaryTarget.ifPresent(target -> upgrade(applications.with(UpgradePolicy.canary), target)); + defaultTarget.ifPresent(target -> upgrade(applications.with(UpgradePolicy.defaultPolicy), target)); + conservativeTarget.ifPresent(target -> upgrade(applications.with(UpgradePolicy.conservative), target)); + } + + private Optional<Version> newestVersionWithConfidence(VespaVersion.Confidence confidence) { + return reversed(controller().versionStatus().versions()).stream() + .filter(v -> v.confidence().equalOrHigherThan(confidence)) + .findFirst() + .map(VespaVersion::versionNumber); + } + + private List<VespaVersion> reversed(List<VespaVersion> versions) { + List<VespaVersion> reversed = new ArrayList<>(versions.size()); + for (int i = 0; i < versions.size(); i++) + reversed.add(versions.get(versions.size() - 1 - i)); + return reversed; } /** Returns a list of all applications */ @@ -89,9 +99,10 @@ public class Upgrader extends Maintainer { } private void cancelUpgradesOf(ApplicationList applications) { - for (Application application : applications.asList()) { + if (applications.isEmpty()) return; + log.info("Cancelling upgrading of " + applications.asList().size() + " applications"); + for (Application application : applications.asList()) controller().applications().deploymentTrigger().cancelChange(application.id()); - } } /** Returns the number of applications to upgrade in this run */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java index 1541e4d35f8..de42a9fba4e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java @@ -120,6 +120,7 @@ public class VespaVersion implements Comparable<VespaVersion> { return ((VespaVersion)other).versionNumber().equals(this.versionNumber()); } + /** The confidence of a version. */ public enum Confidence { /** This version has been proven defective */ @@ -132,7 +133,12 @@ public class VespaVersion implements Comparable<VespaVersion> { normal, /** We have overwhelming evidence that this version is working */ - high + high; + + /** Returns true if this confidence is at least as high as the given confidence */ + public boolean equalOrHigherThan(Confidence other) { + return this.compareTo(other) >= 0; + } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java index 5af5dac714f..79fd717a24f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java @@ -96,6 +96,7 @@ public class DeploymentTester { } public void updateVersionStatus(Version currentVersion) { + configServer().setDefaultVersion(currentVersion); controller().updateVersionStatus(VersionStatus.compute(controller(), currentVersion)); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java index 78b4f7f895f..4886eba40b6 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java @@ -24,13 +24,13 @@ public class OutstandingChangeDeployerTest { @Test public void testChangeDeployer() { DeploymentTester tester = new DeploymentTester(); + tester.configServer().setDefaultVersion(new Version(6, 1)); OutstandingChangeDeployer deployer = new OutstandingChangeDeployer(tester.controller(), Duration.ofMinutes(10), new JobControl(new MockCuratorDb())); - tester.createAndDeploy("app1", 11, "default"); tester.createAndDeploy("app2", 22, "default"); - Version version = new Version(5, 2); + Version version = new Version(6, 2); tester.deploymentTrigger().triggerChange(tester.application("app1").id(), new Change.VersionChange(version)); assertEquals(new Change.VersionChange(version), tester.application("app1").deploying().get()); @@ -52,5 +52,5 @@ public class OutstandingChangeDeployerTest { assertEquals(DeploymentJobs.JobType.systemTest.id(), jobs.get(0).jobName()); assertFalse(tester.application("app1").hasOutstandingChange()); } - + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java index 7261bb3f61e..0414cda3f55 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java @@ -9,6 +9,7 @@ import com.yahoo.test.ManualClock; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; @@ -32,10 +33,8 @@ public class UpgraderTest { public void testUpgrading() { // --- Setup DeploymentTester tester = new DeploymentTester(); - tester.upgrader().maintain(); - assertEquals("No system version: Nothing to do", 0, tester.buildSystem().jobs().size()); - Version version = Version.fromString("5.0"); // (lower than the hardcoded version in the config server client) + Version version = Version.fromString("5.0"); tester.updateVersionStatus(version); tester.upgrader().maintain(); @@ -146,7 +145,55 @@ public class UpgraderTest { tester.updateVersionStatus(version); tester.upgrader().maintain(); - assertEquals("Nothing to do", 0, tester.buildSystem().jobs().size()); + assertEquals("Applications are on 5.3 - nothing to do", 0, tester.buildSystem().jobs().size()); + + // --- Starting upgrading to a new version which breaks, causing upgrades to commence on the previous version + version = Version.fromString("5.4"); + Application default3 = tester.createAndDeploy("default3", 5, "default"); // need 4 to break a version + Application default4 = tester.createAndDeploy("default4", 5, "default"); + tester.updateVersionStatus(version); + tester.upgrader().maintain(); // cause canary upgrades to 5.4 + tester.completeUpgrade(canary0, version, "canary"); + tester.completeUpgrade(canary1, version, "canary"); + tester.updateVersionStatus(version); + assertEquals(VespaVersion.Confidence.normal, tester.controller().versionStatus().systemVersion().get().confidence()); + tester.upgrader().maintain(); + assertEquals("Upgrade of defaults are scheduled", 5, tester.buildSystem().jobs().size()); + assertEquals(version, ((Change.VersionChange)tester.application(default0.id()).deploying().get()).version()); + assertEquals(version, ((Change.VersionChange)tester.application(default1.id()).deploying().get()).version()); + assertEquals(version, ((Change.VersionChange)tester.application(default2.id()).deploying().get()).version()); + assertEquals(version, ((Change.VersionChange)tester.application(default3.id()).deploying().get()).version()); + assertEquals(version, ((Change.VersionChange)tester.application(default4.id()).deploying().get()).version()); + tester.completeUpgrade(default0, version, "default"); + // State: Default applications started upgrading to 5.4 (and one completed) + version = Version.fromString("5.5"); + tester.updateVersionStatus(version); + tester.upgrader().maintain(); // cause canary upgrades to 5.5 + tester.completeUpgrade(canary0, version, "canary"); + tester.completeUpgrade(canary1, version, "canary"); + tester.updateVersionStatus(version); + assertEquals(VespaVersion.Confidence.normal, tester.controller().versionStatus().systemVersion().get().confidence()); + tester.upgrader().maintain(); + assertEquals("Upgrade of defaults are scheduled", 5, tester.buildSystem().jobs().size()); + assertEquals(version, ((Change.VersionChange)tester.application(default0.id()).deploying().get()).version()); + assertEquals(version, ((Change.VersionChange)tester.application(default1.id()).deploying().get()).version()); + assertEquals(version, ((Change.VersionChange)tester.application(default2.id()).deploying().get()).version()); + assertEquals(version, ((Change.VersionChange)tester.application(default3.id()).deploying().get()).version()); + assertEquals(version, ((Change.VersionChange)tester.application(default4.id()).deploying().get()).version()); + // State: Default applications started upgrading to 5.5 + tester.completeUpgradeWithError(default0, version, "default", DeploymentJobs.JobType.stagingTest); + tester.completeUpgradeWithError(default1, version, "default", DeploymentJobs.JobType.stagingTest); + tester.completeUpgradeWithError(default2, version, "default", DeploymentJobs.JobType.stagingTest); + tester.completeUpgradeWithError(default3, version, "default", DeploymentJobs.JobType.productionUsWest1); + tester.completeUpgrade(default4, version, "default"); + tester.updateVersionStatus(version); + assertEquals(VespaVersion.Confidence.broken, tester.controller().versionStatus().systemVersion().get().confidence()); + tester.upgrader().maintain(); + assertEquals("Upgrade of defaults are scheduled on 5.4 instead, since 5.5 broken", + 3, tester.buildSystem().jobs().size()); + assertEquals("5.4", ((Change.VersionChange)tester.application(default1.id()).deploying().get()).version().toString()); + assertEquals("5.4", ((Change.VersionChange)tester.application(default2.id()).deploying().get()).version().toString()); + assertEquals("5.4", ((Change.VersionChange)tester.application(default3.id()).deploying().get()).version().toString()); } @Test diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json index 00bd1ed8208..7fd000b82c5 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json @@ -1,80 +1,79 @@ { - "versions": [ + "versions":[ { - "version": "(ignore)", - "confidence": "high", - "commit": "(ignore)", - "date": 0, - "controllerVersion": false, - "systemVersion": false, - "configServers": [], - "failingApplications": [], - "productionApplications": [ + "version":"(ignore)", + "confidence":"high", + "commit":"(ignore)", + "date":0, + "controllerVersion":false, + "systemVersion":false, + "configServers":[ + + ], + "failingApplications":[ + + ], + "productionApplications":[ { - "tenant": "tenant1", - "application": "application1", - "instance": "default", - "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1", - "upgradePolicy": "default" + "tenant":"tenant1", + "application":"application1", + "instance":"default", + "url":"http://localhost:8080/application/v4/tenant/tenant1/application/application1", + "upgradePolicy":"default" } ] }, { - "version": "(ignore)", - "confidence": "normal", - "commit": "(ignore)", - "date": 0, - "controllerVersion": false, - "systemVersion": false, - "configServers": [], - "failingApplications": [ + "version":"(ignore)", + "confidence":"normal", + "commit":"(ignore)", + "date":0, + "controllerVersion":false, + "systemVersion":true, + "configServers":[ + { + "hostname":"config1.test" + }, { - "tenant": "tenant1", - "application": "application1", - "instance": "default", - "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1", - "upgradePolicy": "default", - "failingSince": "(ignore)" + "hostname":"config2.test" } ], - "productionApplications": [ + "failingApplications":[ { - "tenant": "tenant2", - "application": "application2", - "instance": "default", - "url": "http://localhost:8080/application/v4/tenant/tenant2/application/application2", - "upgradePolicy": "default" + "tenant":"tenant1", + "application":"application1", + "instance":"default", + "url":"http://localhost:8080/application/v4/tenant/tenant1/application/application1", + "upgradePolicy":"default", + "failingSince":"(ignore)" } - ] - }, - { - "version": "(ignore)", - "confidence": "normal", - "commit": "(ignore)", - "date": 0, - "controllerVersion": false, - "systemVersion": true, - "configServers": [ - { - "hostname": "config1.test" - }, + ], + "productionApplications":[ { - "hostname": "config2.test" + "tenant":"tenant2", + "application":"application2", + "instance":"default", + "url":"http://localhost:8080/application/v4/tenant/tenant2/application/application2", + "upgradePolicy":"default" } - ], - "failingApplications": [], - "productionApplications": [] + ] }, { - "version": "(ignore)", - "confidence": "normal", - "commit": "(ignore)", - "date": 0, - "controllerVersion": true, - "systemVersion": false, - "configServers": [], - "failingApplications": [], - "productionApplications": [] + "version":"(ignore)", + "confidence":"normal", + "commit":"(ignore)", + "date":0, + "controllerVersion":true, + "systemVersion":false, + "configServers":[ + + ], + "failingApplications":[ + + ], + "productionApplications":[ + + ] } ] -} +}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java index bc31079cfe0..519c457e73b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java @@ -201,7 +201,7 @@ public class VersionStatusTest { assertTrue("Status for version without applications is removed", tester.controller().versionStatus().versions().stream() .noneMatch(vespaVersion -> vespaVersion.versionNumber().equals(version1))); - + // Another default application upgrades, raising confidence to high tester.completeUpgrade(default8, version2, "default"); tester.updateVersionStatus(); @@ -231,8 +231,14 @@ public class VersionStatusTest { Confidence.high, confidence(tester.controller(), version2)); assertEquals("40% of defaults failed: Broken", VespaVersion.Confidence.broken, confidence(tester.controller(), version3)); - } + // Test version order + List<VespaVersion> versions = tester.controller().versionStatus().versions(); + assertEquals(3, versions.size()); + assertEquals("5", versions.get(0).versionNumber().toString()); + assertEquals("5.2", versions.get(1).versionNumber().toString()); + assertEquals("5.3", versions.get(2).versionNumber().toString()); + } @Test public void testIgnoreConfigdeince() { diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JDiscHttpServlet.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JDiscHttpServlet.java index f396d676afd..543cf8ab43e 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JDiscHttpServlet.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JDiscHttpServlet.java @@ -23,13 +23,13 @@ import static com.yahoo.jdisc.http.core.HttpServletRequestUtils.getConnection; import static com.yahoo.jdisc.http.server.jetty.ConnectorFactory.JDiscServerConnector; /** - * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + * @author Simon Thoresen Hult * @author bjorncs */ @WebServlet(asyncSupported = true, description = "Bridge between Servlet and JDisc APIs") class JDiscHttpServlet extends HttpServlet { - public static final String ATTRIBUTE_NAME_ACCESS_LOG_ENTRY - = JDiscHttpServlet.class.getName() + "_access-log-entry"; + + public static final String ATTRIBUTE_NAME_ACCESS_LOG_ENTRY = JDiscHttpServlet.class.getName() + "_access-log-entry"; private final static Logger log = Logger.getLogger(JDiscHttpServlet.class.getName()); private final JDiscContext context; @@ -39,49 +39,48 @@ class JDiscHttpServlet extends HttpServlet { } @Override - protected void doGet(final HttpServletRequest request, final HttpServletResponse response) + protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { dispatchHttpRequest(request, response); } @Override - protected void doPost(final HttpServletRequest request, final HttpServletResponse response) + protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { dispatchHttpRequest(request, response); } @Override - protected void doHead(final HttpServletRequest request, final HttpServletResponse response) + protected void doHead(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { dispatchHttpRequest(request, response); } @Override - protected void doPut(final HttpServletRequest request, final HttpServletResponse response) + protected void doPut(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { dispatchHttpRequest(request, response); } @Override - protected void doDelete(final HttpServletRequest request, final HttpServletResponse response) + protected void doDelete(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { dispatchHttpRequest(request, response); } @Override - protected void doOptions(final HttpServletRequest request, final HttpServletResponse response) + protected void doOptions(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { dispatchHttpRequest(request, response); } @Override - protected void doTrace(final HttpServletRequest request, final HttpServletResponse response) + protected void doTrace(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { dispatchHttpRequest(request, response); } - private static final Set<String> JETTY_UNSUPPORTED_METHODS = new HashSet<>(Arrays.asList( - "PATCH")); + private static final Set<String> JETTY_UNSUPPORTED_METHODS = new HashSet<>(Arrays.asList("PATCH")); /** * Override to set connector attribute before the request becomes an upgrade request in the web socket case. @@ -89,7 +88,8 @@ class JDiscHttpServlet extends HttpServlet { */ @SuppressWarnings("deprecation") @Override - protected void service(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { + protected void service(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { request.setAttribute(JDiscServerConnector.REQUEST_ATTRIBUTE, getConnector(request)); Metric.Context metricContext = getMetricContext(request); @@ -108,25 +108,21 @@ class JDiscHttpServlet extends HttpServlet { return (JDiscServerConnector)getConnection(request).getConnector(); } - private void dispatchHttpRequest(final HttpServletRequest request, - final HttpServletResponse response) throws IOException { - final AccessLogEntry accessLogEntry = new AccessLogEntry(); + private void dispatchHttpRequest(HttpServletRequest request, HttpServletResponse response) throws IOException { + AccessLogEntry accessLogEntry = new AccessLogEntry(); AccessLogRequestLog.populateAccessLogEntryFromHttpServletRequest(request, accessLogEntry); request.setAttribute(ATTRIBUTE_NAME_ACCESS_LOG_ENTRY, accessLogEntry); try { switch (request.getDispatcherType()) { - case REQUEST: - new HttpRequestDispatch(context, - accessLogEntry, - getMetricContext(request), - request, response).dispatch(); - break; - default: - if (log.isLoggable(Level.INFO)) { - log.info("Unexpected " + request.getDispatcherType() + "; " - + formatAttributes(request)); - } - break; + case REQUEST: + new HttpRequestDispatch(context, accessLogEntry, getMetricContext(request), request, response) + .dispatch(); + break; + default: + if (log.isLoggable(Level.INFO)) { + log.info("Unexpected " + request.getDispatcherType() + "; " + formatAttributes(request)); + } + break; } } catch (OverloadException e) { // nop @@ -136,12 +132,11 @@ class JDiscHttpServlet extends HttpServlet { } private static Metric.Context getMetricContext(ServletRequest request) { - return JDiscServerConnector.fromRequest(request) - .getMetricContext(); + return JDiscServerConnector.fromRequest(request).getMetricContext(); } private static String formatAttributes(final HttpServletRequest request) { - final StringBuilder out = new StringBuilder(); + StringBuilder out = new StringBuilder(); out.append("attributes = {"); for (Enumeration<String> names = request.getAttributeNames(); names.hasMoreElements(); ) { String name = names.nextElement(); diff --git a/libmlr/OWNERS b/libmlr/OWNERS index 76e34e72c9d..6b09ce48bd4 100644 --- a/libmlr/OWNERS +++ b/libmlr/OWNERS @@ -1,2 +1 @@ lesters -tmartins diff --git a/messagebus_test/OWNERS b/messagebus_test/OWNERS index 101fe620dd4..9dc0c2d970d 100644 --- a/messagebus_test/OWNERS +++ b/messagebus_test/OWNERS @@ -1,2 +1 @@ baldersheim - diff --git a/persistencetypes/OWNERS b/persistencetypes/OWNERS index 4cef58fa4f9..dbcff24b338 100644 --- a/persistencetypes/OWNERS +++ b/persistencetypes/OWNERS @@ -1,2 +1 @@ vekterli - diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SetMembershipNode.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SetMembershipNode.java index de600cb7519..f8e44f1087c 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SetMembershipNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SetMembershipNode.java @@ -12,7 +12,6 @@ import java.util.*; * A node which returns true or false depending on a set membership test * * @author bratseth - * @since 5.1.21 */ public class SetMembershipNode extends BooleanNode { diff --git a/storage/OWNERS b/storage/OWNERS index 4cef58fa4f9..dbcff24b338 100644 --- a/storage/OWNERS +++ b/storage/OWNERS @@ -1,2 +1 @@ vekterli - diff --git a/storage/src/tests/common/teststorageapp.h b/storage/src/tests/common/teststorageapp.h index 268ba7aafdc..6906d41ac47 100644 --- a/storage/src/tests/common/teststorageapp.h +++ b/storage/src/tests/common/teststorageapp.h @@ -125,7 +125,9 @@ public: spi::PartitionStateList& getPartitions(); uint16_t getPartition(const document::BucketId&); - StorBucketDatabase& getStorageBucketDatabase() override { return _compReg.getBucketDatabase(); } + StorBucketDatabase& getStorageBucketDatabase() override { + return _compReg.getBucketSpaceRepo().get(document::BucketSpace::placeHolder()).bucketDatabase(); + } private: // For storage server interface implementation we'll get rid of soon. diff --git a/storage/src/tests/distributor/blockingoperationstartertest.cpp b/storage/src/tests/distributor/blockingoperationstartertest.cpp index f91e782c6c0..c2fdc25cebf 100644 --- a/storage/src/tests/distributor/blockingoperationstartertest.cpp +++ b/storage/src/tests/distributor/blockingoperationstartertest.cpp @@ -4,6 +4,9 @@ #include <vespa/storage/distributor/blockingoperationstarter.h> #include <vespa/storage/distributor/pendingmessagetracker.h> #include <tests/distributor/maintenancemocks.h> +#include <vespa/document/test/make_document_bucket.h> + +using document::test::makeDocumentBucket; namespace storage { @@ -18,10 +21,10 @@ class BlockingOperationStarterTest : public CppUnit::TestFixture { CPPUNIT_TEST_SUITE_END(); std::shared_ptr<Operation> createMockOperation() { - return std::shared_ptr<Operation>(new MockOperation(BucketId(16, 1))); + return std::shared_ptr<Operation>(new MockOperation(makeDocumentBucket(BucketId(16, 1)))); } std::shared_ptr<Operation> createBlockingMockOperation() { - std::shared_ptr<MockOperation> op(new MockOperation(BucketId(16, 1))); + std::shared_ptr<MockOperation> op(new MockOperation(makeDocumentBucket(BucketId(16, 1)))); op->setShouldBlock(true); return op; } @@ -57,7 +60,7 @@ BlockingOperationStarterTest::testOperationNotBlockedWhenNoMessagesPending() { CPPUNIT_ASSERT(_operationStarter->start(createMockOperation(), OperationStarter::Priority(0))); - CPPUNIT_ASSERT_EQUAL(std::string("BucketId(0x4000000000000001), pri 0\n"), + CPPUNIT_ASSERT_EQUAL(std::string("Bucket(BucketSpace(0x0000000000000000), BucketId(0x4000000000000001)), pri 0\n"), _starterImpl->toString()); } diff --git a/storage/src/tests/distributor/maintenancemocks.h b/storage/src/tests/distributor/maintenancemocks.h index 75e19ec1033..505a7cece9e 100644 --- a/storage/src/tests/distributor/maintenancemocks.h +++ b/storage/src/tests/distributor/maintenancemocks.h @@ -15,7 +15,7 @@ class MockMaintenancePriorityGenerator : public MaintenancePriorityGenerator { MaintenancePriorityAndType prioritize( - const document::BucketId&, + const document::Bucket&, NodeMaintenanceStatsTracker& stats) const override { stats.incMovingOut(1); @@ -29,17 +29,17 @@ class MockMaintenancePriorityGenerator class MockOperation : public MaintenanceOperation { - document::BucketId _bucketId; + document::Bucket _bucket; std::string _reason; bool _shouldBlock; public: - MockOperation(const document::BucketId& bucketId) - : _bucketId(bucketId), + MockOperation(const document::Bucket &bucket) + : _bucket(bucket), _shouldBlock(false) {} std::string toString() const override { - return _bucketId.toString(); + return _bucket.toString(); } void onClose(DistributorMessageSender&) override {} @@ -61,17 +61,17 @@ class MockMaintenanceOperationGenerator : public MaintenanceOperationGenerator { public: - MaintenanceOperation::SP generate(const document::BucketId& id) const override { - return MaintenanceOperation::SP(new MockOperation(id)); + MaintenanceOperation::SP generate(const document::Bucket&bucket) const override { + return MaintenanceOperation::SP(new MockOperation(bucket)); } std::vector<MaintenanceOperation::SP> generateAll( - const document::BucketId& id, + const document::Bucket &bucket, NodeMaintenanceStatsTracker& tracker) const override { (void) tracker; std::vector<MaintenanceOperation::SP> ret; - ret.push_back(MaintenanceOperation::SP(new MockOperation(id))); + ret.push_back(MaintenanceOperation::SP(new MockOperation(bucket))); return ret; } diff --git a/storage/src/tests/distributor/maintenanceschedulertest.cpp b/storage/src/tests/distributor/maintenanceschedulertest.cpp index 48fcdfe2f4e..7e3d92053f8 100644 --- a/storage/src/tests/distributor/maintenanceschedulertest.cpp +++ b/storage/src/tests/distributor/maintenanceschedulertest.cpp @@ -7,6 +7,9 @@ #include <vespa/storage/distributor/maintenance/maintenancescheduler.h> #include <vespa/storage/bucketdb/mapbucketdatabase.h> #include <tests/distributor/maintenancemocks.h> +#include <vespa/document/test/make_document_bucket.h> + +using document::test::makeDocumentBucket; namespace storage { @@ -57,7 +60,7 @@ MaintenanceSchedulerTest::setUp() void MaintenanceSchedulerTest::testPriorityClearedAfterScheduled() { - _priorityDb->setPriority(PrioritizedBucket(BucketId(16, 1), Priority::VERY_HIGH)); + _priorityDb->setPriority(PrioritizedBucket(makeDocumentBucket(BucketId(16, 1)), Priority::VERY_HIGH)); _scheduler->tick(MaintenanceScheduler::NORMAL_SCHEDULING_MODE); CPPUNIT_ASSERT_EQUAL(std::string(), _priorityDb->toString()); } @@ -65,9 +68,9 @@ MaintenanceSchedulerTest::testPriorityClearedAfterScheduled() void MaintenanceSchedulerTest::testOperationIsScheduled() { - _priorityDb->setPriority(PrioritizedBucket(BucketId(16, 1), Priority::MEDIUM)); + _priorityDb->setPriority(PrioritizedBucket(makeDocumentBucket(BucketId(16, 1)), Priority::MEDIUM)); _scheduler->tick(MaintenanceScheduler::NORMAL_SCHEDULING_MODE); - CPPUNIT_ASSERT_EQUAL(std::string("BucketId(0x4000000000000001), pri 100\n"), + CPPUNIT_ASSERT_EQUAL(std::string("Bucket(BucketSpace(0x0000000000000000), BucketId(0x4000000000000001)), pri 100\n"), _operationStarter->toString()); } @@ -82,24 +85,24 @@ MaintenanceSchedulerTest::testNoOperationsToSchedule() void MaintenanceSchedulerTest::testSuppressLowPrioritiesInEmergencyMode() { - _priorityDb->setPriority(PrioritizedBucket(BucketId(16, 1), Priority::HIGH)); - _priorityDb->setPriority(PrioritizedBucket(BucketId(16, 2), Priority::VERY_HIGH)); + _priorityDb->setPriority(PrioritizedBucket(makeDocumentBucket(BucketId(16, 1)), Priority::HIGH)); + _priorityDb->setPriority(PrioritizedBucket(makeDocumentBucket(BucketId(16, 2)), Priority::VERY_HIGH)); CPPUNIT_ASSERT_EQUAL(WaitTimeMs(0), _scheduler->tick(MaintenanceScheduler::RECOVERY_SCHEDULING_MODE)); CPPUNIT_ASSERT_EQUAL(WaitTimeMs(1), _scheduler->tick(MaintenanceScheduler::RECOVERY_SCHEDULING_MODE)); - CPPUNIT_ASSERT_EQUAL(std::string("BucketId(0x4000000000000002), pri 0\n"), + CPPUNIT_ASSERT_EQUAL(std::string("Bucket(BucketSpace(0x0000000000000000), BucketId(0x4000000000000002)), pri 0\n"), _operationStarter->toString()); - CPPUNIT_ASSERT_EQUAL(std::string("PrioritizedBucket(BucketId(0x4000000000000001), pri HIGH)\n"), + CPPUNIT_ASSERT_EQUAL(std::string("PrioritizedBucket(Bucket(BucketSpace(0x0000000000000000), BucketId(0x4000000000000001)), pri HIGH)\n"), _priorityDb->toString()); } void MaintenanceSchedulerTest::testPriorityNotClearedIfOperationNotStarted() { - _priorityDb->setPriority(PrioritizedBucket(BucketId(16, 1), Priority::HIGH)); + _priorityDb->setPriority(PrioritizedBucket(makeDocumentBucket(BucketId(16, 1)), Priority::HIGH)); _operationStarter->setShouldStartOperations(false); WaitTimeMs waitMs(_scheduler->tick(MaintenanceScheduler::NORMAL_SCHEDULING_MODE)); CPPUNIT_ASSERT_EQUAL(WaitTimeMs(1), waitMs); - CPPUNIT_ASSERT_EQUAL(std::string("PrioritizedBucket(BucketId(0x4000000000000001), pri HIGH)\n"), + CPPUNIT_ASSERT_EQUAL(std::string("PrioritizedBucket(Bucket(BucketSpace(0x0000000000000000), BucketId(0x4000000000000001)), pri HIGH)\n"), _priorityDb->toString()); } diff --git a/storage/src/tests/distributor/simplebucketprioritydatabasetest.cpp b/storage/src/tests/distributor/simplebucketprioritydatabasetest.cpp index 0843d459020..f12917b0936 100644 --- a/storage/src/tests/distributor/simplebucketprioritydatabasetest.cpp +++ b/storage/src/tests/distributor/simplebucketprioritydatabasetest.cpp @@ -3,6 +3,9 @@ #include <vespa/vdstestlib/cppunit/macros.h> #include <string> #include <vespa/storage/distributor/maintenance/simplebucketprioritydatabase.h> +#include <vespa/document/test/make_document_bucket.h> + +using document::test::makeDocumentBucket; namespace storage { @@ -51,7 +54,7 @@ SimpleBucketPriorityDatabaseTest::testCanGetPrioritizedBucket() { SimpleBucketPriorityDatabase queue; - PrioritizedBucket lowPriBucket(BucketId(16, 1234), Priority::VERY_LOW); + PrioritizedBucket lowPriBucket(makeDocumentBucket(BucketId(16, 1234)), Priority::VERY_LOW); queue.setPriority(lowPriBucket); PrioritizedBucket highest(*queue.begin()); @@ -63,8 +66,8 @@ SimpleBucketPriorityDatabaseTest::testIterateOverMultiplePriorities() { SimpleBucketPriorityDatabase queue; - PrioritizedBucket lowPriBucket(BucketId(16, 1234), Priority::LOW); - PrioritizedBucket highPriBucket(BucketId(16, 4321), Priority::HIGH); + PrioritizedBucket lowPriBucket(makeDocumentBucket(BucketId(16, 1234)), Priority::LOW); + PrioritizedBucket highPriBucket(makeDocumentBucket(BucketId(16, 4321)), Priority::HIGH); queue.setPriority(lowPriBucket); queue.setPriority(highPriBucket); @@ -82,8 +85,8 @@ SimpleBucketPriorityDatabaseTest::testMultipleSetPriorityForOneBucket() { SimpleBucketPriorityDatabase queue; - PrioritizedBucket lowPriBucket(BucketId(16, 1234), Priority::LOW); - PrioritizedBucket highPriBucket(BucketId(16, 1234), Priority::HIGH); + PrioritizedBucket lowPriBucket(makeDocumentBucket(BucketId(16, 1234)), Priority::LOW); + PrioritizedBucket highPriBucket(makeDocumentBucket(BucketId(16, 1234)), Priority::HIGH); queue.setPriority(lowPriBucket); queue.setPriority(highPriBucket); @@ -99,8 +102,8 @@ SimpleBucketPriorityDatabaseTest::testNoMaintenanceNeededClearsBucketFromDatabas { SimpleBucketPriorityDatabase queue; - PrioritizedBucket highPriBucket(BucketId(16, 1234), Priority::HIGH); - PrioritizedBucket noPriBucket(BucketId(16, 1234), + PrioritizedBucket highPriBucket(makeDocumentBucket(BucketId(16, 1234)), Priority::HIGH); + PrioritizedBucket noPriBucket(makeDocumentBucket(BucketId(16, 1234)), Priority::NO_MAINTENANCE_NEEDED); queue.setPriority(highPriBucket); queue.setPriority(noPriBucket); @@ -114,11 +117,11 @@ SimpleBucketPriorityDatabaseTest::testIterateOverMultipleBucketsWithMultiplePrio { SimpleBucketPriorityDatabase queue; - PrioritizedBucket lowPriBucket1(BucketId(16, 1), Priority::LOW); - PrioritizedBucket lowPriBucket2(BucketId(16, 2), Priority::LOW); - PrioritizedBucket mediumPriBucket(BucketId(16, 3), Priority::MEDIUM); - PrioritizedBucket highPriBucket1(BucketId(16, 4), Priority::HIGH); - PrioritizedBucket highPriBucket2(BucketId(16, 5), Priority::HIGH); + PrioritizedBucket lowPriBucket1(makeDocumentBucket(BucketId(16, 1)), Priority::LOW); + PrioritizedBucket lowPriBucket2(makeDocumentBucket(BucketId(16, 2)), Priority::LOW); + PrioritizedBucket mediumPriBucket(makeDocumentBucket(BucketId(16, 3)), Priority::MEDIUM); + PrioritizedBucket highPriBucket1(makeDocumentBucket(BucketId(16, 4)), Priority::HIGH); + PrioritizedBucket highPriBucket2(makeDocumentBucket(BucketId(16, 5)), Priority::HIGH); queue.setPriority(highPriBucket1); queue.setPriority(lowPriBucket2); @@ -127,7 +130,7 @@ SimpleBucketPriorityDatabaseTest::testIterateOverMultipleBucketsWithMultiplePrio queue.setPriority(lowPriBucket1); const_iterator iter(queue.begin()); - PrioritizedBucket lastBucket(BucketId(), Priority::PRIORITY_LIMIT); + PrioritizedBucket lastBucket(makeDocumentBucket(BucketId()), Priority::PRIORITY_LIMIT); for (int i = 0; i < 5; ++i) { CPPUNIT_ASSERT(iter != queue.end()); CPPUNIT_ASSERT(!iter->moreImportantThan(lastBucket)); diff --git a/storage/src/tests/distributor/simplemaintenancescannertest.cpp b/storage/src/tests/distributor/simplemaintenancescannertest.cpp index 17f25e4532a..a46419b71a4 100644 --- a/storage/src/tests/distributor/simplemaintenancescannertest.cpp +++ b/storage/src/tests/distributor/simplemaintenancescannertest.cpp @@ -78,7 +78,7 @@ void SimpleMaintenanceScannerTest::testPrioritizeSingleBucket() { addBucketToDb(1); - std::string expected("PrioritizedBucket(BucketId(0x4000000000000001), pri VERY_HIGH)\n"); + std::string expected("PrioritizedBucket(Bucket(BucketSpace(0x0000000000000000), BucketId(0x4000000000000001)), pri VERY_HIGH)\n"); CPPUNIT_ASSERT(!_scanner->scanNext().isDone()); CPPUNIT_ASSERT_EQUAL(expected, _priorityDb->toString()); @@ -107,9 +107,9 @@ SimpleMaintenanceScannerTest::testPrioritizeMultipleBuckets() addBucketToDb(1); addBucketToDb(2); addBucketToDb(3); - std::string expected("PrioritizedBucket(BucketId(0x4000000000000001), pri VERY_HIGH)\n" - "PrioritizedBucket(BucketId(0x4000000000000002), pri VERY_HIGH)\n" - "PrioritizedBucket(BucketId(0x4000000000000003), pri VERY_HIGH)\n"); + std::string expected("PrioritizedBucket(Bucket(BucketSpace(0x0000000000000000), BucketId(0x4000000000000001)), pri VERY_HIGH)\n" + "PrioritizedBucket(Bucket(BucketSpace(0x0000000000000000), BucketId(0x4000000000000002)), pri VERY_HIGH)\n" + "PrioritizedBucket(Bucket(BucketSpace(0x0000000000000000), BucketId(0x4000000000000003)), pri VERY_HIGH)\n"); CPPUNIT_ASSERT(scanEntireDatabase(3)); CPPUNIT_ASSERT_EQUAL(sortLines(expected), @@ -134,8 +134,8 @@ SimpleMaintenanceScannerTest::testReset() addBucketToDb(3); CPPUNIT_ASSERT(scanEntireDatabase(2)); - std::string expected("PrioritizedBucket(BucketId(0x4000000000000001), pri VERY_HIGH)\n" - "PrioritizedBucket(BucketId(0x4000000000000003), pri VERY_HIGH)\n"); + std::string expected("PrioritizedBucket(Bucket(BucketSpace(0x0000000000000000), BucketId(0x4000000000000001)), pri VERY_HIGH)\n" + "PrioritizedBucket(Bucket(BucketSpace(0x0000000000000000), BucketId(0x4000000000000003)), pri VERY_HIGH)\n"); CPPUNIT_ASSERT_EQUAL(expected, _priorityDb->toString()); addBucketToDb(2); @@ -145,9 +145,9 @@ SimpleMaintenanceScannerTest::testReset() _scanner->reset(); CPPUNIT_ASSERT(scanEntireDatabase(3)); - expected = "PrioritizedBucket(BucketId(0x4000000000000001), pri VERY_HIGH)\n" - "PrioritizedBucket(BucketId(0x4000000000000002), pri VERY_HIGH)\n" - "PrioritizedBucket(BucketId(0x4000000000000003), pri VERY_HIGH)\n"; + expected = "PrioritizedBucket(Bucket(BucketSpace(0x0000000000000000), BucketId(0x4000000000000001)), pri VERY_HIGH)\n" + "PrioritizedBucket(Bucket(BucketSpace(0x0000000000000000), BucketId(0x4000000000000002)), pri VERY_HIGH)\n" + "PrioritizedBucket(Bucket(BucketSpace(0x0000000000000000), BucketId(0x4000000000000003)), pri VERY_HIGH)\n"; CPPUNIT_ASSERT_EQUAL(sortLines(expected), sortLines(_priorityDb->toString())); } diff --git a/storage/src/tests/distributor/throttlingoperationstartertest.cpp b/storage/src/tests/distributor/throttlingoperationstartertest.cpp index bc082f65e18..c3aebcafe06 100644 --- a/storage/src/tests/distributor/throttlingoperationstartertest.cpp +++ b/storage/src/tests/distributor/throttlingoperationstartertest.cpp @@ -2,6 +2,9 @@ #include <vespa/vdstestlib/cppunit/macros.h> #include <vespa/storage/distributor/throttlingoperationstarter.h> #include <tests/distributor/maintenancemocks.h> +#include <vespa/document/test/make_document_bucket.h> + +using document::test::makeDocumentBucket; namespace storage { @@ -20,7 +23,7 @@ class ThrottlingOperationStarterTest : public CppUnit::TestFixture { CPPUNIT_TEST_SUITE_END(); std::shared_ptr<Operation> createMockOperation() { - return std::shared_ptr<Operation>(new MockOperation(BucketId(16, 1))); + return std::shared_ptr<Operation>(new MockOperation(makeDocumentBucket(BucketId(16, 1)))); } std::unique_ptr<MockOperationStarter> _starterImpl; @@ -67,7 +70,7 @@ ThrottlingOperationStarterTest::testOperationStartingIsForwardedToImplementation { CPPUNIT_ASSERT(_operationStarter->start(createMockOperation(), OperationStarter::Priority(0))); - CPPUNIT_ASSERT_EQUAL(std::string("BucketId(0x4000000000000001), pri 0\n"), + CPPUNIT_ASSERT_EQUAL(std::string("Bucket(BucketSpace(0x0000000000000000), BucketId(0x4000000000000001)), pri 0\n"), _starterImpl->toString()); } diff --git a/storage/src/vespa/storage/common/CMakeLists.txt b/storage/src/vespa/storage/common/CMakeLists.txt index 59e27307c77..b98058b3c3d 100644 --- a/storage/src/vespa/storage/common/CMakeLists.txt +++ b/storage/src/vespa/storage/common/CMakeLists.txt @@ -3,6 +3,8 @@ vespa_add_library(storage_common OBJECT SOURCES bucketmessages.cpp bucketoperationlogger.cpp + content_bucket_space.cpp + content_bucket_space_repo.cpp distributorcomponent.cpp messagebucket.cpp messagesender.cpp diff --git a/storage/src/vespa/storage/common/content_bucket_space.cpp b/storage/src/vespa/storage/common/content_bucket_space.cpp new file mode 100644 index 00000000000..b78be81c9de --- /dev/null +++ b/storage/src/vespa/storage/common/content_bucket_space.cpp @@ -0,0 +1,12 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "content_bucket_space.h" + +namespace storage { + +ContentBucketSpace::ContentBucketSpace() + : _bucketDatabase() +{ +} + +} diff --git a/storage/src/vespa/storage/common/content_bucket_space.h b/storage/src/vespa/storage/common/content_bucket_space.h new file mode 100644 index 00000000000..2efb2eca06d --- /dev/null +++ b/storage/src/vespa/storage/common/content_bucket_space.h @@ -0,0 +1,21 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/storage/bucketdb/storbucketdb.h> + +namespace storage { + +/** + * Class representing a bucket space (with associated bucket database) on a content node. + */ +class ContentBucketSpace { +private: + StorBucketDatabase _bucketDatabase; + +public: + using UP = std::unique_ptr<ContentBucketSpace>; + ContentBucketSpace(); + StorBucketDatabase &bucketDatabase() { return _bucketDatabase; } +}; + +} diff --git a/storage/src/vespa/storage/common/content_bucket_space_repo.cpp b/storage/src/vespa/storage/common/content_bucket_space_repo.cpp new file mode 100644 index 00000000000..74a2b912602 --- /dev/null +++ b/storage/src/vespa/storage/common/content_bucket_space_repo.cpp @@ -0,0 +1,24 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "content_bucket_space_repo.h" + +using document::BucketSpace; + +namespace storage { + +ContentBucketSpaceRepo::ContentBucketSpaceRepo() + : _map() +{ + _map.emplace(BucketSpace::placeHolder(), std::make_unique<ContentBucketSpace>()); +} + +ContentBucketSpace & +ContentBucketSpaceRepo::get(BucketSpace bucketSpace) const +{ + assert(bucketSpace == BucketSpace::placeHolder()); + auto itr = _map.find(bucketSpace); + assert(itr != _map.end()); + return *itr->second; +} + +} diff --git a/storage/src/vespa/storage/common/content_bucket_space_repo.h b/storage/src/vespa/storage/common/content_bucket_space_repo.h new file mode 100644 index 00000000000..258edf2d0c9 --- /dev/null +++ b/storage/src/vespa/storage/common/content_bucket_space_repo.h @@ -0,0 +1,24 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "content_bucket_space.h" +#include <vespa/document/bucket/bucketspace.h> +#include <unordered_map> + +namespace storage { + +/** + * Class managing the set of bucket spaces (with associated bucket databases) on a content node. + */ +class ContentBucketSpaceRepo { +private: + using BucketSpaceMap = std::unordered_map<document::BucketSpace, ContentBucketSpace::UP, document::BucketSpace::hash>; + + BucketSpaceMap _map; + +public: + ContentBucketSpaceRepo(); + ContentBucketSpace &get(document::BucketSpace bucketSpace) const; +}; + +} diff --git a/storage/src/vespa/storage/common/servicelayercomponent.cpp b/storage/src/vespa/storage/common/servicelayercomponent.cpp index a5f33fb28b8..72f65ebaee1 100644 --- a/storage/src/vespa/storage/common/servicelayercomponent.cpp +++ b/storage/src/vespa/storage/common/servicelayercomponent.cpp @@ -2,6 +2,7 @@ #include "servicelayercomponent.h" +#include <vespa/storage/common/content_bucket_space_repo.h> #include <vespa/storage/common/nodestateupdater.h> #include <vespa/vdslib/distribution/distribution.h> @@ -13,8 +14,8 @@ StorBucketDatabase& ServiceLayerComponent::getBucketDatabase(BucketSpace bucketSpace) const { assert(bucketSpace == BucketSpace::placeHolder()); - assert(_bucketDatabase != 0); - return *_bucketDatabase; + assert(_bucketSpaceRepo != nullptr); + return _bucketSpaceRepo->get(bucketSpace).bucketDatabase(); } uint16_t diff --git a/storage/src/vespa/storage/common/servicelayercomponent.h b/storage/src/vespa/storage/common/servicelayercomponent.h index 3e6ccaaa5a0..bd46e6ba4f9 100644 --- a/storage/src/vespa/storage/common/servicelayercomponent.h +++ b/storage/src/vespa/storage/common/servicelayercomponent.h @@ -30,6 +30,7 @@ namespace storage { +class ContentBucketSpaceRepo; class MinimumUsedBitsTracker; class StorBucketDatabase; @@ -38,7 +39,7 @@ struct ServiceLayerManagedComponent virtual ~ServiceLayerManagedComponent() {} virtual void setDiskCount(uint16_t count) = 0; - virtual void setBucketDatabase(StorBucketDatabase&) = 0; + virtual void setBucketSpaceRepo(ContentBucketSpaceRepo&) = 0; virtual void setMinUsedBitsTracker(MinimumUsedBitsTracker&) = 0; }; @@ -52,12 +53,12 @@ class ServiceLayerComponent : public StorageComponent, private ServiceLayerManagedComponent { uint16_t _diskCount; - StorBucketDatabase* _bucketDatabase; + ContentBucketSpaceRepo* _bucketSpaceRepo; MinimumUsedBitsTracker* _minUsedBitsTracker; // ServiceLayerManagedComponent implementation void setDiskCount(uint16_t count) override { _diskCount = count; } - void setBucketDatabase(StorBucketDatabase& db) override { _bucketDatabase = &db; } + void setBucketSpaceRepo(ContentBucketSpaceRepo& repo) override { _bucketSpaceRepo = &repo; } void setMinUsedBitsTracker(MinimumUsedBitsTracker& tracker) override { _minUsedBitsTracker = &tracker; } @@ -68,8 +69,8 @@ public: vespalib::stringref name) : StorageComponent(compReg, name), _diskCount(0), - _bucketDatabase(0), - _minUsedBitsTracker(0) + _bucketSpaceRepo(nullptr), + _minUsedBitsTracker(nullptr) { compReg.registerServiceLayerComponent(*this); } diff --git a/storage/src/vespa/storage/distributor/CMakeLists.txt b/storage/src/vespa/storage/distributor/CMakeLists.txt index 2d40f0d0e33..3dc08d858ad 100644 --- a/storage/src/vespa/storage/distributor/CMakeLists.txt +++ b/storage/src/vespa/storage/distributor/CMakeLists.txt @@ -7,6 +7,9 @@ vespa_add_library(storage_distributor bucketgctimecalculator.cpp bucketlistmerger.cpp clusterinformation.cpp + distributor_bucket_space.cpp + distributor_bucket_space_component.cpp + distributor_bucket_space_repo.cpp distributor.cpp distributor_host_info_reporter.cpp distributorcomponent.cpp @@ -16,9 +19,6 @@ vespa_add_library(storage_distributor idealstatemanager.cpp idealstatemetricsset.cpp latency_statistics_provider.cpp - managed_bucket_space.cpp - managed_bucket_space_component.cpp - managed_bucket_space_repo.cpp messagetracker.cpp nodeinfo.cpp operation_sequencer.cpp diff --git a/storage/src/vespa/storage/distributor/bucketdbupdater.cpp b/storage/src/vespa/storage/distributor/bucketdbupdater.cpp index 2a056949f9a..97c652c8a89 100644 --- a/storage/src/vespa/storage/distributor/bucketdbupdater.cpp +++ b/storage/src/vespa/storage/distributor/bucketdbupdater.cpp @@ -18,7 +18,7 @@ using document::BucketSpace; namespace storage::distributor { -BucketDBUpdater::BucketDBUpdater(Distributor& owner, ManagedBucketSpace& bucketSpace, +BucketDBUpdater::BucketDBUpdater(Distributor& owner, DistributorBucketSpace& bucketSpace, DistributorMessageSender& sender, DistributorComponentRegister& compReg) : framework::StatusReporter("bucketdb", "Bucket DB Updater"), _bucketSpaceComponent(owner, bucketSpace, compReg, "Bucket DB Updater"), diff --git a/storage/src/vespa/storage/distributor/bucketdbupdater.h b/storage/src/vespa/storage/distributor/bucketdbupdater.h index d3b2bbf86ca..2428b3bd355 100644 --- a/storage/src/vespa/storage/distributor/bucketdbupdater.h +++ b/storage/src/vespa/storage/distributor/bucketdbupdater.h @@ -6,7 +6,7 @@ #include "distributorcomponent.h" #include "distributormessagesender.h" #include "pendingclusterstate.h" -#include "managed_bucket_space_component.h" +#include "distributor_bucket_space_component.h" #include <vespa/document/bucket/bucketid.h> #include <vespa/storageapi/messageapi/returncode.h> #include <vespa/storageapi/message/bucket.h> @@ -30,7 +30,7 @@ public: // TODO take in BucketSpaceRepo instead, this class needs access to all // bucket spaces. BucketDBUpdater(Distributor& owner, - ManagedBucketSpace& bucketSpace, + DistributorBucketSpace& bucketSpace, DistributorMessageSender& sender, DistributorComponentRegister& compReg); ~BucketDBUpdater(); @@ -64,7 +64,7 @@ public: } private: - ManagedBucketSpaceComponent _bucketSpaceComponent; + DistributorBucketSpaceComponent _bucketSpaceComponent; class MergeReplyGuard { public: MergeReplyGuard(BucketDBUpdater& updater, const std::shared_ptr<api::MergeBucketReply>& reply) diff --git a/storage/src/vespa/storage/distributor/distributor.cpp b/storage/src/vespa/storage/distributor/distributor.cpp index 14cb41bca3e..05145d21410 100644 --- a/storage/src/vespa/storage/distributor/distributor.cpp +++ b/storage/src/vespa/storage/distributor/distributor.cpp @@ -5,7 +5,7 @@ #include "throttlingoperationstarter.h" #include "idealstatemetricsset.h" #include "ownership_transfer_safe_time_point_calculator.h" -#include "managed_bucket_space_repo.h" +#include "distributor_bucket_space_repo.h" #include <vespa/storage/bucketdb/mapbucketdatabase.h> #include <vespa/storage/distributor/maintenance/simplemaintenancescanner.h> #include <vespa/storage/distributor/maintenance/simplebucketprioritydatabase.h> @@ -66,7 +66,7 @@ Distributor::Distributor(DistributorComponentRegister& compReg, framework::StatusReporter("distributor", "Distributor"), _compReg(compReg), _component(compReg, "distributor"), - _bucketSpaceRepo(std::make_unique<ManagedBucketSpaceRepo>()), + _bucketSpaceRepo(std::make_unique<DistributorBucketSpaceRepo>()), _metrics(new DistributorMetricSet( _component.getLoadTypes()->getMetricLoadTypes())), _operationOwner(*this, _component.getClock()), @@ -143,11 +143,11 @@ Distributor::getPendingMessageTracker() const return _pendingMessageTracker; } -ManagedBucketSpace& Distributor::getDefaultBucketSpace() noexcept { +DistributorBucketSpace& Distributor::getDefaultBucketSpace() noexcept { return _bucketSpaceRepo->getDefaultSpace(); } -const ManagedBucketSpace& Distributor::getDefaultBucketSpace() const noexcept { +const DistributorBucketSpace& Distributor::getDefaultBucketSpace() const noexcept { return _bucketSpaceRepo->getDefaultSpace(); } @@ -307,7 +307,7 @@ Distributor::handleReply(const std::shared_ptr<api::StorageReply>& reply) } if (_maintenanceOperationOwner.handleReply(reply)) { - _scanner->prioritizeBucket(bucket.getBucketId()); + _scanner->prioritizeBucket(bucket); return true; } diff --git a/storage/src/vespa/storage/distributor/distributor.h b/storage/src/vespa/storage/distributor/distributor.h index 610c7bfef59..3b74676f937 100644 --- a/storage/src/vespa/storage/distributor/distributor.h +++ b/storage/src/vespa/storage/distributor/distributor.h @@ -32,7 +32,7 @@ class HostInfo; namespace distributor { -class ManagedBucketSpaceRepo; +class DistributorBucketSpaceRepo; class SimpleMaintenanceScanner; class BlockingOperationStarter; class ThrottlingOperationStarter; @@ -159,8 +159,8 @@ public: return *_bucketIdHasher; } - ManagedBucketSpace& getDefaultBucketSpace() noexcept; - const ManagedBucketSpace& getDefaultBucketSpace() const noexcept; + DistributorBucketSpace& getDefaultBucketSpace() noexcept; + const DistributorBucketSpace& getDefaultBucketSpace() const noexcept; private: friend class Distributor_Test; @@ -238,7 +238,7 @@ private: DistributorComponentRegister& _compReg; storage::DistributorComponent _component; - std::unique_ptr<ManagedBucketSpaceRepo> _bucketSpaceRepo; + std::unique_ptr<DistributorBucketSpaceRepo> _bucketSpaceRepo; std::shared_ptr<DistributorMetricSet> _metrics; OperationOwner _operationOwner; diff --git a/storage/src/vespa/storage/distributor/managed_bucket_space.cpp b/storage/src/vespa/storage/distributor/distributor_bucket_space.cpp index e507f1a32aa..b33ff72a654 100644 --- a/storage/src/vespa/storage/distributor/managed_bucket_space.cpp +++ b/storage/src/vespa/storage/distributor/distributor_bucket_space.cpp @@ -1,14 +1,14 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "managed_bucket_space.h" +#include "distributor_bucket_space.h" namespace storage { namespace distributor { -ManagedBucketSpace::ManagedBucketSpace() { +DistributorBucketSpace::DistributorBucketSpace() { } -ManagedBucketSpace::~ManagedBucketSpace() { +DistributorBucketSpace::~DistributorBucketSpace() { } } diff --git a/storage/src/vespa/storage/distributor/managed_bucket_space.h b/storage/src/vespa/storage/distributor/distributor_bucket_space.h index e865d888f8d..d2264263725 100644 --- a/storage/src/vespa/storage/distributor/managed_bucket_space.h +++ b/storage/src/vespa/storage/distributor/distributor_bucket_space.h @@ -24,17 +24,17 @@ namespace distributor { * particular so that redundancy, ready copies etc can differ across * bucket spaces. */ -class ManagedBucketSpace { +class DistributorBucketSpace { MapBucketDatabase _bucketDatabase; std::shared_ptr<lib::Distribution> _distribution; public: - ManagedBucketSpace(); - ~ManagedBucketSpace(); + DistributorBucketSpace(); + ~DistributorBucketSpace(); - ManagedBucketSpace(const ManagedBucketSpace&) = delete; - ManagedBucketSpace& operator=(const ManagedBucketSpace&) = delete; - ManagedBucketSpace(ManagedBucketSpace&&) = delete; - ManagedBucketSpace& operator=(ManagedBucketSpace&&) = delete; + DistributorBucketSpace(const DistributorBucketSpace&) = delete; + DistributorBucketSpace& operator=(const DistributorBucketSpace&) = delete; + DistributorBucketSpace(DistributorBucketSpace&&) = delete; + DistributorBucketSpace& operator=(DistributorBucketSpace&&) = delete; BucketDatabase& getBucketDatabase() noexcept { return _bucketDatabase; diff --git a/storage/src/vespa/storage/distributor/managed_bucket_space_component.cpp b/storage/src/vespa/storage/distributor/distributor_bucket_space_component.cpp index 3d50f3c9946..22571fb3d5c 100644 --- a/storage/src/vespa/storage/distributor/managed_bucket_space_component.cpp +++ b/storage/src/vespa/storage/distributor/distributor_bucket_space_component.cpp @@ -1,12 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "managed_bucket_space_component.h" +#include "distributor_bucket_space_component.h" namespace storage::distributor { -ManagedBucketSpaceComponent::ManagedBucketSpaceComponent( +DistributorBucketSpaceComponent::DistributorBucketSpaceComponent( DistributorInterface& distributor, - ManagedBucketSpace& bucketSpace, + DistributorBucketSpace& bucketSpace, DistributorComponentRegister& compReg, const std::string& name) : DistributorComponent(distributor, compReg, name), diff --git a/storage/src/vespa/storage/distributor/managed_bucket_space_component.h b/storage/src/vespa/storage/distributor/distributor_bucket_space_component.h index 0dbd20329cb..9a000d28117 100644 --- a/storage/src/vespa/storage/distributor/managed_bucket_space_component.h +++ b/storage/src/vespa/storage/distributor/distributor_bucket_space_component.h @@ -2,7 +2,7 @@ #pragma once #include "distributorcomponent.h" -#include "managed_bucket_space.h" +#include "distributor_bucket_space.h" namespace storage { namespace distributor { @@ -11,11 +11,11 @@ namespace distributor { * Component bound to a specific bucket space, with utility operations to * operate on buckets in this space. */ -class ManagedBucketSpaceComponent : public DistributorComponent { - ManagedBucketSpace& _bucketSpace; +class DistributorBucketSpaceComponent : public DistributorComponent { + DistributorBucketSpace& _bucketSpace; public: - ManagedBucketSpaceComponent(DistributorInterface& distributor, - ManagedBucketSpace& bucketSpace, + DistributorBucketSpaceComponent(DistributorInterface& distributor, + DistributorBucketSpace& bucketSpace, DistributorComponentRegister& compReg, const std::string& name); diff --git a/storage/src/vespa/storage/distributor/managed_bucket_space_repo.cpp b/storage/src/vespa/storage/distributor/distributor_bucket_space_repo.cpp index efb970ed2dd..b6dc1e856d7 100644 --- a/storage/src/vespa/storage/distributor/managed_bucket_space_repo.cpp +++ b/storage/src/vespa/storage/distributor/distributor_bucket_space_repo.cpp @@ -1,6 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "managed_bucket_space_repo.h" +#include "distributor_bucket_space_repo.h" #include <vespa/vdslib/distribution/distribution.h> #include <vespa/log/log.h> @@ -9,13 +9,13 @@ LOG_SETUP(".distributor.managed_bucket_space_repo"); namespace storage { namespace distributor { -ManagedBucketSpaceRepo::ManagedBucketSpaceRepo() { +DistributorBucketSpaceRepo::DistributorBucketSpaceRepo() { } -ManagedBucketSpaceRepo::~ManagedBucketSpaceRepo() { +DistributorBucketSpaceRepo::~DistributorBucketSpaceRepo() { } -void ManagedBucketSpaceRepo::setDefaultDistribution( +void DistributorBucketSpaceRepo::setDefaultDistribution( std::shared_ptr<lib::Distribution> distr) { LOG(debug, "Got new default distribution '%s'", distr->toString().c_str()); diff --git a/storage/src/vespa/storage/distributor/distributor_bucket_space_repo.h b/storage/src/vespa/storage/distributor/distributor_bucket_space_repo.h new file mode 100644 index 00000000000..1e0fc375eca --- /dev/null +++ b/storage/src/vespa/storage/distributor/distributor_bucket_space_repo.h @@ -0,0 +1,32 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "distributor_bucket_space.h" +#include <memory> + +namespace storage { + +namespace distributor { + +class DistributorBucketSpaceRepo { + // TODO: multiple spaces. This is just to start re-wiring things. + DistributorBucketSpace _defaultSpace; +public: + DistributorBucketSpaceRepo(); + ~DistributorBucketSpaceRepo(); + + DistributorBucketSpaceRepo(const DistributorBucketSpaceRepo&&) = delete; + DistributorBucketSpaceRepo& operator=(const DistributorBucketSpaceRepo&) = delete; + DistributorBucketSpaceRepo(DistributorBucketSpaceRepo&&) = delete; + DistributorBucketSpaceRepo& operator=(DistributorBucketSpaceRepo&&) = delete; + + DistributorBucketSpace& getDefaultSpace() noexcept { return _defaultSpace; } + const DistributorBucketSpace& getDefaultSpace() const noexcept { + return _defaultSpace; + } + + void setDefaultDistribution(std::shared_ptr<lib::Distribution> distr); +}; + +} +} diff --git a/storage/src/vespa/storage/distributor/externaloperationhandler.cpp b/storage/src/vespa/storage/distributor/externaloperationhandler.cpp index e67e889ff7e..5bc2658e242 100644 --- a/storage/src/vespa/storage/distributor/externaloperationhandler.cpp +++ b/storage/src/vespa/storage/distributor/externaloperationhandler.cpp @@ -27,10 +27,10 @@ namespace storage::distributor { ExternalOperationHandler::ExternalOperationHandler( Distributor& owner, - ManagedBucketSpace& bucketSpace, + DistributorBucketSpace& bucketSpace, const MaintenanceOperationGenerator& gen, DistributorComponentRegister& compReg) - : ManagedBucketSpaceComponent(owner, bucketSpace, compReg, "External operation handler"), + : DistributorBucketSpaceComponent(owner, bucketSpace, compReg, "External operation handler"), _operationGenerator(gen), _rejectFeedBeforeTimeReached() // At epoch { } diff --git a/storage/src/vespa/storage/distributor/externaloperationhandler.h b/storage/src/vespa/storage/distributor/externaloperationhandler.h index 654f9c6b70b..6d7078d0405 100644 --- a/storage/src/vespa/storage/distributor/externaloperationhandler.h +++ b/storage/src/vespa/storage/distributor/externaloperationhandler.h @@ -6,7 +6,7 @@ #include <vespa/document/bucket/bucketidfactory.h> #include <vespa/vdslib/state/clusterstate.h> #include <vespa/storage/distributor/distributorcomponent.h> -#include <vespa/storage/distributor/managed_bucket_space_component.h> +#include <vespa/storage/distributor/distributor_bucket_space_component.h> #include <vespa/storageapi/messageapi/messagehandler.h> #include <chrono> @@ -20,7 +20,7 @@ namespace distributor { class Distributor; class MaintenanceOperationGenerator; -class ExternalOperationHandler : public ManagedBucketSpaceComponent, +class ExternalOperationHandler : public DistributorBucketSpaceComponent, public api::MessageHandler { public: @@ -38,7 +38,7 @@ public: DEF_MSG_COMMAND_H(GetBucketList); ExternalOperationHandler(Distributor& owner, - ManagedBucketSpace& bucketSpace, + DistributorBucketSpace& bucketSpace, const MaintenanceOperationGenerator&, DistributorComponentRegister& compReg); diff --git a/storage/src/vespa/storage/distributor/idealstatemanager.cpp b/storage/src/vespa/storage/distributor/idealstatemanager.cpp index 4fd23f99f02..bd9e9ca471d 100644 --- a/storage/src/vespa/storage/distributor/idealstatemanager.cpp +++ b/storage/src/vespa/storage/distributor/idealstatemanager.cpp @@ -22,7 +22,7 @@ namespace distributor { IdealStateManager::IdealStateManager( Distributor& owner, - ManagedBucketSpace& bucketSpace, + DistributorBucketSpace& bucketSpace, DistributorComponentRegister& compReg, bool manageActiveBucketCopies) : HtmlStatusReporter("idealstateman", "Ideal state manager"), @@ -139,10 +139,10 @@ IdealStateManager::runStateCheckers(StateChecker::Context& c) const StateChecker::Result IdealStateManager::generateHighestPriority( - const document::BucketId& bid, + const document::Bucket &bucket, NodeMaintenanceStatsTracker& statsTracker) const { - StateChecker::Context c(_distributorComponent, statsTracker, bid); + StateChecker::Context c(_distributorComponent, statsTracker, bucket.getBucketId()); fillParentAndChildBuckets(c); fillSiblingBucket(c); @@ -158,11 +158,11 @@ IdealStateManager::generateHighestPriority( MaintenancePriorityAndType IdealStateManager::prioritize( - const document::BucketId& bucketId, + const document::Bucket &bucket, NodeMaintenanceStatsTracker& statsTracker) const { StateChecker::Result generated( - generateHighestPriority(bucketId, statsTracker)); + generateHighestPriority(bucket, statsTracker)); MaintenancePriority priority(generated.getPriority()); MaintenanceOperation::Type type(priority.requiresMaintenance() ? generated.getType() @@ -193,11 +193,11 @@ IdealStateManager::generateInterceptingSplit(const BucketDatabase::Entry& e, } MaintenanceOperation::SP -IdealStateManager::generate(const document::BucketId& bucketId) const +IdealStateManager::generate(const document::Bucket &bucket) const { NodeMaintenanceStatsTracker statsTracker; IdealStateOperation::SP op( - generateHighestPriority(bucketId, statsTracker).createOperation()); + generateHighestPriority(bucket, statsTracker).createOperation()); if (op.get()) { op->setIdealStateManager( const_cast<IdealStateManager*>(this)); @@ -206,10 +206,10 @@ IdealStateManager::generate(const document::BucketId& bucketId) const } std::vector<MaintenanceOperation::SP> -IdealStateManager::generateAll(const document::BucketId& bucketId, +IdealStateManager::generateAll(const document::Bucket &bucket, NodeMaintenanceStatsTracker& statsTracker) const { - StateChecker::Context c(_distributorComponent, statsTracker, bucketId); + StateChecker::Context c(_distributorComponent, statsTracker, bucket.getBucketId()); fillParentAndChildBuckets(c); fillSiblingBucket(c); BucketDatabase::Entry* e(getEntryForPrimaryBucket(c)); @@ -232,6 +232,7 @@ IdealStateManager::generateAll(const document::BucketId& bucketId, void IdealStateManager::getBucketStatus( + document::BucketSpace bucketSpace, const BucketDatabase::Entry& entry, NodeMaintenanceStatsTracker& statsTracker, std::ostream& out) const @@ -239,8 +240,9 @@ IdealStateManager::getBucketStatus( LOG(debug, "Dumping bucket database valid at cluster state version %u", _distributorComponent.getDistributor().getClusterState().getVersion()); + document::Bucket bucket(bucketSpace, entry.getBucketId()); std::vector<MaintenanceOperation::SP> operations( - generateAll(entry.getBucketId(), statsTracker)); + generateAll(bucket, statsTracker)); if (operations.empty()) { out << entry.getBucketId() << " : "; } else { @@ -262,7 +264,7 @@ IdealStateManager::getBucketStatus( void IdealStateManager::getBucketStatus(std::ostream& out) const { - StatusBucketVisitor proc(*this, out); + StatusBucketVisitor proc(*this, document::BucketSpace::placeHolder(), out); _distributorComponent.getBucketDatabase().forEach(proc); } diff --git a/storage/src/vespa/storage/distributor/idealstatemanager.h b/storage/src/vespa/storage/distributor/idealstatemanager.h index 36287c73f78..567efb9f347 100644 --- a/storage/src/vespa/storage/distributor/idealstatemanager.h +++ b/storage/src/vespa/storage/distributor/idealstatemanager.h @@ -4,7 +4,7 @@ #include <deque> #include <map> #include <set> -#include <vespa/storage/distributor/managed_bucket_space_component.h> +#include <vespa/storage/distributor/distributor_bucket_space_component.h> #include <vespa/storage/distributor/statechecker.h> #include <vespa/storage/distributor/maintenance/maintenanceprioritygenerator.h> #include <vespa/storage/distributor/maintenance/maintenanceoperationgenerator.h> @@ -40,7 +40,7 @@ class IdealStateManager : public framework::HtmlStatusReporter, public: IdealStateManager(Distributor& owner, - ManagedBucketSpace& bucketSpace, + DistributorBucketSpace& bucketSpace, DistributorComponentRegister& compReg, bool manageActiveBucketCopies); @@ -51,16 +51,15 @@ public: // MaintenancePriorityGenerator interface MaintenancePriorityAndType prioritize( - const document::BucketId& bucketId, + const document::Bucket &bucket, NodeMaintenanceStatsTracker& statsTracker) const override; // MaintenanceOperationGenerator - MaintenanceOperation::SP generate( - const document::BucketId& bucketId) const override; + MaintenanceOperation::SP generate(const document::Bucket &bucket) const override; // MaintenanceOperationGenerator std::vector<MaintenanceOperation::SP> generateAll( - const document::BucketId& bucketId, + const document::Bucket &bucket, NodeMaintenanceStatsTracker& statsTracker) const override; /** @@ -90,7 +89,7 @@ private: void fillParentAndChildBuckets(StateChecker::Context& c) const; void fillSiblingBucket(StateChecker::Context& c) const; StateChecker::Result generateHighestPriority( - const document::BucketId& bucketId, + const document::Bucket &bucket, NodeMaintenanceStatsTracker& statsTracker) const; StateChecker::Result runStateCheckers(StateChecker::Context& c) const; @@ -111,7 +110,7 @@ private: std::vector<StateChecker::SP> _stateCheckers; SplitBucketStateChecker* _splitBucketStateChecker; - ManagedBucketSpaceComponent _distributorComponent; + DistributorBucketSpaceComponent _distributorComponent; std::vector<IdealStateOperation::SP> generateOperationsForBucket( StateChecker::Context& c) const; @@ -123,19 +122,21 @@ private: // to create a new hash map for each single bucket processed. NodeMaintenanceStatsTracker _statsTracker; const IdealStateManager& _ism; + document::BucketSpace _bucketSpace; std::ostream& _out; public: - StatusBucketVisitor(const IdealStateManager& ism, std::ostream& out) - : _ism(ism), _out(out) {} + StatusBucketVisitor(const IdealStateManager& ism, document::BucketSpace bucketSpace, std::ostream& out) + : _statsTracker(), _ism(ism), _bucketSpace(bucketSpace), _out(out) {} bool process(const BucketDatabase::Entry& e) override { - _ism.getBucketStatus(e, _statsTracker, _out); + _ism.getBucketStatus(_bucketSpace, e, _statsTracker, _out); return true; } }; friend class StatusBucketVisitor; - void getBucketStatus(const BucketDatabase::Entry& entry, + void getBucketStatus(document::BucketSpace bucketSpace, + const BucketDatabase::Entry& entry, NodeMaintenanceStatsTracker& statsTracker, std::ostream& out) const; diff --git a/storage/src/vespa/storage/distributor/maintenance/maintenanceoperationgenerator.h b/storage/src/vespa/storage/distributor/maintenance/maintenanceoperationgenerator.h index 181420f7c68..1afc0990222 100644 --- a/storage/src/vespa/storage/distributor/maintenance/maintenanceoperationgenerator.h +++ b/storage/src/vespa/storage/distributor/maintenance/maintenanceoperationgenerator.h @@ -18,8 +18,7 @@ public: * the given bucket. If the bucket does not need maintenance, a nullptr * shared_ptr is returned. */ - virtual MaintenanceOperation::SP generate( - const document::BucketId&) const = 0; + virtual MaintenanceOperation::SP generate(const document::Bucket &bucket) const = 0; /** * Generate all possible maintenance operations for the given bucket and @@ -27,18 +26,17 @@ public: * does not need maintenance, the returned vector will be empty. */ virtual std::vector<MaintenanceOperation::SP> generateAll( - const document::BucketId&, - NodeMaintenanceStatsTracker&) const = 0; + const document::Bucket &bucket, + NodeMaintenanceStatsTracker &statsTracker) const = 0; /** * Convenience wrapper around generateAll() for when there's no need for * an explicit stats tracker */ - std::vector<MaintenanceOperation::SP> generateAll( - const document::BucketId& bucketId) const + std::vector<MaintenanceOperation::SP> generateAll(const document::Bucket &bucket) const { NodeMaintenanceStatsTracker dummyTracker; - return generateAll(bucketId, dummyTracker); + return generateAll(bucket, dummyTracker); } }; diff --git a/storage/src/vespa/storage/distributor/maintenance/maintenanceprioritygenerator.h b/storage/src/vespa/storage/distributor/maintenance/maintenanceprioritygenerator.h index 38e5137d63a..ca7fb6ae81f 100644 --- a/storage/src/vespa/storage/distributor/maintenance/maintenanceprioritygenerator.h +++ b/storage/src/vespa/storage/distributor/maintenance/maintenanceprioritygenerator.h @@ -15,8 +15,8 @@ public: virtual ~MaintenancePriorityGenerator() {} virtual MaintenancePriorityAndType prioritize( - const document::BucketId&, - NodeMaintenanceStatsTracker&) const = 0; + const document::Bucket &bucket, + NodeMaintenanceStatsTracker &statsTarcker) const = 0; }; } diff --git a/storage/src/vespa/storage/distributor/maintenance/maintenancescheduler.cpp b/storage/src/vespa/storage/distributor/maintenance/maintenancescheduler.cpp index 5e339757fc0..c158b499ebb 100644 --- a/storage/src/vespa/storage/distributor/maintenance/maintenancescheduler.cpp +++ b/storage/src/vespa/storage/distributor/maintenance/maintenancescheduler.cpp @@ -64,7 +64,7 @@ MaintenanceScheduler::possibleToScheduleInEmergency( void MaintenanceScheduler::clearPriority(const PrioritizedBucket& bucket) { - _priorityDb.setPriority(PrioritizedBucket(bucket.getBucketId(), + _priorityDb.setPriority(PrioritizedBucket(bucket.getBucket(), MaintenancePriority::NO_MAINTENANCE_NEEDED)); } @@ -91,7 +91,7 @@ MaintenanceScheduler::convertToOperationPriority(MaintenancePriority::Priority p bool MaintenanceScheduler::startOperation(const PrioritizedBucket& bucket) { - Operation::SP operation(_operationGenerator.generate(bucket.getBucketId())); + Operation::SP operation(_operationGenerator.generate(bucket.getBucket())); if (!operation) { return true; } diff --git a/storage/src/vespa/storage/distributor/maintenance/prioritizedbucket.h b/storage/src/vespa/storage/distributor/maintenance/prioritizedbucket.h index 55fb4220a73..fde73de0358 100644 --- a/storage/src/vespa/storage/distributor/maintenance/prioritizedbucket.h +++ b/storage/src/vespa/storage/distributor/maintenance/prioritizedbucket.h @@ -2,7 +2,7 @@ #pragma once #include <iosfwd> -#include <vespa/document/bucket/bucketid.h> +#include <vespa/document/bucket/bucket.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/storage/distributor/maintenance/maintenancepriority.h> @@ -17,37 +17,34 @@ public: static const PrioritizedBucket INVALID; PrioritizedBucket() - : _bucketId(), + : _bucket(), _priority(MaintenancePriority::NO_MAINTENANCE_NEEDED) {} - PrioritizedBucket(const document::BucketId& bid, - Priority pri) - : _bucketId(bid), + PrioritizedBucket(const document::Bucket &bucket, Priority pri) + : _bucket(bucket), _priority(pri) { } - const document::BucketId& getBucketId() const { - return _bucketId; - } + document::Bucket getBucket() const { return _bucket; } Priority getPriority() const { return _priority; } bool valid() const { - return _bucketId.getRawId() != 0; + return _bucket.getBucketId().getRawId() != 0; } std::string toString() const { return vespalib::make_string("PrioritizedBucket(%s, pri %s)", - _bucketId.toString().c_str(), + _bucket.toString().c_str(), MaintenancePriority::toString(_priority).c_str()); } bool operator==(const PrioritizedBucket& other) const { - return _bucketId == other._bucketId && _priority == other._priority; + return _bucket == other._bucket && _priority == other._priority; } bool requiresMaintenance() const { @@ -63,7 +60,7 @@ public: } private: - document::BucketId _bucketId; + document::Bucket _bucket; Priority _priority; }; diff --git a/storage/src/vespa/storage/distributor/maintenance/simplebucketprioritydatabase.cpp b/storage/src/vespa/storage/distributor/maintenance/simplebucketprioritydatabase.cpp index af770859950..56f43db1ac5 100644 --- a/storage/src/vespa/storage/distributor/maintenance/simplebucketprioritydatabase.cpp +++ b/storage/src/vespa/storage/distributor/maintenance/simplebucketprioritydatabase.cpp @@ -11,24 +11,23 @@ SimpleBucketPriorityDatabase::~SimpleBucketPriorityDatabase() } void -SimpleBucketPriorityDatabase::clearAllEntriesForBucket( - const document::BucketId& bucketId) +SimpleBucketPriorityDatabase::clearAllEntriesForBucket(const document::Bucket &bucket) { for (PriorityMap::iterator priIter(_prioritizedBuckets.begin()), priEnd(_prioritizedBuckets.end()); priIter != priEnd; ++priIter) { - priIter->second.erase(bucketId); + priIter->second.erase(bucket); } } void SimpleBucketPriorityDatabase::setPriority(const PrioritizedBucket& bucket) { - clearAllEntriesForBucket(bucket.getBucketId()); + clearAllEntriesForBucket(bucket.getBucket()); if (bucket.requiresMaintenance()) { - _prioritizedBuckets[bucket.getPriority()].insert(bucket.getBucketId()); + _prioritizedBuckets[bucket.getPriority()].insert(bucket.getBucket()); } } diff --git a/storage/src/vespa/storage/distributor/maintenance/simplebucketprioritydatabase.h b/storage/src/vespa/storage/distributor/maintenance/simplebucketprioritydatabase.h index 1876b2cf5ed..70b007decb6 100644 --- a/storage/src/vespa/storage/distributor/maintenance/simplebucketprioritydatabase.h +++ b/storage/src/vespa/storage/distributor/maintenance/simplebucketprioritydatabase.h @@ -21,7 +21,7 @@ public: std::string toString() const; private: - typedef std::set<document::BucketId> BucketSet; + typedef std::set<document::Bucket> BucketSet; typedef std::map<Priority, BucketSet> PriorityMap; class SimpleConstIteratorImpl : public ConstIteratorImpl @@ -56,7 +56,7 @@ private: PrioritizedBucket dereference() const override; }; - void clearAllEntriesForBucket(const document::BucketId& bucketId); + void clearAllEntriesForBucket(const document::Bucket &bucket); PriorityMap _prioritizedBuckets; }; diff --git a/storage/src/vespa/storage/distributor/maintenance/simplemaintenancescanner.cpp b/storage/src/vespa/storage/distributor/maintenance/simplemaintenancescanner.cpp index d2ddb3161b0..2bdef7ed320 100644 --- a/storage/src/vespa/storage/distributor/maintenance/simplemaintenancescanner.cpp +++ b/storage/src/vespa/storage/distributor/maintenance/simplemaintenancescanner.cpp @@ -18,7 +18,7 @@ SimpleMaintenanceScanner::scanNext() if (!entry.valid()) { return ScanResult::createDone(); } - prioritizeBucket(entry.getBucketId()); + prioritizeBucket(document::Bucket(document::BucketSpace::placeHolder(), entry.getBucketId())); _bucketCursor = entry.getBucketId(); return ScanResult::createNotDone(entry); } @@ -31,11 +31,11 @@ SimpleMaintenanceScanner::reset() } void -SimpleMaintenanceScanner::prioritizeBucket(const document::BucketId& id) +SimpleMaintenanceScanner::prioritizeBucket(const document::Bucket &bucket) { - MaintenancePriorityAndType pri(_priorityGenerator.prioritize(id, _pendingMaintenance.perNodeStats)); + MaintenancePriorityAndType pri(_priorityGenerator.prioritize(bucket, _pendingMaintenance.perNodeStats)); if (pri.requiresMaintenance()) { - _bucketPriorityDb.setPriority(PrioritizedBucket(id, pri.getPriority().getPriority())); + _bucketPriorityDb.setPriority(PrioritizedBucket(bucket, pri.getPriority().getPriority())); assert(pri.getType() != MaintenanceOperation::OPERATION_COUNT); ++_pendingMaintenance.global.pending[pri.getType()]; } diff --git a/storage/src/vespa/storage/distributor/maintenance/simplemaintenancescanner.h b/storage/src/vespa/storage/distributor/maintenance/simplemaintenancescanner.h index f1452656e47..05de7674d6a 100644 --- a/storage/src/vespa/storage/distributor/maintenance/simplemaintenancescanner.h +++ b/storage/src/vespa/storage/distributor/maintenance/simplemaintenancescanner.h @@ -51,7 +51,7 @@ public: void reset() override; // TODO: move out into own interface! - void prioritizeBucket(const document::BucketId& id); + void prioritizeBucket(const document::Bucket &id); const PendingMaintenanceStats& getPendingMaintenanceStats() const { return _pendingMaintenance; diff --git a/storage/src/vespa/storage/distributor/managed_bucket_space_repo.h b/storage/src/vespa/storage/distributor/managed_bucket_space_repo.h deleted file mode 100644 index afda84dc9e5..00000000000 --- a/storage/src/vespa/storage/distributor/managed_bucket_space_repo.h +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include "managed_bucket_space.h" -#include <memory> - -namespace storage { - -namespace distributor { - -class ManagedBucketSpaceRepo { - // TODO: multiple spaces. This is just to start re-wiring things. - ManagedBucketSpace _defaultSpace; -public: - ManagedBucketSpaceRepo(); - ~ManagedBucketSpaceRepo(); - - ManagedBucketSpaceRepo(const ManagedBucketSpaceRepo&&) = delete; - ManagedBucketSpaceRepo& operator=(const ManagedBucketSpaceRepo&) = delete; - ManagedBucketSpaceRepo(ManagedBucketSpaceRepo&&) = delete; - ManagedBucketSpaceRepo& operator=(ManagedBucketSpaceRepo&&) = delete; - - ManagedBucketSpace& getDefaultSpace() noexcept { return _defaultSpace; } - const ManagedBucketSpace& getDefaultSpace() const noexcept { - return _defaultSpace; - } - - void setDefaultDistribution(std::shared_ptr<lib::Distribution> distr); -}; - -} -} diff --git a/storage/src/vespa/storage/distributor/operations/external/statbucketlistoperation.cpp b/storage/src/vespa/storage/distributor/operations/external/statbucketlistoperation.cpp index fea6c539c81..4b7cff41ad1 100644 --- a/storage/src/vespa/storage/distributor/operations/external/statbucketlistoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/external/statbucketlistoperation.cpp @@ -23,8 +23,8 @@ void StatBucketListOperation::getBucketStatus(const BucketDatabase::Entry& entry, std::ostream& ost) const { - std::vector<MaintenanceOperation::SP> operations( - _generator.generateAll(entry.getBucketId())); + document::Bucket bucket(_command->getBucket().getBucketSpace(), entry.getBucketId()); + std::vector<MaintenanceOperation::SP> operations(_generator.generateAll(bucket)); for (uint32_t i = 0; i < operations.size(); ++i) { const MaintenanceOperation& op(*operations[i]); diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/idealstateoperation.h b/storage/src/vespa/storage/distributor/operations/idealstate/idealstateoperation.h index 54a85b1873f..9824ae0630f 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/idealstateoperation.h +++ b/storage/src/vespa/storage/distributor/operations/idealstate/idealstateoperation.h @@ -29,18 +29,18 @@ public: /** Constructor for operations having only one node. - @param id Target bucket + @param bucket Target bucket @param node Target node */ - BucketAndNodes(const document::Bucket &id, uint16_t node); + BucketAndNodes(const document::Bucket &bucket, uint16_t node); /** Constructor for operations with multiple target nodes. - @param id Target bucket + @param bucket Target bucket @param nodes Target nodes */ - BucketAndNodes(const document::Bucket &id, + BucketAndNodes(const document::Bucket &bucket, const std::vector<uint16_t>& nodes); /** diff --git a/storage/src/vespa/storage/frameworkimpl/component/servicelayercomponentregisterimpl.cpp b/storage/src/vespa/storage/frameworkimpl/component/servicelayercomponentregisterimpl.cpp index a2699edd258..da734c07c2d 100644 --- a/storage/src/vespa/storage/frameworkimpl/component/servicelayercomponentregisterimpl.cpp +++ b/storage/src/vespa/storage/frameworkimpl/component/servicelayercomponentregisterimpl.cpp @@ -9,7 +9,7 @@ using vespalib::IllegalStateException; ServiceLayerComponentRegisterImpl::ServiceLayerComponentRegisterImpl() : _diskCount(0), - _bucketDatabase() + _bucketSpaceRepo() { } void @@ -19,7 +19,7 @@ ServiceLayerComponentRegisterImpl::registerServiceLayerComponent( vespalib::LockGuard lock(_componentLock); _components.push_back(&smc); smc.setDiskCount(_diskCount); - smc.setBucketDatabase(_bucketDatabase); + smc.setBucketSpaceRepo(_bucketSpaceRepo); smc.setMinUsedBitsTracker(_minUsedBitsTracker); } diff --git a/storage/src/vespa/storage/frameworkimpl/component/servicelayercomponentregisterimpl.h b/storage/src/vespa/storage/frameworkimpl/component/servicelayercomponentregisterimpl.h index b46d795cfcc..5b3e54e3831 100644 --- a/storage/src/vespa/storage/frameworkimpl/component/servicelayercomponentregisterimpl.h +++ b/storage/src/vespa/storage/frameworkimpl/component/servicelayercomponentregisterimpl.h @@ -9,7 +9,7 @@ #include "storagecomponentregisterimpl.h" #include <vespa/storage/bucketdb/minimumusedbitstracker.h> -#include <vespa/storage/bucketdb/storbucketdb.h> +#include <vespa/storage/common/content_bucket_space_repo.h> #include <vespa/storage/common/servicelayercomponent.h> namespace storage { @@ -21,7 +21,7 @@ class ServiceLayerComponentRegisterImpl vespalib::Lock _componentLock; std::vector<ServiceLayerManagedComponent*> _components; uint16_t _diskCount; - StorBucketDatabase _bucketDatabase; + ContentBucketSpaceRepo _bucketSpaceRepo; MinimumUsedBitsTracker _minUsedBitsTracker; public: @@ -30,7 +30,7 @@ public: ServiceLayerComponentRegisterImpl(); uint16_t getDiskCount() const { return _diskCount; } - StorBucketDatabase& getBucketDatabase() { return _bucketDatabase; } + ContentBucketSpaceRepo& getBucketSpaceRepo() { return _bucketSpaceRepo; } MinimumUsedBitsTracker& getMinUsedBitsTracker() { return _minUsedBitsTracker; } diff --git a/vespa-documentgen-plugin/OWNERS b/vespa-documentgen-plugin/OWNERS index f1a0dc35056..255614a9af9 100644 --- a/vespa-documentgen-plugin/OWNERS +++ b/vespa-documentgen-plugin/OWNERS @@ -1,2 +1 @@ kraune - |