diff options
227 files changed, 3386 insertions, 3372 deletions
diff --git a/application/src/main/java/com/yahoo/application/Application.java b/application/src/main/java/com/yahoo/application/Application.java index f7b5174b0e5..fb812ba6107 100644 --- a/application/src/main/java/com/yahoo/application/Application.java +++ b/application/src/main/java/com/yahoo/application/Application.java @@ -46,7 +46,7 @@ public final class Application implements AutoCloseable { /** * This system property is set to "true" upon creation of an Application. - * This is useful for components which are created by dependendcy injection which needs to modify + * This is useful for components which are created by dependency injection which needs to modify * their behavior to function without reliance on any processes outside the JVM. */ public static final String vespaLocalProperty = "vespa.local"; diff --git a/application/src/main/java/com/yahoo/application/container/JDisc.java b/application/src/main/java/com/yahoo/application/container/JDisc.java index 10ff84d3ede..ee22c58a56f 100644 --- a/application/src/main/java/com/yahoo/application/container/JDisc.java +++ b/application/src/main/java/com/yahoo/application/container/JDisc.java @@ -21,7 +21,6 @@ import com.yahoo.jdisc.handler.RequestHandler; import com.yahoo.jdisc.test.TestDriver; import com.yahoo.processing.handler.ProcessingHandler; import com.yahoo.search.handler.SearchHandler; -import com.yahoo.search.searchchain.ExecutionFactory; import java.nio.file.Path; diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ConfigserverSslContextFactoryProvider.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ConfigserverSslContextFactoryProvider.java index bb3216ba3ba..2bda2eb3627 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ConfigserverSslContextFactoryProvider.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ConfigserverSslContextFactoryProvider.java @@ -37,8 +37,6 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.logging.Logger; -import static com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl.Utils.getZoneConfig; - /** * Configures the JDisc https connector with the configserver's Athenz provider certificate and private key. * @@ -56,7 +54,7 @@ public class ConfigserverSslContextFactoryProvider extends AbstractComponent imp Executors.newSingleThreadScheduledExecutor(runnable -> new Thread(runnable, "configserver-ssl-context-factory-provider")); private final ZtsClient ztsClient; private final KeyProvider keyProvider; - private final AthenzProviderServiceConfig.Zones zoneConfig; + private final AthenzProviderServiceConfig athenzProviderServiceConfig; private final AthenzService configserverIdentity; @Inject @@ -64,14 +62,14 @@ public class ConfigserverSslContextFactoryProvider extends AbstractComponent imp KeyProvider keyProvider, AthenzProviderServiceConfig config, Zone zone) { - this.zoneConfig = getZoneConfig(config, zone); - this.ztsClient = new DefaultZtsClient(URI.create(zoneConfig.ztsUrl()), bootstrapIdentity); + this.athenzProviderServiceConfig = config; + this.ztsClient = new DefaultZtsClient(URI.create(athenzProviderServiceConfig.ztsUrl()), bootstrapIdentity); this.keyProvider = keyProvider; - this.configserverIdentity = new AthenzService(zoneConfig.domain(), zoneConfig.serviceName()); + this.configserverIdentity = new AthenzService(athenzProviderServiceConfig.domain(), athenzProviderServiceConfig.serviceName()); Duration updatePeriod = Duration.ofDays(config.updatePeriodDays()); Path trustStoreFile = Paths.get(config.athenzCaTrustStore()); - this.sslContextFactory = initializeSslContextFactory(keyProvider, trustStoreFile, updatePeriod, configserverIdentity, ztsClient, zoneConfig); + this.sslContextFactory = initializeSslContextFactory(keyProvider, trustStoreFile, updatePeriod, configserverIdentity, ztsClient, athenzProviderServiceConfig); scheduler.scheduleAtFixedRate(new KeystoreUpdater(sslContextFactory), updatePeriod.toDays()/*initial delay*/, updatePeriod.toDays(), @@ -108,7 +106,7 @@ public class ConfigserverSslContextFactoryProvider extends AbstractComponent imp Duration updatePeriod, AthenzService configserverIdentity, ZtsClient ztsClient, - AthenzProviderServiceConfig.Zones zoneConfig) { + AthenzProviderServiceConfig zoneConfig) { // TODO Use DefaultTlsContext to configure SslContextFactory (ensure that cipher/protocol configuration is same across all TLS endpoints) @@ -150,7 +148,7 @@ public class ConfigserverSslContextFactoryProvider extends AbstractComponent imp char[] keystorePwd, KeyProvider keyProvider, ZtsClient ztsClient, - AthenzProviderServiceConfig.Zones zoneConfig) { + AthenzProviderServiceConfig zoneConfig) { PrivateKey privateKey = keyProvider.getPrivateKey(zoneConfig.secretVersion()); PublicKey publicKey = KeyUtils.extractPublicKey(privateKey); Identity serviceIdentity = ztsClient.getServiceIdentity(configserverIdentity, @@ -184,7 +182,7 @@ public class ConfigserverSslContextFactoryProvider extends AbstractComponent imp try { log.log(LogLevel.INFO, "Updating configserver provider certificate from ZTS"); char[] keystorePwd = generateKeystorePassword(); - KeyStore keyStore = updateKeystore(configserverIdentity, keystorePwd, keyProvider, ztsClient, zoneConfig); + KeyStore keyStore = updateKeystore(configserverIdentity, keystorePwd, keyProvider, ztsClient, athenzProviderServiceConfig); sslContextFactory.reload(scf -> { scf.setKeyStore(keyStore); scf.setKeyStorePassword(new String(keystorePwd)); diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentGenerator.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentGenerator.java index 8d3e37e1ebd..c328b8b6c21 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentGenerator.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentGenerator.java @@ -11,7 +11,6 @@ import com.yahoo.vespa.athenz.identityprovider.api.VespaUniqueInstanceId; import com.yahoo.vespa.athenz.identityprovider.client.IdentityDocumentSigner; import com.yahoo.vespa.hosted.athenz.instanceproviderservice.KeyProvider; import com.yahoo.vespa.hosted.athenz.instanceproviderservice.config.AthenzProviderServiceConfig; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl.Utils; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Allocation; @@ -33,14 +32,14 @@ public class IdentityDocumentGenerator { private final NodeRepository nodeRepository; private final Zone zone; private final KeyProvider keyProvider; - private final AthenzProviderServiceConfig.Zones zoneConfig; + private final AthenzProviderServiceConfig athenzProviderServiceConfig; @Inject public IdentityDocumentGenerator(AthenzProviderServiceConfig config, NodeRepository nodeRepository, Zone zone, KeyProvider keyProvider) { - this.zoneConfig = Utils.getZoneConfig(config, zone); + this.athenzProviderServiceConfig = config; this.nodeRepository = nodeRepository; this.zone = zone; this.keyProvider = keyProvider; @@ -62,8 +61,8 @@ public class IdentityDocumentGenerator { Set<String> ips = new HashSet<>(node.ipAddresses()); - PrivateKey privateKey = keyProvider.getPrivateKey(zoneConfig.secretVersion()); - AthenzService providerService = new AthenzService(zoneConfig.domain(), zoneConfig.serviceName()); + PrivateKey privateKey = keyProvider.getPrivateKey(athenzProviderServiceConfig.secretVersion()); + AthenzService providerService = new AthenzService(athenzProviderServiceConfig.domain(), athenzProviderServiceConfig.serviceName()); String configServerHostname = HostName.getLocalhost(); Instant createdAt = Instant.now(); diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/impl/CkmsKeyProvider.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/impl/CkmsKeyProvider.java index 40003d4ccf3..bc044f12b15 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/impl/CkmsKeyProvider.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/impl/CkmsKeyProvider.java @@ -14,8 +14,6 @@ import java.security.PublicKey; import java.util.HashMap; import java.util.Map; -import static com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl.Utils.getZoneConfig; - /** * @author mortent * @author bjorncs @@ -32,7 +30,7 @@ public class CkmsKeyProvider implements KeyProvider { Zone zone, AthenzProviderServiceConfig config) { this.secretStore = secretStore; - this.secretName = getZoneConfig(config, zone).secretName(); + this.secretName = config.secretName(); this.secrets = new HashMap<>(); } diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/impl/Utils.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/impl/Utils.java index ad54aa341bf..f52493375f1 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/impl/Utils.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/impl/Utils.java @@ -3,8 +3,6 @@ package com.yahoo.vespa.hosted.athenz.instanceproviderservice.impl; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; -import com.yahoo.config.provision.Zone; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.config.AthenzProviderServiceConfig; /** * @author bjorncs @@ -23,9 +21,4 @@ public class Utils { return mapper; } - public static AthenzProviderServiceConfig.Zones getZoneConfig(AthenzProviderServiceConfig config, Zone zone) { - String key = zone.environment().value() + "." + zone.region().value(); - return config.zones(key); - } - } diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/TestUtils.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/TestUtils.java index 9271fa74363..4a97ea7b09c 100644 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/TestUtils.java +++ b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/TestUtils.java @@ -1,8 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.athenz.instanceproviderservice; -import com.google.common.collect.ImmutableMap; -import com.yahoo.config.provision.Zone; import com.yahoo.vespa.hosted.athenz.instanceproviderservice.config.AthenzProviderServiceConfig; /** @@ -12,10 +10,9 @@ public class TestUtils { public static AthenzProviderServiceConfig getAthenzProviderConfig(String domain, String service, - String dnsSuffix, - Zone zone) { - AthenzProviderServiceConfig.Zones.Builder zoneConfig = - new AthenzProviderServiceConfig.Zones.Builder() + String dnsSuffix) { + AthenzProviderServiceConfig.Builder zoneConfig = + new AthenzProviderServiceConfig.Builder() .serviceName(service) .secretVersion(0) .domain(domain) @@ -23,9 +20,7 @@ public class TestUtils { .ztsUrl("localhost/zts") .secretName("s3cr3t"); return new AthenzProviderServiceConfig( - new AthenzProviderServiceConfig.Builder() - .zones(ImmutableMap.of(zone.environment().value() + "." + zone.region().value(), zoneConfig)) - .athenzCaTrustStore("/dummy/path/to/athenz-ca.jks")); + zoneConfig.athenzCaTrustStore("/dummy/path/to/athenz-ca.jks")); } } diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentGeneratorTest.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentGeneratorTest.java index 0688981a1c7..f496b177bdd 100644 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentGeneratorTest.java +++ b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentGeneratorTest.java @@ -78,7 +78,7 @@ public class IdentityDocumentGeneratorTest { AutoGeneratedKeyProvider keyProvider = new AutoGeneratedKeyProvider(); String dnsSuffix = "vespa.dns.suffix"; - AthenzProviderServiceConfig config = getAthenzProviderConfig("domain", "service", dnsSuffix, ZONE); + AthenzProviderServiceConfig config = getAthenzProviderConfig("domain", "service", dnsSuffix); IdentityDocumentGenerator identityDocumentGenerator = new IdentityDocumentGenerator(config, nodeRepository, ZONE, keyProvider); SignedIdentityDocument signedIdentityDocument = identityDocumentGenerator.generateSignedIdentityDocument(containerHostname, IdentityType.TENANT); diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/Endpoint.java b/config-model-api/src/main/java/com/yahoo/config/application/api/Endpoint.java index 73328027540..7d31e07dac5 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/Endpoint.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/Endpoint.java @@ -5,6 +5,7 @@ import com.yahoo.config.provision.RegionName; import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.regex.Pattern; import java.util.stream.Collectors; /** @@ -13,19 +14,37 @@ import java.util.stream.Collectors; * should point to. * * If the endpointId is not set, it will default to the same as the containerId. + * + * @author ogronnesby */ public class Endpoint { + + /* + * Endpoint IDs must be: + * - lowercase + * - alphanumeric + * - begin with a character + * - contain zero consecutive dashes + * - have a length between 1 and 12 + */ + private static final Pattern endpointPattern = Pattern.compile("^[a-z](?:-?[a-z0-9]+)*$"); + private static final int endpointMaxLength = 12; + private final Optional<String> endpointId; private final String containerId; private final Set<RegionName> regions; public Endpoint(Optional<String> endpointId, String containerId, Set<String> regions) { - this.endpointId = endpointId; - this.containerId = containerId; + this.endpointId = Objects.requireNonNull(endpointId, "endpointId must be non-null"); + this.containerId = Objects.requireNonNull(containerId, "containerId must be non-null"); this.regions = Set.copyOf( Objects.requireNonNull( regions.stream().map(RegionName::from).collect(Collectors.toList()), "Missing 'regions' parameter")); + + if (endpointId().length() > endpointMaxLength || !endpointPattern.matcher(endpointId()).matches()) { + throw new IllegalArgumentException("Invalid endpoint ID: '" + endpointId() + "'"); + } } public String endpointId() { @@ -54,4 +73,5 @@ public class Endpoint { public int hashCode() { return Objects.hash(endpointId, containerId, regions); } + } diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java b/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java index 58795f6ea9e..650f68591b6 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java @@ -183,7 +183,11 @@ public class DeploymentSpecXmlReader { regions.add(region); } - endpoints.add(new Endpoint(rotationId, containerId.get(), regions)); + var endpoint = new Endpoint(rotationId, containerId.get(), regions); + if (endpoints.contains(endpoint)) { + throw new IllegalArgumentException("Duplicate 'endpoint' in 'endpoints' tag"); + } + endpoints.add(endpoint); } return endpoints; diff --git a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java index 3049d511bf1..c161b04087f 100644 --- a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java +++ b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java @@ -493,4 +493,49 @@ public class DeploymentSpecTest { assertEquals(Set.of(RegionName.from("us-east")), spec.endpoints().get(0).regions()); } + @Test + public void invalidEndpoints() { + assertInvalid("<endpoint id='FOO' container-id='qrs'/>"); // Uppercase + assertInvalid("<endpoint id='123' container-id='qrs'/>"); // Starting with non-character + assertInvalid("<endpoint id='foo!' container-id='qrs'/>"); // Non-alphanumeric + assertInvalid("<endpoint id='foo.bar' container-id='qrs'/>"); + assertInvalid("<endpoint id='foo--bar' container-id='qrs'/>"); // Multiple consecutive dashes + assertInvalid("<endpoint id='foo-' container-id='qrs'/>"); // Trailing dash + assertInvalid("<endpoint id='foooooooooooo' container-id='qrs'/>"); // Too long + assertInvalid("<endpoint id='foo' container-id='qrs'/><endpoint id='foo' container-id='qrs'/>"); // Duplicate + } + + @Test + public void validEndpoints() { + assertEquals(List.of("qrs"), endpointIds("<endpoint container-id='qrs'/>")); + assertEquals(List.of("qrs"), endpointIds("<endpoint id='' container-id='qrs'/>")); + assertEquals(List.of("f"), endpointIds("<endpoint id='f' container-id='qrs'/>")); + assertEquals(List.of("foo"), endpointIds("<endpoint id='foo' container-id='qrs'/>")); + assertEquals(List.of("foo-bar"), endpointIds("<endpoint id='foo-bar' container-id='qrs'/>")); + assertEquals(List.of("foo", "bar"), endpointIds("<endpoint id='foo' container-id='qrs'/><endpoint id='bar' container-id='qrs'/>")); + assertEquals(List.of("fooooooooooo"), endpointIds("<endpoint id='fooooooooooo' container-id='qrs'/>")); + } + + private static void assertInvalid(String endpointTag) { + try { + endpointIds(endpointTag); + fail("Expected exception for input '" + endpointTag + "'"); + } catch (IllegalArgumentException ignored) {} + } + + private static List<String> endpointIds(String endpointTag) { + var xml = "<deployment>" + + " <prod>" + + " <region active=\"true\">us-east</region>" + + " </prod>" + + " <endpoints>" + + endpointTag + + " </endpoints>" + + "</deployment>"; + + return DeploymentSpec.fromXml(xml).endpoints().stream() + .map(Endpoint::endpointId) + .collect(Collectors.toList()); + } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java index 44c10b0738b..03c8055dd12 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java @@ -221,7 +221,9 @@ public class Admin extends AbstractConfigProducer implements Serializable { metricsProxyCluster = new MetricsProxyContainerCluster(this, "metrics", deployState); int index = 0; for (var host : hosts) { - var container = new MetricsProxyContainer(metricsProxyCluster, index++, deployState.isHosted()); + // Send hostname to be used in configId (instead of index), as the sorting of hosts seems to be unstable + // between config changes, even when the set of hosts is unchanged. + var container = new MetricsProxyContainer(metricsProxyCluster, host.getHostname(), index, deployState.isHosted()); addAndInitializeService(deployState.getDeployLogger(), host, container); metricsProxyCluster.addContainer(container); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java index e683b70bbde..3bc38cad1d1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java @@ -45,8 +45,8 @@ public class MetricsProxyContainer extends Container implements private final boolean isHostedVespa; - public MetricsProxyContainer(AbstractConfigProducer parent, int index, boolean isHostedVespa) { - super(parent, "metricsproxy." + index, index); + public MetricsProxyContainer(AbstractConfigProducer parent, String hostname, int index, boolean isHostedVespa) { + super(parent, hostname, index); this.isHostedVespa = isHostedVespa; setProp("clustertype", "admin"); setProp("index", String.valueOf(index)); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java index fcc8cc8fa41..aa793b3c6a2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java @@ -5,7 +5,6 @@ import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.ConfigModelContext; import com.yahoo.config.model.api.ConfigServerSpec; import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.log.LogLevel; import com.yahoo.vespa.model.HostResource; @@ -22,7 +21,6 @@ import org.w3c.dom.Element; import java.util.ArrayList; import java.util.Collection; import java.util.List; -import java.util.Objects; import java.util.Optional; import java.util.stream.Collectors; @@ -33,8 +31,6 @@ import java.util.stream.Collectors; */ public class DomAdminV4Builder extends DomAdminBuilderBase { - private ApplicationId ZONE_APPLICATION_ID = ApplicationId.from("hosted-vespa", "routing", "default"); - private final Collection<ContainerModel> containerModels; private final ConfigModelContext context; @@ -134,34 +130,17 @@ public class DomAdminV4Builder extends DomAdminBuilderBase { * @param minHostsPerContainerCluster the desired number of hosts per cluster */ private List<HostResource> pickContainerHostsForSlobrok(int count, int minHostsPerContainerCluster) { - Collection<ContainerModel> containerModelsWithSlobrok = containerModels.stream() - .filter(this::shouldHaveSlobrok) - .collect(Collectors.toList()); int hostsPerCluster = (int) Math.max(minHostsPerContainerCluster, - Math.ceil((double) count / containerModelsWithSlobrok.size())); + Math.ceil((double) count / containerModels.size())); // Pick from all container clusters to make sure we don't lose all nodes at once if some clusters are removed. // This will overshoot the desired size (due to ceil and picking at least one node per cluster). List<HostResource> picked = new ArrayList<>(); - for (ContainerModel containerModel : containerModelsWithSlobrok) + for (ContainerModel containerModel : containerModels) picked.addAll(pickContainerHostsFrom(containerModel, hostsPerCluster)); return picked; } - private boolean shouldHaveSlobrok(ContainerModel containerModel) { - // Avoid Slobroks on node-admin container cluster, as node-admin is migrating - // TODO: Remove after removing tenant hosts from zone-app - - ApplicationId applicationId = context.getDeployState().getProperties().applicationId(); - if (!applicationId.equals(ZONE_APPLICATION_ID)) { - return true; - } - - // aka clustername, aka application-model's ClusterId - String clustername = containerModel.getCluster().getName(); - return !Objects.equals(clustername, "node-admin"); - } - private List<HostResource> pickContainerHostsFrom(ContainerModel model, int count) { boolean retired = true; List<HostResource> picked = sortedContainerHostsFrom(model, count, !retired); diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java index af31a09101e..82841b52984 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java @@ -16,7 +16,6 @@ import com.yahoo.container.core.ApplicationMetadataConfig; import com.yahoo.search.config.QrStartConfig; import com.yahoo.searchdefinition.parser.ParseException; import com.yahoo.vespa.config.search.core.ProtonConfig; -import com.yahoo.vespa.model.AbstractService; import com.yahoo.vespa.model.HostResource; import com.yahoo.vespa.model.HostSystem; import com.yahoo.vespa.model.VespaModel; @@ -51,9 +50,6 @@ import java.util.stream.Collectors; import static com.yahoo.config.model.test.TestUtil.joinLines; import static com.yahoo.vespa.defaults.Defaults.getDefaults; import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.collection.IsIn.isIn; -import static org.hamcrest.core.Every.everyItem; -import static org.hamcrest.core.IsNot.not; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -762,42 +758,6 @@ public class ModelProvisioningTest { assertEquals("Included in addition because it is retired", "default03", model.getAdmin().getSlobroks().get(5).getHostName()); } - @Test - public void testSlobroksAreSpreadOverAllContainerClustersExceptNodeAdmin() { - String services = - "<?xml version='1.0' encoding='utf-8' ?>\n" + - "<services>" + - " <admin version='4.0'/>" + - " <container version='1.0' id='routing'>" + - " <nodes count='10'/>" + - " </container>" + - " <container version='1.0' id='node-admin'>" + - " <nodes count='3'/>" + - " </container>" + - "</services>"; - - int numberOfHosts = 13; - VespaModelTester tester = new VespaModelTester(); - tester.addHosts(numberOfHosts); - tester.setApplicationId("hosted-vespa", "routing", "default"); - VespaModel model = tester.createModel(services, true); - assertThat(model.getRoot().getHostSystem().getHosts().size(), is(numberOfHosts)); - - Set<String> routingHosts = getClusterHostnames(model, "routing"); - assertEquals(10, routingHosts.size()); - - Set<String> nodeAdminHosts = getClusterHostnames(model, "node-admin"); - assertEquals(3, nodeAdminHosts.size()); - - Set<String> slobrokHosts = model.getAdmin().getSlobroks().stream() - .map(AbstractService::getHostName) - .collect(Collectors.toSet()); - assertEquals(3, slobrokHosts.size()); - - assertThat(slobrokHosts, everyItem(isIn(routingHosts))); - assertThat(slobrokHosts, everyItem(not(isIn(nodeAdminHosts)))); - } - private Set<String> getClusterHostnames(VespaModel model, String clusterId) { return model.getHosts().stream() .filter(host -> host.getServices().stream() diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorTestCase.java index 80440ac8eb4..1b03825eef1 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorTestCase.java @@ -2,9 +2,10 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.ExpectedException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; /** * @author geirst @@ -138,23 +139,29 @@ public class RankingExpressionWithTensorTestCase { f.assertRankProperty("tensor(x{})", "constant(my_tensor).type", "my_profile"); } - @Rule - public ExpectedException exception = ExpectedException.none(); - @Test public void requireThatInvalidTensorTypeSpecThrowsException() throws ParseException { - exception.expect(IllegalArgumentException.class); - exception.expectMessage("For constant tensor 'my_tensor' in rank profile 'my_profile': Illegal tensor type spec: A tensor type spec must be on the form tensor[<valuetype>]?(dimensionidentifier[{}|[length?]*), but was 'tensor(x)'. Dimension 'x' is on the wrong format. Examples: tensor(x[]), tensor<float>(name{}, x[10])"); - RankProfileSearchFixture f = new RankProfileSearchFixture( - " rank-profile my_profile {\n" + - " constants {\n" + - " my_tensor {\n" + - " value: { {x:1}:1 }\n" + - " type: tensor(x)\n" + - " }\n" + - " }\n" + - " }"); - f.compileRankProfile("my_profile"); + try { + RankProfileSearchFixture f = new RankProfileSearchFixture( + " rank-profile my_profile {\n" + + " constants {\n" + + " my_tensor {\n" + + " value: { {x:1}:1 }\n" + + " type: tensor(x)\n" + + " }\n" + + " }\n" + + " }"); + f.compileRankProfile("my_profile"); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertStartsWith("For constant tensor 'my_tensor' in rank profile 'my_profile': Illegal tensor type spec", + e.getMessage()); + } + } + + private void assertStartsWith(String prefix, String string) { + assertEquals(prefix, string.substring(0, Math.min(prefix.length(), string.length()))); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorFieldTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorFieldTestCase.java index f53ca15635f..b6569357495 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorFieldTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorFieldTestCase.java @@ -3,48 +3,68 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.searchdefinition.SearchBuilder; import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.ExpectedException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; /** * @author geirst */ public class TensorFieldTestCase { - @Rule - public ExpectedException exception = ExpectedException.none(); - @Test public void requireThatTensorFieldCannotBeOfCollectionType() throws ParseException { - exception.expect(IllegalArgumentException.class); - exception.expectMessage("For search 'test', field 'f1': A field with collection type of tensor is not supported. Use simple type 'tensor' instead."); - SearchBuilder.createFromString(getSd("field f1 type array<tensor(x{})> {}")); + try { + SearchBuilder.createFromString(getSd("field f1 type array<tensor(x{})> {}")); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("For search 'test', field 'f1': A field with collection type of tensor is not supported. Use simple type 'tensor' instead.", + e.getMessage()); + } } @Test public void requireThatTensorFieldCannotBeIndexField() throws ParseException { - exception.expect(IllegalArgumentException.class); - exception.expectMessage("For search 'test', field 'f1': A field of type 'tensor' cannot be specified as an 'index' field."); - SearchBuilder.createFromString(getSd("field f1 type tensor(x{}) { indexing: index }")); + try { + SearchBuilder.createFromString(getSd("field f1 type tensor(x{}) { indexing: index }")); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("For search 'test', field 'f1': A field of type 'tensor' cannot be specified as an 'index' field.", + e.getMessage()); + } } @Test public void requireThatTensorAttributeCannotBeFastSearch() throws ParseException { - exception.expect(IllegalArgumentException.class); - exception.expectMessage("For search 'test', field 'f1': An attribute of type 'tensor' cannot be 'fast-search'."); - SearchBuilder.createFromString(getSd("field f1 type tensor(x{}) { indexing: attribute \n attribute: fast-search }")); + try { + SearchBuilder.createFromString(getSd("field f1 type tensor(x{}) { indexing: attribute \n attribute: fast-search }")); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("For search 'test', field 'f1': An attribute of type 'tensor' cannot be 'fast-search'.", e.getMessage()); + } } @Test public void requireThatIllegalTensorTypeSpecThrowsException() throws ParseException { - exception.expect(IllegalArgumentException.class); - exception.expectMessage("Field type: Illegal tensor type spec: A tensor type spec must be on the form tensor[<valuetype>]?(dimensionidentifier[{}|[length?]*), but was 'tensor(invalid)'. Dimension 'invalid' is on the wrong format. Examples: tensor(x[]), tensor<float>(name{}, x[10])"); - SearchBuilder.createFromString(getSd("field f1 type tensor(invalid) { indexing: attribute }")); + try { + SearchBuilder.createFromString(getSd("field f1 type tensor(invalid) { indexing: attribute }")); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertStartsWith("Field type: Illegal tensor type spec:", e.getMessage()); + } } private static String getSd(String field) { return "search test {\n document test {\n" + field + "}\n}\n"; } + private void assertStartsWith(String prefix, String string) { + assertEquals(prefix, string.substring(0, Math.min(prefix.length(), string.length()))); + } + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java index a10a5dcf4cc..d2bf4b601a6 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java @@ -9,6 +9,7 @@ import com.yahoo.vespa.model.test.VespaModelTester; import org.junit.Test; import static com.yahoo.config.model.api.container.ContainerServiceType.METRICS_PROXY_CONTAINER; +import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.CLUSTER_CONFIG_ID; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.CONTAINER_CONFIG_ID; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.MY_FLAVOR; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getHostedModel; @@ -101,7 +102,9 @@ public class MetricsProxyContainerTest { public void hosted_application_propagates_node_dimensions() { String services = servicesWithContent(); VespaModel hostedModel = getHostedModel(services); - NodeDimensionsConfig config = getNodeDimensionsConfig(hostedModel); + assertEquals(1, hostedModel.getHosts().size()); + String configId = CLUSTER_CONFIG_ID + "/" + hostedModel.getHosts().iterator().next().getHostname(); + NodeDimensionsConfig config = getNodeDimensionsConfig(hostedModel, configId); assertEquals("content", config.dimensions(NodeDimensionNames.CLUSTER_TYPE)); assertEquals("my-content", config.dimensions(NodeDimensionNames.CLUSTER_ID)); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java index 81b06e54585..13589c763e2 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java @@ -32,7 +32,7 @@ class MetricsProxyModelTester { static final String CLUSTER_CONFIG_ID = "admin/metrics"; // Used for all configs that are produced by the container, not the cluster. - static final String CONTAINER_CONFIG_ID = CLUSTER_CONFIG_ID + "/metricsproxy.0"; + static final String CONTAINER_CONFIG_ID = CLUSTER_CONFIG_ID + "/localhost"; static VespaModel getModel(String servicesXml) { var numberOfHosts = 1; @@ -87,8 +87,8 @@ class MetricsProxyModelTester { return new QrStartConfig((QrStartConfig.Builder) model.getConfig(new QrStartConfig.Builder(), CLUSTER_CONFIG_ID)); } - static NodeDimensionsConfig getNodeDimensionsConfig(VespaModel model) { - return new NodeDimensionsConfig((NodeDimensionsConfig.Builder) model.getConfig(new NodeDimensionsConfig.Builder(), CONTAINER_CONFIG_ID)); + static NodeDimensionsConfig getNodeDimensionsConfig(VespaModel model, String configId) { + return new NodeDimensionsConfig((NodeDimensionsConfig.Builder) model.getConfig(new NodeDimensionsConfig.Builder(), configId)); } static VespaServicesConfig getVespaServicesConfig(String servicesXml) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidatorTest.java index 3fbfbf33fb3..6d9eabf326b 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidatorTest.java @@ -61,7 +61,7 @@ public class ConfigValueChangeValidatorTest { assertEquals(3, changes.size()); assertComponentsEquals(changes, "default/container.0", 0); assertComponentsEquals(changes, "admin/cluster-controllers/0", 1); - assertComponentsEquals(changes, "admin/metrics/metricsproxy.0", 2); + assertComponentsEquals(changes, "admin/metrics/localhost", 2); } @Test diff --git a/config-provisioning/abi-spec.json b/config-provisioning/abi-spec.json index e88947b3fdb..aa9b196c0e4 100644 --- a/config-provisioning/abi-spec.json +++ b/config-provisioning/abi-spec.json @@ -260,7 +260,6 @@ "public com.yahoo.component.Version vespaVersion()", "public java.util.Optional group()", "public boolean isExclusive()", - "public java.util.Set rotations()", "public com.yahoo.config.provision.ClusterSpec with(java.util.Optional)", "public com.yahoo.config.provision.ClusterSpec exclusive(boolean)", "public static com.yahoo.config.provision.ClusterSpec request(com.yahoo.config.provision.ClusterSpec$Type, com.yahoo.config.provision.ClusterSpec$Id, com.yahoo.component.Version, boolean)", diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterMembership.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterMembership.java index c0099878b45..f041823bf04 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterMembership.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterMembership.java @@ -3,14 +3,9 @@ package com.yahoo.config.provision; import com.yahoo.component.Version; -import java.util.Arrays; -import java.util.Collections; -import java.util.Set; -import java.util.stream.Collectors; - /** * A node's membership in a cluster. This is a value object. - * The format is "clusterType/clusterId/groupId/index[/exclusive][/retired][/rotationId,...]" + * The format is "clusterType/clusterId/groupId/index[/exclusive][/retired]" * * @author bratseth */ @@ -25,26 +20,23 @@ public class ClusterMembership { private ClusterMembership(String stringValue, Version vespaVersion) { String[] components = stringValue.split("/"); - if (components.length < 4 || components.length > 7) + if (components.length < 4) throw new RuntimeException("Could not parse '" + stringValue + "' to a cluster membership. " + - "Expected 'clusterType/clusterId/groupId/index[/retired][/exclusive][/rotationId,...]'"); + "Expected 'clusterType/clusterId/groupId/index[/retired][/exclusive]'"); boolean exclusive = false; - Set<RotationName> rotations = Collections.emptySet(); if (components.length > 4) { for (int i = 4; i < components.length; i++) { String component = components[i]; switch (component) { case "exclusive": exclusive = true; break; case "retired": retired = true; break; - default: rotations = rotationsFrom(component); break; } } } this.cluster = ClusterSpec.from(ClusterSpec.Type.valueOf(components[0]), ClusterSpec.Id.from(components[1]), - ClusterSpec.Group.from(Integer.valueOf(components[2])), vespaVersion, exclusive, - rotations); + ClusterSpec.Group.from(Integer.valueOf(components[2])), vespaVersion, exclusive); this.index = Integer.parseInt(components[3]); this.stringValue = toStringValue(); } @@ -62,8 +54,7 @@ public class ClusterMembership { (cluster.group().isPresent() ? "/" + cluster.group().get().index() : "") + "/" + index + ( cluster.isExclusive() ? "/exclusive" : "") + - ( retired ? "/retired" : "") + - ( !cluster.rotations().isEmpty() ? "/" + rotationsAsString(cluster.rotations()) : ""); + ( retired ? "/retired" : ""); } @@ -121,12 +112,4 @@ public class ClusterMembership { return new ClusterMembership(cluster, index, true); } - private static Set<RotationName> rotationsFrom(String s) { - return Arrays.stream(s.split(",")).map(RotationName::from).collect(Collectors.toUnmodifiableSet()); - } - - private static String rotationsAsString(Set<RotationName> rotations) { - return rotations.stream().map(RotationName::value).collect(Collectors.joining(",")); - } - } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java index 35ee538178a..8ed56b98705 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.provision; -import com.google.common.collect.ImmutableSortedSet; import com.yahoo.component.Version; import java.util.Objects; @@ -23,19 +22,13 @@ public final class ClusterSpec { private final Optional<Group> groupId; private final Version vespaVersion; private boolean exclusive; - private final Set<RotationName> rotations; - private ClusterSpec(Type type, Id id, Optional<Group> groupId, Version vespaVersion, boolean exclusive, - Set<RotationName> rotations) { - if (type != Type.container && !rotations.isEmpty()) { - throw new IllegalArgumentException("Rotations can only be declared for clusters of type " + Type.container); - } + private ClusterSpec(Type type, Id id, Optional<Group> groupId, Version vespaVersion, boolean exclusive) { this.type = type; this.id = id; this.groupId = groupId; this.vespaVersion = vespaVersion; this.exclusive = exclusive; - this.rotations = ImmutableSortedSet.copyOf(rotations); } /** Returns the cluster type */ @@ -57,35 +50,30 @@ public final class ClusterSpec { */ public boolean isExclusive() { return exclusive; } - /** Returns the rotations of which this cluster should be a member */ - public Set<RotationName> rotations() { - return rotations; - } - public ClusterSpec with(Optional<Group> newGroup) { - return new ClusterSpec(type, id, newGroup, vespaVersion, exclusive, rotations); + return new ClusterSpec(type, id, newGroup, vespaVersion, exclusive); } public ClusterSpec exclusive(boolean exclusive) { - return new ClusterSpec(type, id, groupId, vespaVersion, exclusive, rotations); + return new ClusterSpec(type, id, groupId, vespaVersion, exclusive); } public static ClusterSpec request(Type type, Id id, Version vespaVersion, boolean exclusive) { - return new ClusterSpec(type, id, Optional.empty(), vespaVersion, exclusive, Set.of()); + return new ClusterSpec(type, id, Optional.empty(), vespaVersion, exclusive); } // TODO: Remove after June 2019 public static ClusterSpec request(Type type, Id id, Version vespaVersion, boolean exclusive, Set<RotationName> rotations) { - return new ClusterSpec(type, id, Optional.empty(), vespaVersion, exclusive, rotations); + return new ClusterSpec(type, id, Optional.empty(), vespaVersion, exclusive); } public static ClusterSpec from(Type type, Id id, Group groupId, Version vespaVersion, boolean exclusive) { - return new ClusterSpec(type, id, Optional.of(groupId), vespaVersion, exclusive, Set.of()); + return new ClusterSpec(type, id, Optional.of(groupId), vespaVersion, exclusive); } // TODO: Remove after June 2019 public static ClusterSpec from(Type type, Id id, Group groupId, Version vespaVersion, boolean exclusive, Set<RotationName> rotations) { - return new ClusterSpec(type, id, Optional.of(groupId), vespaVersion, exclusive, rotations); + return new ClusterSpec(type, id, Optional.of(groupId), vespaVersion, exclusive); } @Override diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/UpgradePolicy.java b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/UpgradePolicy.java index d5831efdbaa..1baaac772c8 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/UpgradePolicy.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/UpgradePolicy.java @@ -13,29 +13,29 @@ import java.util.List; */ public class UpgradePolicy { - private final List<List<ZoneId>> zones; + private final List<List<ZoneApi>> zones; - private UpgradePolicy(List<List<ZoneId>> zones) { + private UpgradePolicy(List<List<ZoneApi>> zones) { this.zones = zones; } - public List<List<ZoneId>> asList() { - return Collections.unmodifiableList(zones); + public List<List<ZoneApi>> asList() { + return List.copyOf(zones); } - private UpgradePolicy with(ZoneId... zone) { - List<List<ZoneId>> zones = new ArrayList<>(this.zones); + private UpgradePolicy with(ZoneApi... zone) { + List<List<ZoneApi>> zones = new ArrayList<>(this.zones); zones.add(Arrays.asList(zone)); return new UpgradePolicy(zones); } /** Upgrade given zone as the next step */ - public UpgradePolicy upgrade(ZoneId zone) { + public UpgradePolicy upgrade(ZoneApi zone) { return with(zone); } /** Upgrade given zones in parallel as the next step */ - public UpgradePolicy upgradeInParallel(ZoneId... zone) { + public UpgradePolicy upgradeInParallel(ZoneApi... zone) { return with(zone); } diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/ClusterMembershipTest.java b/config-provisioning/src/test/java/com/yahoo/config/provision/ClusterMembershipTest.java index 9bd0680b691..5eee55a1886 100644 --- a/config-provisioning/src/test/java/com/yahoo/config/provision/ClusterMembershipTest.java +++ b/config-provisioning/src/test/java/com/yahoo/config/provision/ClusterMembershipTest.java @@ -6,7 +6,6 @@ import com.yahoo.component.Vtag; import org.junit.Test; import java.util.Collections; -import java.util.Set; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -37,26 +36,25 @@ public class ClusterMembershipTest { assertTrue(instance.cluster().isExclusive()); } + // TODO: Remove after June 2019. This ensures stale rotation data is handled { ClusterMembership instance = ClusterMembership.from("container/id1/4/37/rotation1,rotation2", Vtag.currentVersion); assertFalse(instance.retired()); assertFalse(instance.cluster().isExclusive()); - assertEquals(Set.of(RotationName.from("rotation1"), RotationName.from("rotation2")), instance.cluster().rotations()); } { ClusterMembership instance = ClusterMembership.from("container/id1/4/37/exclusive/rotation1,rotation2", Vtag.currentVersion); assertFalse(instance.retired()); assertTrue(instance.cluster().isExclusive()); - assertEquals(Set.of(RotationName.from("rotation1"), RotationName.from("rotation2")), instance.cluster().rotations()); } { ClusterMembership instance = ClusterMembership.from("container/id1/4/37/exclusive/retired/rotation1,rotation2", Vtag.currentVersion); assertTrue(instance.retired()); assertTrue(instance.cluster().isExclusive()); - assertEquals(Set.of(RotationName.from("rotation1"), RotationName.from("rotation2")), instance.cluster().rotations()); } + // end TODO } @Test @@ -101,7 +99,6 @@ public class ClusterMembershipTest { assertFalse(instance.cluster().group().isPresent()); assertEquals(3, instance.index()); assertEquals("container/id1/3", instance.stringValue()); - assertTrue(instance.cluster().rotations().isEmpty()); } private void assertContentService(ClusterMembership instance) { @@ -111,7 +108,6 @@ public class ClusterMembershipTest { assertEquals(37, instance.index()); assertFalse(instance.retired()); assertEquals("content/id1/37", instance.stringValue()); - assertTrue(instance.cluster().rotations().isEmpty()); } private void assertContentServiceWithGroup(ClusterMembership instance) { @@ -121,7 +117,6 @@ public class ClusterMembershipTest { assertEquals(37, instance.index()); assertFalse(instance.retired()); assertEquals("content/id1/4/37", instance.stringValue()); - assertTrue(instance.cluster().rotations().isEmpty()); } /** Serializing a spec without a group assigned works, but not deserialization */ @@ -131,7 +126,6 @@ public class ClusterMembershipTest { assertEquals(37, instance.index()); assertTrue(instance.retired()); assertEquals("content/id1/37/retired", instance.stringValue()); - assertTrue(instance.cluster().rotations().isEmpty()); } private void assertContentServiceWithGroupAndRetire(ClusterMembership instance) { @@ -141,7 +135,6 @@ public class ClusterMembershipTest { assertEquals(37, instance.index()); assertTrue(instance.retired()); assertEquals("content/id1/4/37/retired", instance.stringValue()); - assertTrue(instance.cluster().rotations().isEmpty()); } } diff --git a/configdefinitions/src/vespa/athenz-provider-service.def b/configdefinitions/src/vespa/athenz-provider-service.def index 281db6fb43d..7a06b13d435 100644 --- a/configdefinitions/src/vespa/athenz-provider-service.def +++ b/configdefinitions/src/vespa/athenz-provider-service.def @@ -2,22 +2,22 @@ namespace=vespa.hosted.athenz.instanceproviderservice.config # Athenz domain -zones{}.domain string +domain string # Athenz service name -zones{}.serviceName string +serviceName string # Secret name of private Key -zones{}.secretName string +secretName string # Secret version -zones{}.secretVersion int +secretVersion int # Certificate DNS suffix -zones{}.certDnsSuffix string +certDnsSuffix string # Athenz ZTS server url -zones{}.ztsUrl string +ztsUrl string # Path to Athenz CA JKS trust store athenzCaTrustStore string diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigConvergenceChecker.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigConvergenceChecker.java index 74a1eb6391b..4d0df545c39 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigConvergenceChecker.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigConvergenceChecker.java @@ -7,7 +7,6 @@ import com.yahoo.component.AbstractComponent; import com.yahoo.config.model.api.HostInfo; import com.yahoo.config.model.api.PortInfo; import com.yahoo.config.model.api.ServiceInfo; -import com.yahoo.config.provision.ApplicationId; import com.yahoo.log.LogLevel; import com.yahoo.slime.Cursor; import com.yahoo.vespa.config.server.http.JSONResponse; @@ -23,13 +22,12 @@ import javax.ws.rs.client.WebTarget; import java.net.URI; import java.time.Duration; import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.logging.Logger; import java.util.stream.Collectors; import static com.yahoo.config.model.api.container.ContainerServiceType.CONTAINER; @@ -44,18 +42,17 @@ import static com.yahoo.config.model.api.container.ContainerServiceType.QRSERVER */ public class ConfigConvergenceChecker extends AbstractComponent { - private static final java.util.logging.Logger log = java.util.logging.Logger.getLogger(ConfigConvergenceChecker.class.getName()); - private static final ApplicationId routingApplicationId = ApplicationId.from("hosted-vespa", "routing", "default"); + private static final Logger log = Logger.getLogger(ConfigConvergenceChecker.class.getName()); private static final String statePath = "/state/v1/"; private static final String configSubPath = "config"; - private final static Set<String> serviceTypesToCheck = new HashSet<>(Arrays.asList( + private final static Set<String> serviceTypesToCheck = Set.of( CONTAINER.serviceName, QRSERVER.serviceName, LOGSERVER_CONTAINER.serviceName, "searchnode", "storagenode", "distributor" - )); + ); private final StateApiFactory stateApiFactory; @@ -75,9 +72,6 @@ public class ConfigConvergenceChecker extends AbstractComponent { application.getModel().getHosts() .forEach(host -> host.getServices().stream() .filter(service -> serviceTypesToCheck.contains(service.getServiceType())) - - // TODO: Remove after removing tenant hosts from zone-app - .filter(service -> ! isHostAdminService(application.getId(), service)) .forEach(service -> getStatePort(service).ifPresent(port -> servicesToCheck.add(service)))); Map<ServiceInfo, Long> currentGenerations = getServiceGenerations(servicesToCheck, timeoutPerService); @@ -181,13 +175,6 @@ public class ConfigConvergenceChecker extends AbstractComponent { return WebResourceFactory.newResource(StateApi.class, target); } - private static boolean isHostAdminService(ApplicationId id, ServiceInfo service) { - return routingApplicationId.equals(id) - && service.getProperty("clustername") - .map("node-admin"::equals) - .orElse(false); - } - private static class ServiceListResponse extends JSONResponse { // Pre-condition: servicesToCheck has a state port diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpoint.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpoint.java new file mode 100644 index 00000000000..fb1035b5a03 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpoint.java @@ -0,0 +1,54 @@ +// Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.tenant; + +import com.yahoo.vespa.applicationmodel.ClusterId; + +import java.util.List; +import java.util.Objects; + +/** + * ContainerEndpoint tracks the service names that a Container Cluster should be + * known as. This is used during request routing both for regular requests and + * for health checks in traffic distribution. + * + * @author ogronnesby + */ +public class ContainerEndpoint { + private final ClusterId clusterId; + private final List<String> names; + + ContainerEndpoint(ClusterId clusterId, List<String> names) { + this.clusterId = Objects.requireNonNull(clusterId); + this.names = List.copyOf(Objects.requireNonNull(names)); + } + + public ClusterId clusterId() { + return clusterId; + } + + public List<String> names() { + return names; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ContainerEndpoint that = (ContainerEndpoint) o; + return Objects.equals(clusterId, that.clusterId) && + Objects.equals(names, that.names); + } + + @Override + public int hashCode() { + return Objects.hash(clusterId, names); + } + + @Override + public String toString() { + return "ContainerEndpoint{" + + "clusterId=" + clusterId + + ", names=" + names + + '}'; + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointSerializer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointSerializer.java new file mode 100644 index 00000000000..83d65f5b38b --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointSerializer.java @@ -0,0 +1,76 @@ +// Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.tenant; + +import com.yahoo.slime.ArrayTraverser; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.applicationmodel.ClusterId; + +import java.util.ArrayList; +import java.util.List; + +/** + * Contains all methods for de-/serializing ContainerEndpoints to/from JSON. + * Also supports de-/serializing lists of these values. + * + * @author ogronnesby + */ +public class ContainerEndpointSerializer { + private static final String clusterIdField = "clusterId"; + private static final String namesField = "names"; + + public static ContainerEndpoint endpointFromSlime(Inspector inspector) { + final var clusterId = inspector.field(clusterIdField).asString(); + final var namesInspector = inspector.field(namesField); + + if (clusterId.isEmpty()) { + throw new IllegalStateException("'clusterId' missing on serialized ContainerEndpoint"); + } + + if (! namesInspector.valid()) { + throw new IllegalStateException("'names' missing on serialized ContainerEndpoint"); + } + + final var names = new ArrayList<String>(); + + namesInspector.traverse((ArrayTraverser) (idx, nameInspector) -> { + final var containerName = nameInspector.asString(); + names.add(containerName); + }); + + return new ContainerEndpoint(new ClusterId(clusterId), names); + } + + public static List<ContainerEndpoint> endpointListFromSlime(Slime slime) { + final var inspector = slime.get(); + final var endpoints = new ArrayList<ContainerEndpoint>(); + + inspector.traverse((ArrayTraverser) (idx, endpointInspector) -> { + final var containerEndpoint = endpointFromSlime(endpointInspector); + endpoints.add(containerEndpoint); + }); + + return endpoints; + } + + + public static void endpointToSlime(Cursor cursor, ContainerEndpoint endpoint) { + cursor.setString(clusterIdField, endpoint.clusterId().toString()); + + final var namesInspector = cursor.setArray(namesField); + endpoint.names().forEach(namesInspector::addString); + } + + public static Slime endpointListToSlime(List<ContainerEndpoint> endpoints) { + final var slime = new Slime(); + final var cursor = slime.setArray(); + + endpoints.forEach(endpoint -> { + final var endpointCursor = cursor.addObject(); + endpointToSlime(endpointCursor, endpoint); + }); + + return slime; + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointsCache.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointsCache.java new file mode 100644 index 00000000000..06f93f2006f --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointsCache.java @@ -0,0 +1,59 @@ +// Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.tenant; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.path.Path; +import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.vespa.curator.Curator; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.List; + + +/** + * Persists assignment of rotations to an application to ZooKeeper. + * The entries are {@link ContainerEndpoint} instances, which keep track of the container + * cluster that is the target, the endpoint name, and the rotation used to + * give availability to that cluster. + * + * This is v2 of that storage in a new directory. Previously we only stored + * the name of the rotation, since all the other information could be + * calculated runtime. + * + * @author ogronnesby + */ +public class ContainerEndpointsCache { + private final Path cachePath; + private final Curator curator; + + ContainerEndpointsCache(Path tenantPath, Curator curator) { + this.cachePath = tenantPath.append("containerEndpointsCache/"); + this.curator = curator; + } + + public List<ContainerEndpoint> read(ApplicationId applicationId) { + final var optionalData = curator.getData(applicationPath(applicationId)); + return optionalData + .map(SlimeUtils::jsonToSlime) + .map(ContainerEndpointSerializer::endpointListFromSlime) + .orElse(List.of()); + } + + public void write(ApplicationId applicationId, List<ContainerEndpoint> endpoints) { + if (endpoints.isEmpty()) return; + + final var slime = ContainerEndpointSerializer.endpointListToSlime(endpoints); + + try { + final var bytes = SlimeUtils.toJsonBytes(slime); + curator.set(applicationPath(applicationId), bytes); + } catch (IOException e) { + throw new UncheckedIOException("Error writing endpoints of: " + applicationId, e); + } + } + + private Path applicationPath(ApplicationId applicationId) { + return cachePath.append(applicationId.serializedForm()); + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointSerializerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointSerializerTest.java new file mode 100644 index 00000000000..b4d52e6d37c --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointSerializerTest.java @@ -0,0 +1,45 @@ +package com.yahoo.vespa.config.server.tenant; + +import com.yahoo.slime.Slime; +import com.yahoo.vespa.applicationmodel.ClusterId; +import org.junit.Test; + +import java.util.List; + +import static org.junit.Assert.assertEquals; + +public class ContainerEndpointSerializerTest { + @Test + public void readSingleEndpoint() { + final var slime = new Slime(); + final var entry = slime.setObject(); + + entry.setString("clusterId", "foobar"); + final var entryNames = entry.setArray("names"); + entryNames.addString("a"); + entryNames.addString("b"); + + final var endpoint = ContainerEndpointSerializer.endpointFromSlime(slime.get()); + assertEquals("foobar", endpoint.clusterId().toString()); + assertEquals(List.of("a", "b"), endpoint.names()); + } + + @Test + public void writeReadSingleEndpoint() { + final var endpoint = new ContainerEndpoint(new ClusterId("foo"), List.of("a", "b")); + final var serialized = new Slime(); + ContainerEndpointSerializer.endpointToSlime(serialized.setObject(), endpoint); + final var deserialized = ContainerEndpointSerializer.endpointFromSlime(serialized.get()); + + assertEquals(endpoint, deserialized); + } + + @Test + public void writeReadEndpoints() { + final var endpoints = List.of(new ContainerEndpoint(new ClusterId("foo"), List.of("a", "b"))); + final var serialized = ContainerEndpointSerializer.endpointListToSlime(endpoints); + final var deserialized = ContainerEndpointSerializer.endpointListFromSlime(serialized); + + assertEquals(endpoints, deserialized); + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointsCacheTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointsCacheTest.java new file mode 100644 index 00000000000..3598b6e63c3 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointsCacheTest.java @@ -0,0 +1,36 @@ +// Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.tenant; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.path.Path; +import com.yahoo.vespa.applicationmodel.ClusterId; +import com.yahoo.vespa.curator.mock.MockCurator; +import org.junit.Test; + +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class ContainerEndpointsCacheTest { + @Test + public void readWriteFromCache() { + final var cache = new ContainerEndpointsCache(Path.createRoot(), new MockCurator()); + final var endpoints = List.of( + new ContainerEndpoint(new ClusterId("the-cluster-1"), List.of("a", "b", "c")) + ); + + cache.write(ApplicationId.defaultId(), endpoints); + + final var deserialized = cache.read(ApplicationId.defaultId()); + + assertEquals(endpoints, deserialized); + } + + @Test + public void readingNonExistingEntry() { + final var cache = new ContainerEndpointsCache(Path.createRoot(), new MockCurator()); + final var endpoints = cache.read(ApplicationId.defaultId()); + assertTrue(endpoints.isEmpty()); + } +}
\ No newline at end of file diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java index 457e587da40..bf0abb1132f 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java @@ -124,7 +124,7 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM } private void trimResult(Execution execution) { - if (trimResult) { + if (trimResult || result.hits().size() > query.getHits()) { result.hits().trim(query.getOffset(), query.getHits()); } } diff --git a/container-search/src/main/java/com/yahoo/search/query/Model.java b/container-search/src/main/java/com/yahoo/search/query/Model.java index d2f59e0710e..5ed58a74627 100644 --- a/container-search/src/main/java/com/yahoo/search/query/Model.java +++ b/container-search/src/main/java/com/yahoo/search/query/Model.java @@ -377,7 +377,7 @@ public class Model implements Cloneable { * from a sources */ public void setRestrict(String restrictString) { - setFromString(restrictString,restrict); + setFromString(restrictString, restrict); } /** diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java index 8818a441fbd..159eb234aa7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.config.provision.zone.UpgradePolicy; @@ -42,9 +43,9 @@ public abstract class InfrastructureUpgrader extends Maintainer { /** Deploy a list of system applications until they converge on the given version */ private void upgradeAll(Version target, List<SystemApplication> applications) { - for (List<ZoneId> zones : upgradePolicy.asList()) { + for (List<ZoneApi> zones : upgradePolicy.asList()) { boolean converged = true; - for (ZoneId zone : zones) { + for (ZoneApi zone : zones) { try { converged &= upgradeAll(target, applications, zone); } catch (UnreachableNodeRepositoryException e) { @@ -62,7 +63,7 @@ public abstract class InfrastructureUpgrader extends Maintainer { } /** Returns whether all applications have converged to the target version in zone */ - private boolean upgradeAll(Version target, List<SystemApplication> applications, ZoneId zone) { + private boolean upgradeAll(Version target, List<SystemApplication> applications, ZoneApi zone) { boolean converged = true; for (SystemApplication application : applications) { if (convergedOn(target, application.dependencies(), zone)) { @@ -76,28 +77,28 @@ public abstract class InfrastructureUpgrader extends Maintainer { return converged; } - private boolean convergedOn(Version target, List<SystemApplication> applications, ZoneId zone) { + private boolean convergedOn(Version target, List<SystemApplication> applications, ZoneApi zone) { return applications.stream().allMatch(application -> convergedOn(target, application, zone)); } /** Upgrade component to target version. Implementation should be idempotent */ - protected abstract void upgrade(Version target, SystemApplication application, ZoneId zone); + protected abstract void upgrade(Version target, SystemApplication application, ZoneApi zone); /** Returns whether application has converged to target version in zone */ - protected abstract boolean convergedOn(Version target, SystemApplication application, ZoneId zone); + protected abstract boolean convergedOn(Version target, SystemApplication application, ZoneApi zone); /** Returns the target version for the component upgraded by this, if any */ protected abstract Optional<Version> targetVersion(); /** Returns whether the upgrader should require given node to upgrade */ - protected abstract boolean requireUpgradeOf(Node node, SystemApplication application, ZoneId zone); + protected abstract boolean requireUpgradeOf(Node node, SystemApplication application, ZoneApi zone); /** Find the minimum value of a version field in a zone */ - protected final Optional<Version> minVersion(ZoneId zone, SystemApplication application, Function<Node, Version> versionField) { + protected final Optional<Version> minVersion(ZoneApi zone, SystemApplication application, Function<Node, Version> versionField) { try { return controller().configServer() .nodeRepository() - .list(zone, application.id()) + .list(zone.getId(), application.id()) .stream() .filter(node -> requireUpgradeOf(node, application, zone)) .map(versionField) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java index ed3dd552085..8845f4c652f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java @@ -4,9 +4,9 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.google.common.collect.ImmutableSet; import com.yahoo.component.Version; import com.yahoo.config.provision.CloudName; +import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; -import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.versions.OsVersion; @@ -38,22 +38,22 @@ public class OsUpgrader extends InfrastructureUpgrader { } @Override - protected void upgrade(Version target, SystemApplication application, ZoneId zone) { + protected void upgrade(Version target, SystemApplication application, ZoneApi zone) { if (!application.isEligibleForOsUpgrades() || wantedVersion(zone, application, target).equals(target)) { return; } - log.info(String.format("Upgrading OS of %s to version %s in %s", application.id(), target, zone)); - controller().configServer().nodeRepository().upgradeOs(zone, application.nodeType(), target); + log.info(String.format("Upgrading OS of %s to version %s in %s in cloud %s", application.id(), target, zone.getId(), zone.getCloudName())); + controller().configServer().nodeRepository().upgradeOs(zone.getId(), application.nodeType(), target); } @Override - protected boolean convergedOn(Version target, SystemApplication application, ZoneId zone) { + protected boolean convergedOn(Version target, SystemApplication application, ZoneApi zone) { return currentVersion(zone, application, target).equals(target); } @Override - protected boolean requireUpgradeOf(Node node, SystemApplication application, ZoneId zone) { - return cloud.equals(zone.cloud()) && eligibleForUpgrade(node, application); + protected boolean requireUpgradeOf(Node node, SystemApplication application, ZoneApi zone) { + return cloud.equals(zone.getCloudName()) && eligibleForUpgrade(node, application); } @Override @@ -65,11 +65,11 @@ public class OsUpgrader extends InfrastructureUpgrader { .map(OsVersion::version); } - private Version currentVersion(ZoneId zone, SystemApplication application, Version defaultVersion) { + private Version currentVersion(ZoneApi zone, SystemApplication application, Version defaultVersion) { return minVersion(zone, application, Node::currentOsVersion).orElse(defaultVersion); } - private Version wantedVersion(ZoneId zone, SystemApplication application, Version defaultVersion) { + private Version wantedVersion(ZoneApi zone, SystemApplication application, Version defaultVersion) { return minVersion(zone, application, Node::wantedOsVersion).orElse(defaultVersion); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java index 8ef353248f8..5a40dd591fd 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; -import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.application.SystemApplication; @@ -30,26 +30,26 @@ public class SystemUpgrader extends InfrastructureUpgrader { } @Override - protected void upgrade(Version target, SystemApplication application, ZoneId zone) { + protected void upgrade(Version target, SystemApplication application, ZoneApi zone) { if (minVersion(zone, application, Node::wantedVersion).map(target::isAfter) .orElse(true)) { - log.info(String.format("Deploying %s version %s in %s", application.id(), target, zone)); - controller().applications().deploy(application, zone, target); + log.info(String.format("Deploying %s version %s in %s", application.id(), target, zone.getId())); + controller().applications().deploy(application, zone.toDeprecatedId(), target); } } @Override - protected boolean convergedOn(Version target, SystemApplication application, ZoneId zone) { + protected boolean convergedOn(Version target, SystemApplication application, ZoneApi zone) { Optional<Version> minVersion = minVersion(zone, application, Node::currentVersion); // Skip application convergence check if there are no nodes belonging to the application in the zone if (minVersion.isEmpty()) return true; return minVersion.get().equals(target) - && application.configConvergedIn(zone, controller(), Optional.of(target)); + && application.configConvergedIn(zone.toDeprecatedId(), controller(), Optional.of(target)); } @Override - protected boolean requireUpgradeOf(Node node, SystemApplication application, ZoneId zone) { + protected boolean requireUpgradeOf(Node node, SystemApplication application, ZoneApi zone) { return eligibleForUpgrade(node); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java index f9ce75d2297..f5b9d8263e5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java @@ -7,6 +7,7 @@ import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.SystemApplication; @@ -70,12 +71,12 @@ public class OsVersionStatus { if (!application.isEligibleForOsUpgrades()) { continue; // Avoid querying applications that are not eligible for OS upgrades } - for (ZoneId zone : zonesToUpgrade(controller)) { - controller.configServer().nodeRepository().list(zone, application.id()).stream() + for (ZoneApi zone : zonesToUpgrade(controller)) { + controller.configServer().nodeRepository().list(zone.getId(), application.id()).stream() .filter(node -> OsUpgrader.eligibleForUpgrade(node, application)) - .map(node -> new Node(node.hostname(), node.currentOsVersion(), zone.environment(), zone.region())) + .map(node -> new Node(node.hostname(), node.currentOsVersion(), zone.getEnvironment(), zone.getRegionName())) .forEach(node -> { - var version = new OsVersion(node.version(), zone.cloud()); + var version = new OsVersion(node.version(), zone.getCloudName()); versions.putIfAbsent(version, new ArrayList<>()); versions.get(version).add(node); }); @@ -85,7 +86,7 @@ public class OsVersionStatus { return new OsVersionStatus(versions); } - private static List<ZoneId> zonesToUpgrade(Controller controller) { + private static List<ZoneApi> zonesToUpgrade(Controller controller) { return controller.zoneRegistry().osUpgradePolicies().stream() .flatMap(upgradePolicy -> upgradePolicy.asList().stream()) .flatMap(Collection::stream) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java index 33bffaae96a..39ff29f4ae0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java @@ -49,10 +49,10 @@ public class OsUpgraderTest { public void upgrade_os() { OsUpgrader osUpgrader = osUpgrader( UpgradePolicy.create() - .upgrade(zone1.toDeprecatedId()) - .upgradeInParallel(zone2.toDeprecatedId(), zone3.toDeprecatedId()) - .upgrade(zone5.toDeprecatedId()) // Belongs to a different cloud and is ignored by this upgrader - .upgrade(zone4.toDeprecatedId()), + .upgrade(zone1) + .upgradeInParallel(zone2, zone3) + .upgrade(zone5) // Belongs to a different cloud and is ignored by this upgrader + .upgrade(zone4), SystemName.cd ); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdaterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdaterTest.java index d8e6f573592..fe7f39fd66d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdaterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdaterTest.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; import com.yahoo.config.provision.CloudName; +import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.config.provision.zone.UpgradePolicy; import com.yahoo.config.provision.zone.ZoneId; @@ -32,7 +33,7 @@ public class OsVersionStatusUpdaterTest { new JobControl(new MockCuratorDb())); // Add all zones to upgrade policy UpgradePolicy upgradePolicy = UpgradePolicy.create(); - for (ZoneId zone : tester.zoneRegistry().zones().controllerUpgraded().ids()) { + for (ZoneApi zone : tester.zoneRegistry().zones().controllerUpgraded().zones()) { upgradePolicy = upgradePolicy.upgrade(zone); } tester.zoneRegistry().setOsUpgradePolicy(CloudName.defaultName(), upgradePolicy); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java index c8fd7e90ccd..cb5e7cc90a1 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java @@ -3,11 +3,13 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; import com.yahoo.config.provision.zone.UpgradePolicy; +import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.integration.NodeRepositoryMock; +import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import org.junit.Before; import org.junit.Test; @@ -25,10 +27,10 @@ import static org.junit.Assert.assertTrue; */ public class SystemUpgraderTest { - private static final ZoneId zone1 = ZoneId.from("prod", "eu-west-1"); - private static final ZoneId zone2 = ZoneId.from("prod", "us-west-1"); - private static final ZoneId zone3 = ZoneId.from("prod", "us-central-1"); - private static final ZoneId zone4 = ZoneId.from("prod", "us-east-3"); + private static final ZoneApi zone1 = ZoneApiMock.fromId("prod.eu-west-1"); + private static final ZoneApi zone2 = ZoneApiMock.fromId("prod.us-west-1"); + private static final ZoneApi zone3 = ZoneApiMock.fromId("prod.us-central-1"); + private static final ZoneApi zone4 = ZoneApiMock.fromId("prod.us-east-3"); private DeploymentTester tester; @@ -48,8 +50,8 @@ public class SystemUpgraderTest { Version version1 = Version.fromString("6.5"); // Bootstrap a system without host applications - tester.configServer().bootstrap(List.of(zone1, zone2, zone3, zone4), SystemApplication.configServer, - SystemApplication.proxy); + tester.configServer().bootstrap(List.of(zone1.toDeprecatedId(), zone2.toDeprecatedId(), zone3.toDeprecatedId(), zone4.toDeprecatedId()), + SystemApplication.configServer, SystemApplication.proxy); // Fail a few nodes. Failed nodes should not affect versions failNodeIn(zone1, SystemApplication.configServer); failNodeIn(zone3, SystemApplication.proxy); @@ -142,7 +144,7 @@ public class SystemUpgraderTest { SystemUpgrader systemUpgrader = systemUpgrader(UpgradePolicy.create().upgrade(zone1)); // Bootstrap system - tester.configServer().bootstrap(List.of(zone1), SystemApplication.configServer, + tester.configServer().bootstrap(List.of(zone1.toDeprecatedId()), SystemApplication.configServer, SystemApplication.proxy); Version version1 = Version.fromString("6.5"); tester.upgradeSystem(version1); @@ -182,7 +184,7 @@ public class SystemUpgraderTest { ); Version version1 = Version.fromString("6.5"); - tester.configServer().bootstrap(List.of(zone1, zone2, zone3, zone4), SystemApplication.all()); + tester.configServer().bootstrap(List.of(zone1.toDeprecatedId(), zone2.toDeprecatedId(), zone3.toDeprecatedId(), zone4.toDeprecatedId()), SystemApplication.all()); tester.upgradeSystem(version1); systemUpgrader.maintain(); assertCurrentVersion(SystemApplication.all(), version1, zone1, zone2, zone3, zone4); @@ -280,7 +282,7 @@ public class SystemUpgraderTest { public void does_not_deploy_proxy_app_in_zones_without_proxy() { List<SystemApplication> applications = List.of( SystemApplication.configServerHost, SystemApplication.configServer, SystemApplication.tenantHost); - tester.configServer().bootstrap(List.of(zone1), applications); + tester.configServer().bootstrap(List.of(zone1.toDeprecatedId()), applications); tester.configServer().disallowConvergenceCheck(SystemApplication.proxy.id()); SystemUpgrader systemUpgrader = systemUpgrader(UpgradePolicy.create().upgrade(zone1)); @@ -292,36 +294,38 @@ public class SystemUpgraderTest { } /** Simulate upgrade of nodes allocated to given application. In a real system this is done by the node itself */ - private void completeUpgrade(SystemApplication application, Version version, ZoneId... zones) { + private void completeUpgrade(SystemApplication application, Version version, ZoneApi... zones) { assertWantedVersion(application, version, zones); - for (ZoneId zone : zones) { + for (ZoneApi zone : zones) { for (Node node : listNodes(zone, application)) { - nodeRepository().putByHostname(zone, new Node(node.hostname(), node.state(), node.type(), node.owner(), - node.wantedVersion(), node.wantedVersion())); + nodeRepository().putByHostname( + zone.getId(), + new Node(node.hostname(), node.state(), node.type(), node.owner(), node.wantedVersion(), node.wantedVersion())); } assertCurrentVersion(application, version, zone); } } - private void convergeServices(SystemApplication application, ZoneId... zones) { - for (ZoneId zone : zones) { - tester.controllerTester().configServer().convergeServices(application.id(), zone); + private void convergeServices(SystemApplication application, ZoneApi... zones) { + for (ZoneApi zone : zones) { + tester.controllerTester().configServer().convergeServices(application.id(), zone.toDeprecatedId()); } } - private void completeUpgrade(List<SystemApplication> applications, Version version, ZoneId... zones) { + private void completeUpgrade(List<SystemApplication> applications, Version version, ZoneApi... zones) { applications.forEach(application -> completeUpgrade(application, version, zones)); } - private void failNodeIn(ZoneId zone, SystemApplication application) { - List<Node> nodes = nodeRepository().list(zone, application.id()); + private void failNodeIn(ZoneApi zone, SystemApplication application) { + List<Node> nodes = nodeRepository().list(zone.getId(), application.id()); if (nodes.isEmpty()) { throw new IllegalArgumentException("No nodes allocated to " + application.id()); } Node node = nodes.get(0); - nodeRepository().putByHostname(zone, new Node(node.hostname(), Node.State.failed, node.type(), node.owner(), - node.currentVersion(), node.wantedVersion())); + nodeRepository().putByHostname( + zone.getId(), + new Node(node.hostname(), Node.State.failed, node.type(), node.owner(), node.currentVersion(), node.wantedVersion())); } private void assertSystemVersion(Version version) { @@ -332,33 +336,33 @@ public class SystemUpgraderTest { assertEquals(version, tester.controller().versionStatus().controllerVersion().get().versionNumber()); } - private void assertWantedVersion(SystemApplication application, Version version, ZoneId... zones) { + private void assertWantedVersion(SystemApplication application, Version version, ZoneApi... zones) { assertVersion(application, version, Node::wantedVersion, zones); } - private void assertCurrentVersion(SystemApplication application, Version version, ZoneId... zones) { + private void assertCurrentVersion(SystemApplication application, Version version, ZoneApi... zones) { assertVersion(application, version, Node::currentVersion, zones); } - private void assertWantedVersion(List<SystemApplication> applications, Version version, ZoneId... zones) { + private void assertWantedVersion(List<SystemApplication> applications, Version version, ZoneApi... zones) { applications.forEach(application -> assertVersion(application, version, Node::wantedVersion, zones)); } - private void assertCurrentVersion(List<SystemApplication> applications, Version version, ZoneId... zones) { + private void assertCurrentVersion(List<SystemApplication> applications, Version version, ZoneApi... zones) { applications.forEach(application -> assertVersion(application, version, Node::currentVersion, zones)); } private void assertVersion(SystemApplication application, Version version, Function<Node, Version> versionField, - ZoneId... zones) { - for (ZoneId zone : requireNonEmpty(zones)) { + ZoneApi... zones) { + for (ZoneApi zone : requireNonEmpty(zones)) { for (Node node : listNodes(zone, application)) { assertEquals(application + " version", version, versionField.apply(node)); } } } - private List<Node> listNodes(ZoneId zone, SystemApplication application) { - return nodeRepository().list(zone, application.id()).stream() + private List<Node> listNodes(ZoneApi zone, SystemApplication application) { + return nodeRepository().list(zone.getId(), application.id()).stream() .filter(SystemUpgrader::eligibleForUpgrade) .collect(Collectors.toList()); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java index f7b5a75fe15..b2dfd7b4cb6 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java @@ -53,8 +53,8 @@ public class OsApiTest extends ControllerContainerTest { addUserToHostedOperatorRole(operator); zoneRegistryMock().setSystemName(SystemName.cd) .setZones(zone1, zone2, zone3) - .setOsUpgradePolicy(cloud1, UpgradePolicy.create().upgrade(zone1.toDeprecatedId()).upgrade(zone2.toDeprecatedId())) - .setOsUpgradePolicy(cloud2, UpgradePolicy.create().upgrade(zone3.toDeprecatedId())); + .setOsUpgradePolicy(cloud1, UpgradePolicy.create().upgrade(zone1).upgrade(zone2)) + .setOsUpgradePolicy(cloud2, UpgradePolicy.create().upgrade(zone3)); osUpgraders = List.of( new OsUpgrader(tester.controller(), Duration.ofDays(1), new JobControl(tester.controller().curator()), diff --git a/document/src/vespa/document/base/idstring.cpp b/document/src/vespa/document/base/idstring.cpp index 7606ec58f9f..223baa6fd8d 100644 --- a/document/src/vespa/document/base/idstring.cpp +++ b/document/src/vespa/document/base/idstring.cpp @@ -6,6 +6,7 @@ #include <vespa/vespalib/util/md5.h> #include <vespa/vespalib/util/stringfmt.h> #include <limits> +#include <cerrno> using vespalib::string; using vespalib::stringref; diff --git a/document/src/vespa/document/datatype/referencedatatype.cpp b/document/src/vespa/document/datatype/referencedatatype.cpp index bc91f6b30ed..6caa6fdf7a9 100644 --- a/document/src/vespa/document/datatype/referencedatatype.cpp +++ b/document/src/vespa/document/datatype/referencedatatype.cpp @@ -3,6 +3,7 @@ #include "referencedatatype.h" #include <vespa/document/fieldvalue/referencefieldvalue.h> #include <vespa/vespalib/util/exceptions.h> +#include <ostream> using vespalib::make_string; using vespalib::IllegalArgumentException; diff --git a/document/src/vespa/document/fieldvalue/arrayfieldvalue.cpp b/document/src/vespa/document/fieldvalue/arrayfieldvalue.cpp index 8d928f7fb12..2f7f2208cbe 100644 --- a/document/src/vespa/document/fieldvalue/arrayfieldvalue.cpp +++ b/document/src/vespa/document/fieldvalue/arrayfieldvalue.cpp @@ -8,6 +8,7 @@ #include <vespa/vespalib/util/polymorphicarrays.h> #include <vespa/vespalib/util/xmlstream.h> #include <vespa/log/log.h> +#include <ostream> LOG_SETUP(".document.fieldvalue.array"); diff --git a/document/src/vespa/document/fieldvalue/mapfieldvalue.cpp b/document/src/vespa/document/fieldvalue/mapfieldvalue.cpp index 9c1b101e4ab..b5464401df2 100644 --- a/document/src/vespa/document/fieldvalue/mapfieldvalue.cpp +++ b/document/src/vespa/document/fieldvalue/mapfieldvalue.cpp @@ -9,6 +9,7 @@ #include <vespa/vespalib/stllike/hash_set.hpp> #include <cassert> #include <algorithm> +#include <ostream> #include <vespa/log/log.h> LOG_SETUP(".document.fieldvalue.map"); diff --git a/document/src/vespa/document/fieldvalue/referencefieldvalue.cpp b/document/src/vespa/document/fieldvalue/referencefieldvalue.cpp index dcda102f656..281161fccbf 100644 --- a/document/src/vespa/document/fieldvalue/referencefieldvalue.cpp +++ b/document/src/vespa/document/fieldvalue/referencefieldvalue.cpp @@ -4,6 +4,7 @@ #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/stringfmt.h> #include <cassert> +#include <ostream> using vespalib::IllegalArgumentException; using vespalib::make_string; diff --git a/document/src/vespa/document/fieldvalue/stringfieldvalue.cpp b/document/src/vespa/document/fieldvalue/stringfieldvalue.cpp index d79535d54cd..1d38653dba2 100644 --- a/document/src/vespa/document/fieldvalue/stringfieldvalue.cpp +++ b/document/src/vespa/document/fieldvalue/stringfieldvalue.cpp @@ -11,6 +11,7 @@ #include <vespa/document/serialization/annotationdeserializer.h> #include <vespa/document/repo/fixedtyperepo.h> #include <vespa/document/serialization/vespadocumentserializer.h> +#include <ostream> using vespalib::nbostream; using vespalib::ConstBufferRef; diff --git a/document/src/vespa/document/fieldvalue/structfieldvalue.cpp b/document/src/vespa/document/fieldvalue/structfieldvalue.cpp index ac37970213c..078e4d0ec21 100644 --- a/document/src/vespa/document/fieldvalue/structfieldvalue.cpp +++ b/document/src/vespa/document/fieldvalue/structfieldvalue.cpp @@ -14,6 +14,7 @@ #include <vespa/document/util/bytebuffer.h> #include <vespa/vespalib/util/xmlstream.h> #include <algorithm> +#include <ostream> #include <vespa/log/log.h> LOG_SETUP(".document.structfieldvalue"); diff --git a/document/src/vespa/document/select/doctype.cpp b/document/src/vespa/document/select/doctype.cpp index ba0338b9b61..0b55407092e 100644 --- a/document/src/vespa/document/select/doctype.cpp +++ b/document/src/vespa/document/select/doctype.cpp @@ -6,6 +6,7 @@ #include <vespa/document/update/documentupdate.h> #include <vespa/document/fieldvalue/document.h> #include <vespa/document/datatype/documenttype.h> +#include <ostream> namespace document::select { diff --git a/document/src/vespa/document/select/operator.cpp b/document/src/vespa/document/select/operator.cpp index 85dbef5ad9b..eaa795549bf 100644 --- a/document/src/vespa/document/select/operator.cpp +++ b/document/src/vespa/document/select/operator.cpp @@ -5,6 +5,7 @@ #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/stllike/hash_map.hpp> #include <cassert> +#include <ostream> #include <vespa/log/log.h> LOG_SETUP(".document.select.operator"); diff --git a/document/src/vespa/document/select/simpleparser.cpp b/document/src/vespa/document/select/simpleparser.cpp index a19d6086abe..879a1f195e8 100644 --- a/document/src/vespa/document/select/simpleparser.cpp +++ b/document/src/vespa/document/select/simpleparser.cpp @@ -2,6 +2,7 @@ #include "simpleparser.h" #include "compare.h" +#include <cerrno> namespace document { diff --git a/document/src/vespa/document/select/value.cpp b/document/src/vespa/document/select/value.cpp index 6b4bf15fc3b..5ebf527b82b 100644 --- a/document/src/vespa/document/select/value.cpp +++ b/document/src/vespa/document/select/value.cpp @@ -3,6 +3,7 @@ #include "value.h" #include "operator.h" #include <vespa/document/fieldvalue/fieldvalue.h> +#include <ostream> namespace document { namespace select { diff --git a/document/src/vespa/document/update/addvalueupdate.cpp b/document/src/vespa/document/update/addvalueupdate.cpp index e5a99b49a9e..e1132b2b571 100644 --- a/document/src/vespa/document/update/addvalueupdate.cpp +++ b/document/src/vespa/document/update/addvalueupdate.cpp @@ -7,6 +7,7 @@ #include <vespa/document/util/serializableexceptions.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/util/xmlstream.h> +#include <ostream> using vespalib::IllegalArgumentException; using vespalib::IllegalStateException; diff --git a/document/src/vespa/document/update/arithmeticvalueupdate.cpp b/document/src/vespa/document/update/arithmeticvalueupdate.cpp index 3af9350062e..90286da521a 100644 --- a/document/src/vespa/document/update/arithmeticvalueupdate.cpp +++ b/document/src/vespa/document/update/arithmeticvalueupdate.cpp @@ -5,6 +5,7 @@ #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/xmlstream.h> +#include <ostream> using vespalib::IllegalArgumentException; using vespalib::IllegalStateException; diff --git a/document/src/vespa/document/update/assignfieldpathupdate.cpp b/document/src/vespa/document/update/assignfieldpathupdate.cpp index bec717874dc..63e61ed3221 100644 --- a/document/src/vespa/document/update/assignfieldpathupdate.cpp +++ b/document/src/vespa/document/update/assignfieldpathupdate.cpp @@ -9,6 +9,7 @@ #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/util/exceptions.h> #include <boost/numeric/conversion/cast.hpp> +#include <ostream> #include <vespa/log/log.h> LOG_SETUP(".document.update.fieldpathupdate"); diff --git a/document/src/vespa/document/update/assignvalueupdate.cpp b/document/src/vespa/document/update/assignvalueupdate.cpp index cfedc1eb01b..48b30f13437 100644 --- a/document/src/vespa/document/update/assignvalueupdate.cpp +++ b/document/src/vespa/document/update/assignvalueupdate.cpp @@ -7,6 +7,7 @@ #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/xmlstream.h> +#include <ostream> using vespalib::IllegalArgumentException; using vespalib::IllegalStateException; diff --git a/document/src/vespa/document/update/documentupdate.cpp b/document/src/vespa/document/update/documentupdate.cpp index 48f32cb7d65..f2b8d40e0a3 100644 --- a/document/src/vespa/document/update/documentupdate.cpp +++ b/document/src/vespa/document/update/documentupdate.cpp @@ -12,6 +12,7 @@ #include <vespa/document/datatype/documenttype.h> #include <vespa/document/repo/documenttyperepo.h> #include <vespa/vespalib/util/xmlstream.h> +#include <ostream> using vespalib::IllegalArgumentException; using vespalib::IllegalStateException; diff --git a/document/src/vespa/document/update/fieldupdate.cpp b/document/src/vespa/document/update/fieldupdate.cpp index d36a134ecaa..3498d14d96e 100644 --- a/document/src/vespa/document/update/fieldupdate.cpp +++ b/document/src/vespa/document/update/fieldupdate.cpp @@ -5,6 +5,7 @@ #include <vespa/document/fieldvalue/document.h> #include <vespa/document/datatype/documenttype.h> #include <vespa/vespalib/objects/nbostream.h> +#include <ostream> namespace document { diff --git a/document/src/vespa/document/update/mapvalueupdate.cpp b/document/src/vespa/document/update/mapvalueupdate.cpp index be970b3c30a..0837615e4fb 100644 --- a/document/src/vespa/document/update/mapvalueupdate.cpp +++ b/document/src/vespa/document/update/mapvalueupdate.cpp @@ -7,6 +7,7 @@ #include <vespa/document/util/serializableexceptions.h> #include <vespa/vespalib/util/xmlstream.h> #include <vespa/vespalib/objects/nbostream.h> +#include <ostream> using vespalib::IllegalArgumentException; using vespalib::IllegalStateException; diff --git a/document/src/vespa/document/update/removevalueupdate.cpp b/document/src/vespa/document/update/removevalueupdate.cpp index fdbee3cb394..60176d93dce 100644 --- a/document/src/vespa/document/update/removevalueupdate.cpp +++ b/document/src/vespa/document/update/removevalueupdate.cpp @@ -8,6 +8,7 @@ #include <vespa/vespalib/objects/nbostream.h> #include <vespa/document/util/serializableexceptions.h> #include <vespa/vespalib/util/xmlstream.h> +#include <ostream> using vespalib::IllegalArgumentException; using vespalib::IllegalStateException; diff --git a/eval/src/vespa/eval/eval/operation.cpp b/eval/src/vespa/eval/eval/operation.cpp index 2c10e070bbd..fa0a99de461 100644 --- a/eval/src/vespa/eval/eval/operation.cpp +++ b/eval/src/vespa/eval/eval/operation.cpp @@ -2,6 +2,7 @@ #include "operation.h" #include <vespa/vespalib/util/approx.h> +#include <algorithm> namespace vespalib::eval::operation { diff --git a/fastos/src/vespa/fastos/unix_file.h b/fastos/src/vespa/fastos/unix_file.h index 3bca340cb90..3dffe1fc089 100644 --- a/fastos/src/vespa/fastos/unix_file.h +++ b/fastos/src/vespa/fastos/unix_file.h @@ -10,6 +10,7 @@ #pragma once #include <vespa/fastos/file.h> +#include <cerrno> /** * This is the generic UNIX implementation of @ref FastOS_FileInterface. diff --git a/fbench/src/util/clientstatus.h b/fbench/src/util/clientstatus.h index 9b15cdf4095..f8a223e121a 100644 --- a/fbench/src/util/clientstatus.h +++ b/fbench/src/util/clientstatus.h @@ -3,6 +3,7 @@ #include <map> #include <vector> +#include <string> /** * This is a helper struct that is used by the @ref Client class to diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index f7068e7147e..906a56d3f34 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -145,11 +145,6 @@ public class Flags { "Configserver RPC authorizer. Allowed values: ['disable', 'log-only', 'enforce']", "Takes effect on restart of configserver"); - public static final UnboundBooleanFlag ENABLE_TENANT_HOST_APP = defineFeatureFlag( - "enable-tenant-host-app", false, - "Enable tenant host infrastructure application", - "Takes effect immediately"); - /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, String description, diff --git a/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java b/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java index 5a38154b7c0..421d946c5db 100644 --- a/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java +++ b/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java @@ -6,7 +6,9 @@ import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.security.KeyUtils; import com.yahoo.security.SslContextBuilder; +import com.yahoo.security.X509CertificateUtils; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; @@ -63,11 +65,21 @@ public abstract class ControllerHttpClient { } /** Creates an HTTP client against the given endpoint, which uses the given key to authenticate as the given application. */ + public static ControllerHttpClient withSignatureKey(URI endpoint, String privateKey, ApplicationId id) { + return new SigningControllerHttpClient(endpoint, privateKey, id); + } + + /** Creates an HTTP client against the given endpoint, which uses the given key to authenticate as the given application. */ public static ControllerHttpClient withSignatureKey(URI endpoint, Path privateKeyFile, ApplicationId id) { return new SigningControllerHttpClient(endpoint, privateKeyFile, id); } /** Creates an HTTP client against the given endpoint, which uses the given private key and certificate identity. */ + public static ControllerHttpClient withKeyAndCertificate(URI endpoint, String privateKey, String certificate) { + return new MutualTlsControllerHttpClient(endpoint, privateKey, certificate); + } + + /** Creates an HTTP client against the given endpoint, which uses the given private key and certificate identity. */ public static ControllerHttpClient withKeyAndCertificate(URI endpoint, Path privateKeyFile, Path certificateFile) { return new MutualTlsControllerHttpClient(endpoint, privateKeyFile, certificateFile); } @@ -299,9 +311,13 @@ public abstract class ControllerHttpClient { private final RequestSigner signer; - private SigningControllerHttpClient(URI endpoint, Path privateKeyFile, ApplicationId id) { + private SigningControllerHttpClient(URI endpoint, String privateKey, ApplicationId id) { super(endpoint, HttpClient.newBuilder()); - this.signer = new RequestSigner(unchecked(() -> Files.readString(privateKeyFile, UTF_8)), id.serializedForm()); + this.signer = new RequestSigner(privateKey, id.serializedForm()); + } + + private SigningControllerHttpClient(URI endpoint, Path privateKeyFile, ApplicationId id) { + this(endpoint, unchecked(() -> Files.readString(privateKeyFile, UTF_8)), id); } @Override @@ -317,7 +333,18 @@ public abstract class ControllerHttpClient { private MutualTlsControllerHttpClient(URI endpoint, Path privateKeyFile, Path certificateFile) { super(endpoint, - HttpClient.newBuilder().sslContext(new SslContextBuilder().withKeyStore(privateKeyFile, certificateFile).build())); + HttpClient.newBuilder() + .sslContext(new SslContextBuilder().withKeyStore(privateKeyFile, + certificateFile) + .build())); + } + + private MutualTlsControllerHttpClient(URI endpoint, String privateKey, String certificate) { + super(endpoint, + HttpClient.newBuilder() + .sslContext(new SslContextBuilder().withKeyStore(KeyUtils.fromPemEncodedPrivateKey(privateKey), + X509CertificateUtils.certificateListFromPem(certificate)) + .build())); } } diff --git a/hosted-api/src/main/java/ai/vespa/hosted/api/RequestVerifier.java b/hosted-api/src/main/java/ai/vespa/hosted/api/RequestVerifier.java index dc53439ef3b..9d85ec9bf6b 100644 --- a/hosted-api/src/main/java/ai/vespa/hosted/api/RequestVerifier.java +++ b/hosted-api/src/main/java/ai/vespa/hosted/api/RequestVerifier.java @@ -29,6 +29,7 @@ public class RequestVerifier { this(pemPublicKey, Clock.systemUTC()); } + /** Creates a new request verifier from the given PEM encoded ECDSA public key, with the given clock. */ public RequestVerifier(String pemPublicKey, Clock clock) { this.verifier = SignatureUtils.createVerifier(KeyUtils.fromPemEncodedPublicKey(pemPublicKey), SHA256_WITH_ECDSA); this.clock = clock; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/VespaMetrics.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/VespaMetrics.java index becfd9a54ce..054fa704ecb 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/VespaMetrics.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/VespaMetrics.java @@ -41,19 +41,16 @@ import static com.yahoo.log.LogLevel.DEBUG; public class VespaMetrics { private static final Logger log = Logger.getLogger(VespaMetrics.class.getPackage().getName()); - // MUST be the same as the constant defined in config-model public static final ConsumerId VESPA_CONSUMER_ID = toConsumerId("Vespa"); public static final DimensionId METRIC_TYPE_DIMENSION_ID = toDimensionId("metrictype"); public static final DimensionId INSTANCE_DIMENSION_ID = toDimensionId("instance"); - private static final Set<ConsumerId> DEFAULT_CONSUMERS = Collections.singleton(VESPA_CONSUMER_ID); - private final MetricsConsumers metricsConsumers; private static final MetricsFormatter formatter = new MetricsFormatter(false, false); - public VespaMetrics(MetricsConsumers metricsConsumers, VespaServices vespaServices) { + public VespaMetrics(MetricsConsumers metricsConsumers) { this.metricsConsumers = metricsConsumers; } diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/TestUtil.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/TestUtil.java index 6f86be3aa30..5e8322c4c01 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/TestUtil.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/TestUtil.java @@ -31,7 +31,7 @@ public class TestUtil { MetricsConsumers consumers, ApplicationDimensions applicationDimensions, NodeDimensions nodeDimensions) { - VespaMetrics metrics = new VespaMetrics(consumers, vespaServices); + VespaMetrics metrics = new VespaMetrics(consumers); return new MetricsManager(vespaServices, metrics, new ExternalMetrics(consumers), applicationDimensions, nodeDimensions); } diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/IntegrationTester.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/IntegrationTester.java index 9507f01491e..df1ef9e5035 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/IntegrationTester.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/IntegrationTester.java @@ -68,7 +68,7 @@ public class IntegrationTester implements AutoCloseable { vespaServices = new VespaServices(servicesConfig(), monitoringConfig(), null); MetricsConsumers consumers = new MetricsConsumers(consumersConfig()); - VespaMetrics vespaMetrics = new VespaMetrics(consumers, vespaServices); + VespaMetrics vespaMetrics = new VespaMetrics(consumers); ExternalMetrics externalMetrics = new ExternalMetrics(consumers); ApplicationDimensions appDimensions = new ApplicationDimensions(applicationDimensionsConfig()); NodeDimensions nodeDimensions = new NodeDimensions(nodeDimensionsConfig()); diff --git a/metrics/src/vespa/metrics/metricset.cpp b/metrics/src/vespa/metrics/metricset.cpp index 9fb731d7583..b4238dc0a7c 100644 --- a/metrics/src/vespa/metrics/metricset.cpp +++ b/metrics/src/vespa/metrics/metricset.cpp @@ -8,6 +8,7 @@ #include <list> #include <cassert> #include <algorithm> +#include <ostream> #include <vespa/log/log.h> LOG_SETUP(".metrics.metricsset"); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdmin.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdmin.java index ca9083e9d27..00ec985ba0c 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdmin.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdmin.java @@ -9,7 +9,7 @@ import java.util.Set; /** * NodeAdmin manages the life cycle of NodeAgents. - * @author dybis + * @author Haakon Dybdahl */ public interface NodeAdmin { @@ -19,6 +19,9 @@ public interface NodeAdmin { /** Gather node agent and its docker container metrics and forward them to the {@code MetricReceiverWrapper} */ void updateNodeAgentMetrics(); + /** Gather node admin metrics and forward them to the {@code MetricReceiverWrapper} */ + void updateNodeAdminMetrics(); + /** * Attempts to freeze/unfreeze all NodeAgents and itself. To freeze a NodeAgent means that * they will not pick up any changes from NodeRepository. @@ -29,7 +32,7 @@ public interface NodeAdmin { boolean setFrozen(boolean frozen); /** - * Returns whether the NodeAdmin itself is currently frozen, meaning it will not pick up any changes + * Returns whether NodeAdmin itself is currently frozen, meaning it will not pick up any changes * from NodeRepository. */ boolean isFrozen(); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java index 5b5d13ca346..0d520241ac8 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java @@ -43,6 +43,9 @@ public class NodeAdminImpl implements NodeAdmin { private final Map<String, NodeAgentWithScheduler> nodeAgentWithSchedulerByHostname = new ConcurrentHashMap<>(); private final GaugeWrapper numberOfContainersInLoadImageState; + private final GaugeWrapper jvmHeapUsed; + private final GaugeWrapper jvmHeapFree; + private final GaugeWrapper jvmHeapTotal; private final CounterWrapper numberOfUnhandledExceptionsInNodeAgent; public NodeAdminImpl(NodeAgentFactory nodeAgentFactory, MetricReceiverWrapper metricReceiver, Clock clock) { @@ -70,6 +73,10 @@ public class NodeAdminImpl implements NodeAdmin { Dimensions dimensions = new Dimensions.Builder().add("role", "docker").build(); this.numberOfContainersInLoadImageState = metricReceiver.declareGauge(MetricReceiverWrapper.APPLICATION_DOCKER, dimensions, "nodes.image.loading"); this.numberOfUnhandledExceptionsInNodeAgent = metricReceiver.declareCounter(MetricReceiverWrapper.APPLICATION_DOCKER, dimensions, "nodes.unhandled_exceptions"); + + this.jvmHeapUsed = metricReceiver.declareGauge(MetricReceiverWrapper.APPLICATION_HOST, new Dimensions.Builder().build(), "mem.heap.used"); + this.jvmHeapFree = metricReceiver.declareGauge(MetricReceiverWrapper.APPLICATION_HOST, new Dimensions.Builder().build(), "mem.heap.free"); + this.jvmHeapTotal = metricReceiver.declareGauge(MetricReceiverWrapper.APPLICATION_HOST, new Dimensions.Builder().build(), "mem.heap.total"); } @Override @@ -113,6 +120,17 @@ public class NodeAdminImpl implements NodeAdmin { } @Override + public void updateNodeAdminMetrics() { + Runtime runtime = Runtime.getRuntime(); + long freeMemory = runtime.freeMemory(); + long totalMemory = runtime.totalMemory(); + long usedMemory = totalMemory - freeMemory; + jvmHeapFree.sample(freeMemory); + jvmHeapUsed.sample(usedMemory); + jvmHeapTotal.sample(totalMemory); + } + + @Override public boolean setFrozen(boolean wantFrozen) { if (wantFrozen != previousWantFrozen) { if (wantFrozen) { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java index 41c4544c533..2cd15a3ebe4 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java @@ -81,6 +81,7 @@ public class NodeAdminStateUpdater { try { if (suspendedStates.contains(currentState)) return; nodeAdmin.updateNodeAgentMetrics(); + nodeAdmin.updateNodeAdminMetrics(); } catch (Throwable e) { log.log(Level.WARNING, "Metric fetcher scheduler failed", e); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancer.java index a6d311604fd..58c576d3f44 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancer.java @@ -1,12 +1,9 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.lb; -import com.google.common.collect.ImmutableSortedSet; -import com.yahoo.config.provision.RotationName; import com.yahoo.vespa.hosted.provision.maintenance.LoadBalancerExpirer; import java.util.Objects; -import java.util.Set; /** * Represents a load balancer for an application's cluster. This is immutable. @@ -17,13 +14,11 @@ public class LoadBalancer { private final LoadBalancerId id; private final LoadBalancerInstance instance; - private final Set<RotationName> rotations; private final boolean inactive; - public LoadBalancer(LoadBalancerId id, LoadBalancerInstance instance, Set<RotationName> rotations, boolean inactive) { + public LoadBalancer(LoadBalancerId id, LoadBalancerInstance instance, boolean inactive) { this.id = Objects.requireNonNull(id, "id must be non-null"); this.instance = Objects.requireNonNull(instance, "instance must be non-null"); - this.rotations = ImmutableSortedSet.copyOf(Objects.requireNonNull(rotations, "rotations must be non-null")); this.inactive = inactive; } @@ -32,11 +27,6 @@ public class LoadBalancer { return id; } - /** The rotations of which this is a member */ - public Set<RotationName> rotations() { - return rotations; - } - /** The instance associated with this */ public LoadBalancerInstance instance() { return instance; @@ -52,7 +42,7 @@ public class LoadBalancer { /** Return a copy of this that is set inactive */ public LoadBalancer deactivate() { - return new LoadBalancer(id, instance, rotations, true); + return new LoadBalancer(id, instance, true); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveExpirer.java index 9efde8cf673..013fd169f45 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveExpirer.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; -import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Agent; @@ -51,8 +50,7 @@ public class InactiveExpirer extends Expirer { @Override protected boolean isExpired(Node node) { return super.isExpired(node) - || node.allocation().get().owner().instance().isTester() - || node.type() == NodeType.host; // TODO: Remove after removing tenant hosts from zone-app + || node.allocation().get().owner().instance().isTester(); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java index 18ae7e17d6d..25549abe9ed 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java @@ -174,7 +174,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { rebootInterval = Duration.ofDays(30); nodeRetirerInterval = Duration.ofMinutes(30); metricsInterval = Duration.ofMinutes(1); - infrastructureProvisionInterval = Duration.ofMinutes(3); + infrastructureProvisionInterval = Duration.ofMinutes(1); throttlePolicy = NodeFailer.ThrottlePolicy.hosted; loadBalancerExpiry = Duration.ofHours(1); reservationExpiry = Duration.ofMinutes(20); // Need to be long enough for deployment to be finished for all config model versions diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainer.java index 4c5310d69b6..46571fd0deb 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainer.java @@ -13,7 +13,6 @@ import java.time.Clock; import java.time.Duration; import java.time.Instant; import java.util.LinkedHashSet; -import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -30,8 +29,6 @@ import java.util.stream.Collectors; */ public class OperatorChangeApplicationMaintainer extends ApplicationMaintainer { - private static final ApplicationId ZONE_APPLICATION_ID = ApplicationId.from("hosted-vespa", "routing", "default"); - private final Clock clock; private Instant previousRun; @@ -47,9 +44,9 @@ public class OperatorChangeApplicationMaintainer extends ApplicationMaintainer { Instant windowEnd = clock.instant(); Instant windowStart = previousRun; previousRun = windowEnd; - return nodeRepository().getNodes().stream() + return nodeRepository().getNodes(NodeType.tenant).stream() .filter(node -> hasManualStateChangeSince(windowStart, node)) - .flatMap(node -> owner(node).stream()) + .flatMap(node -> node.allocation().map(Allocation::owner).stream()) .collect(Collectors.toCollection(LinkedHashSet::new)); } @@ -58,13 +55,6 @@ public class OperatorChangeApplicationMaintainer extends ApplicationMaintainer { .anyMatch(event -> event.agent() == Agent.operator && event.at().isAfter(instant)); } - private Optional<ApplicationId> owner(Node node) { - if (node.allocation().isPresent()) return node.allocation().map(Allocation::owner); - - // TODO: Remove after removing tenant hosts from zone-app - return node.type() == NodeType.host ? Optional.of(ZONE_APPLICATION_ID) : Optional.empty(); - } - /** * Deploy in the maintenance thread to avoid scheduling multiple deployments of the same application if it takes * longer to deploy than the (short) maintenance interval of this diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java index 17f2d7364a6..a4b915a6128 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.hosted.provision.persistence; import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.RotationName; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; @@ -35,7 +34,6 @@ public class LoadBalancerSerializer { private static final String portsField = "ports"; private static final String networksField = "networks"; private static final String realsField = "reals"; - private static final String rotationsField = "rotations"; private static final String nameField = "name"; private static final String ipAddressField = "ipAddress"; private static final String portField = "port"; @@ -58,11 +56,6 @@ public class LoadBalancerSerializer { realObject.setString(ipAddressField, real.ipAddress()); realObject.setLong(portField, real.port()); }); - Cursor rotationArray = root.setArray(rotationsField); - loadBalancer.rotations().forEach(rotation -> { - Cursor rotationObject = rotationArray.addObject(); - rotationObject.setString(nameField, rotation.value()); - }); root.setBool(inactiveField, loadBalancer.inactive()); try { @@ -89,11 +82,6 @@ public class LoadBalancerSerializer { Set<String> networks = new LinkedHashSet<>(); object.field(networksField).traverse((ArrayTraverser) (i, network) -> networks.add(network.asString())); - Set<RotationName> rotations = new LinkedHashSet<>(); - object.field(rotationsField).traverse((ArrayTraverser) (i, rotation) -> { - rotations.add(RotationName.from(rotation.field(nameField).asString())); - }); - return new LoadBalancer(LoadBalancerId.fromSerializedForm(object.field(idField).asString()), new LoadBalancerInstance( HostName.from(object.field(hostnameField).asString()), @@ -102,7 +90,6 @@ public class LoadBalancerSerializer { networks, reals ), - rotations, object.field(inactiveField).asBool()); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java index f5f8ed53d2a..372dca84a53 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java @@ -57,7 +57,7 @@ public class LoadBalancerProvisioner { LoadBalancerInstance instance = create(application, kv.getKey().id(), kv.getValue()); // Load balancer is always re-activated here to avoid reallocation if an application/cluster is // deleted and then redeployed. - LoadBalancer loadBalancer = new LoadBalancer(id, instance, kv.getKey().rotations(), false); + LoadBalancer loadBalancer = new LoadBalancer(id, instance, false); loadBalancers.put(loadBalancer.id(), loadBalancer); db.writeLoadBalancer(loadBalancer); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/LoadBalancersResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/LoadBalancersResponse.java index 69e13f77a09..d31834567ab 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/LoadBalancersResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/LoadBalancersResponse.java @@ -76,11 +76,7 @@ public class LoadBalancersResponse extends HttpResponse { realObject.setLong("port", real.port()); }); - Cursor rotationArray = lbObject.setArray("rotations"); - lb.rotations().forEach(rotation -> { - Cursor rotationObject = rotationArray.addObject(); - rotationObject.setString("name", rotation.value()); - }); + lbObject.setArray("rotations"); // To avoid changing the API. This can be removed when clients stop expecting this lbObject.setBool("inactive", lb.inactive()); }); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ZoneAppMigrationTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ZoneAppMigrationTest.java deleted file mode 100644 index ffdcea973e5..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ZoneAppMigrationTest.java +++ /dev/null @@ -1,171 +0,0 @@ -package com.yahoo.vespa.hosted.provision.maintenance; - -import com.yahoo.component.Version; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.Capacity; -import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.HostSpec; -import com.yahoo.config.provision.NodeResources; -import com.yahoo.config.provision.NodeType; -import com.yahoo.test.ManualClock; -import com.yahoo.vespa.hosted.provision.Node; -import com.yahoo.vespa.hosted.provision.node.Agent; -import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester; -import org.junit.Before; -import org.junit.Test; - -import java.time.Duration; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Set; -import java.util.stream.Collector; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static com.yahoo.config.provision.ClusterSpec.Type.container; -import static com.yahoo.config.provision.ClusterSpec.Type.content; -import static org.junit.Assert.assertEquals; - -/** - * This is a temporary test to verify the requirements needed for a successful migration of tenant - * host nodes out of the zone-application. - * - * TODO: Remove after removing tenant hosts from zone-app - * - * @author freva - */ -public class ZoneAppMigrationTest { - - private final ManualClock clock = new ManualClock(); - private final ProvisioningTester tester = new ProvisioningTester.Builder().build(); - private final InactiveExpirer inactiveExpirer = new InactiveExpirer(tester.nodeRepository(), clock, Duration.ofDays(99)); - - private final Version version = Version.fromString("7.42.23"); - - private final ApplicationId zoneApp = ApplicationId.from("hosted-vespa", "routing", "default"); - private final ApplicationId proxyHostApp = ApplicationId.from("hosted-vespa", "proxy-host", "default"); - private final ApplicationId tenantHostApp = ApplicationId.from("hosted-vespa", "tenant-host", "default"); - private final ApplicationId app1 = tester.makeApplicationId(); - private final ApplicationId app2 = tester.makeApplicationId(); - - - @Test - public void tenant_host_deallocation_test() { - assertEquals(5, tester.nodeRepository().getNodes(NodeType.proxy, Node.State.active).size()); - assertEquals(20, tester.nodeRepository().getNodes(NodeType.host, Node.State.active).size()); - assertEquals(15, tester.nodeRepository().getNodes(NodeType.tenant, Node.State.active).size()); - - Set<Node> tenantNodes = Set.copyOf(tester.nodeRepository().getNodes(NodeType.tenant)); - - // Activate zone-app with only proxy nodes, all tenant hosts become inactive, no change to other nodes - tester.activate(zoneApp, prepareSystemApplication(zoneApp, NodeType.proxy, "routing")); - assertEquals(5, tester.nodeRepository().getNodes(NodeType.proxy, Node.State.active).size()); - assertEquals(20, tester.nodeRepository().getNodes(NodeType.host, Node.State.inactive).size()); - assertEquals(tenantNodes, Set.copyOf(tester.nodeRepository().getNodes(NodeType.tenant))); - - // All tenant hosts become dirty, no change to other nodes - inactiveExpirer.maintain(); - assertEquals(5, tester.nodeRepository().getNodes(NodeType.proxy, Node.State.active).size()); - assertEquals(20, tester.nodeRepository().getNodes(NodeType.host, Node.State.dirty).size()); - assertEquals(tenantNodes, Set.copyOf(tester.nodeRepository().getNodes(NodeType.tenant))); - // No reboot generation incrementation - assertEquals(0, tester.nodeRepository().getNodes(NodeType.host).stream().mapToLong(node -> node.status().reboot().wanted()).sum()); - - tester.nodeRepository().getNodes(NodeType.host) - .forEach(node -> tester.nodeRepository().setReady(node.hostname(), Agent.operator, "Readied by host-admin")); - assertEquals(5, tester.nodeRepository().getNodes(NodeType.proxy, Node.State.active).size()); - assertEquals(20, tester.nodeRepository().getNodes(NodeType.host, Node.State.ready).size()); - assertEquals(tenantNodes, Set.copyOf(tester.nodeRepository().getNodes(NodeType.tenant))); - - tester.activate(tenantHostApp, prepareSystemApplication(tenantHostApp, NodeType.host, "tenant-host")); - assertEquals(5, tester.nodeRepository().getNodes(NodeType.proxy, Node.State.active).size()); - assertEquals(20, tester.nodeRepository().getNodes(NodeType.host, Node.State.active).size()); - assertEquals(tenantNodes, Set.copyOf(tester.nodeRepository().getNodes(NodeType.tenant))); - - // All tenant hosts are allocated to tenant host application - assertEquals(Set.copyOf(tester.nodeRepository().getNodes(NodeType.host)), - Set.copyOf(tester.nodeRepository().getNodes(tenantHostApp))); - - // All proxy nodes are still allocated to zone-app - assertEquals(Set.copyOf(tester.nodeRepository().getNodes(NodeType.proxy)), - Set.copyOf(tester.nodeRepository().getNodes(zoneApp))); - } - - @Test - public void conflicting_type_allocation_test() { - // Re-allocate tenant host from zone-app to tenant-host app - tester.activate(zoneApp, prepareSystemApplication(zoneApp, NodeType.proxy, "routing")); - inactiveExpirer.maintain(); - tester.nodeRepository().getNodes(NodeType.host) - .forEach(node -> tester.nodeRepository().setReady(node.hostname(), Agent.operator, "Readied by host-admin")); - tester.activate(tenantHostApp, prepareSystemApplication(tenantHostApp, NodeType.host, "tenant-host")); - - // Re-deploying zone-app with both type proxy and host has no effect (no tenant hosts are re-allocated from tenant-host app) - Set<Node> allNodes = Set.copyOf(tester.nodeRepository().getNodes()); - List<HostSpec> proxyHostSpecs = prepareSystemApplication(zoneApp, NodeType.proxy, "routing"); - List<HostSpec> nodeAdminHostSpecs = prepareSystemApplication(zoneApp, NodeType.host, "node-admin"); - List<HostSpec> zoneAppHostSpecs = concat(proxyHostSpecs, nodeAdminHostSpecs, Collectors.toList()); - tester.activate(zoneApp, zoneAppHostSpecs); - assertEquals(0, nodeAdminHostSpecs.size()); - assertEquals(allNodes, Set.copyOf(tester.nodeRepository().getNodes())); - - // Provision another host and redeploy zone-app - Node newHost = tester.makeReadyNodes(1, "large", NodeType.host).get(0); - proxyHostSpecs = prepareSystemApplication(zoneApp, NodeType.proxy, "routing"); - nodeAdminHostSpecs = prepareSystemApplication(zoneApp, NodeType.host, "node-admin"); - zoneAppHostSpecs = concat(proxyHostSpecs, nodeAdminHostSpecs, Collectors.toList()); - tester.activate(zoneApp, zoneAppHostSpecs); - - assertEquals(1, nodeAdminHostSpecs.size()); // The newly provisioned host is prepared - newHost = tester.nodeRepository().getNode(newHost.hostname()).orElseThrow(); // Update newHost after it has been allocated - Set<Node> allNodesWithNewHost = concat(allNodes, Set.of(newHost), Collectors.toSet()); - assertEquals(allNodesWithNewHost, Set.copyOf(tester.nodeRepository().getNodes())); - // The new host is allocated to zone-app, while the old ones are still allocated to tenant-host app - assertEquals(zoneApp, newHost.allocation().get().owner()); - } - - @Before - public void setup() { - tester.makeReadyNodes(5, "large", NodeType.proxyhost); - tester.makeReadyNodes(5, "large", NodeType.proxy); - tester.makeReadyNodes(20, "large", NodeType.host, 3); - - tester.activate(proxyHostApp, prepareSystemApplication(proxyHostApp, NodeType.proxyhost, "proxy-host")); - List<HostSpec> proxyHostSpecs = prepareSystemApplication(zoneApp, NodeType.proxy, "routing"); - List<HostSpec> nodeAdminHostSpecs = prepareSystemApplication(zoneApp, NodeType.host, "node-admin"); - List<HostSpec> zoneAppHostSpecs = concat(proxyHostSpecs, nodeAdminHostSpecs, Collectors.toList()); - tester.activate(zoneApp, zoneAppHostSpecs); - - activateTenantApplication(app1, 3, 4); - activateTenantApplication(app2, 5, 3); - } - - private List<HostSpec> prepareSystemApplication(ApplicationId applicationId, NodeType nodeType, String clusterId) { - return tester.prepare(applicationId, - ClusterSpec.request(container, ClusterSpec.Id.from(clusterId), version, false, Set.of()), - Capacity.fromRequiredNodeType(nodeType), - 1); - } - - private void activateTenantApplication(ApplicationId app, int numContainerNodes, int numContentNodes) { - List<HostSpec> combinedHostSpecs = new ArrayList<>(numContainerNodes + numContentNodes); - - combinedHostSpecs.addAll(tester.prepare(app, - ClusterSpec.request(container, ClusterSpec.Id.from("web"), version, false, Set.of()), - Capacity.fromCount(numContainerNodes, new NodeResources(2, 2, 50)), - 1)); - - combinedHostSpecs.addAll(tester.prepare(app, - ClusterSpec.request(content, ClusterSpec.Id.from("store"), version, false, Set.of()), - Capacity.fromCount(numContentNodes, new NodeResources(1, 4, 50)), - 1)); - - tester.activate(app, combinedHostSpecs); - } - - private <T, R, A> R concat(Collection<T> c1, Collection<T> c2, Collector<? super T, A, R> collector) { - return Stream.concat(c1.stream(), c2.stream()) - .collect(collector); - } -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializerTest.java index 6de93c2ae65..460764b50db 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializerTest.java @@ -5,7 +5,6 @@ import com.google.common.collect.ImmutableSet; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.RotationName; import com.yahoo.vespa.hosted.provision.lb.DnsZone; import com.yahoo.vespa.hosted.provision.lb.LoadBalancer; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerId; @@ -39,8 +38,6 @@ public class LoadBalancerSerializerTest { new Real(HostName.from("real-2"), "127.0.0.2", 4080))), - ImmutableSet.of(RotationName.from("eu-cluster"), - RotationName.from("us-cluster")), false); LoadBalancer serialized = LoadBalancerSerializer.fromJson(LoadBalancerSerializer.toJson(loadBalancer)); @@ -49,7 +46,6 @@ public class LoadBalancerSerializerTest { assertEquals(loadBalancer.instance().dnsZone(), serialized.instance().dnsZone()); assertEquals(loadBalancer.instance().ports(), serialized.instance().ports()); assertEquals(loadBalancer.instance().networks(), serialized.instance().networks()); - assertEquals(loadBalancer.rotations(), serialized.rotations()); assertEquals(loadBalancer.inactive(), serialized.inactive()); assertEquals(loadBalancer.instance().reals(), serialized.instance().reals()); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java index 58c0b3ed9cc..f97460713a5 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java @@ -61,7 +61,6 @@ public class LoadBalancerProvisionerTest { assertEquals(4080, get(loadBalancers.get().get(0).instance().reals(), 0).port()); assertEquals("127.0.0.2", get(loadBalancers.get().get(0).instance().reals(), 1).ipAddress()); assertEquals(4080, get(loadBalancers.get().get(0).instance().reals(), 1).port()); - assertEquals(rotationsCluster1, loadBalancers.get().get(0).rotations()); // A container is failed Supplier<List<Node>> containers = () -> tester.getNodes(app1).type(ClusterSpec.Type.container).asList(); @@ -105,7 +104,6 @@ public class LoadBalancerProvisionerTest { .map(Real::hostname) .sorted() .collect(Collectors.toList()); - assertEquals(rotationsCluster2, loadBalancers.get().get(1).rotations()); assertEquals(activeContainers, reals); // Application is removed and load balancer is deactivated diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/VespaModelUtil.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/VespaModelUtil.java index db9fe76dc62..c53d35c16dc 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/VespaModelUtil.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/VespaModelUtil.java @@ -35,11 +35,7 @@ public class VespaModelUtil { public static final ApplicationId TENANT_HOST_APPLICATION_ID = ApplicationId.from("hosted-vespa", "tenant-host", "default"); - // TODO: Remove after removing tenant hosts from zone-app - public static final ApplicationId ZONE_APPLICATION_ID = - ApplicationId.from("hosted-vespa", "routing", "default"); public static final ClusterId ADMIN_CLUSTER_ID = new ClusterId("admin"); - public static final ClusterId NODE_ADMIN_CLUSTER_ID = new ClusterId("node-admin"); public static final ServiceType SLOBROK_SERVICE_TYPE = new ServiceType("slobrok"); public static final ServiceType CLUSTER_CONTROLLER_SERVICE_TYPE = new ServiceType("container-clustercontroller"); diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicy.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicy.java index 7190473dd41..065defec1cd 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicy.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicy.java @@ -84,12 +84,6 @@ public class HostedVespaClusterPolicy implements ClusterPolicy { return ConcurrentSuspensionLimitForCluster.TWENTY_PERCENT; } - // TODO: Remove after removing tenant hosts from zone-app - if (clusterApi.getApplication().applicationId().equals(VespaModelUtil.ZONE_APPLICATION_ID) && - clusterApi.clusterId().equals(VespaModelUtil.NODE_ADMIN_CLUSTER_ID)) { - return ConcurrentSuspensionLimitForCluster.TWENTY_PERCENT; - } - return ConcurrentSuspensionLimitForCluster.TEN_PERCENT; } } diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicyTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicyTest.java index 6fc826c1b5f..d834034c9a8 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicyTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicyTest.java @@ -52,15 +52,6 @@ public class HostedVespaClusterPolicyTest { policy.getConcurrentSuspensionLimit(clusterApi)); } - @Test // TODO: Remove after removing tenant hosts from zone-app - public void testNodeAdminSuspensionLimit() { - when(applicationApi.applicationId()).thenReturn(VespaModelUtil.ZONE_APPLICATION_ID); - when(clusterApi.clusterId()).thenReturn(VespaModelUtil.NODE_ADMIN_CLUSTER_ID); - when(clusterApi.isStorageCluster()).thenReturn(false); - assertEquals(ConcurrentSuspensionLimitForCluster.TWENTY_PERCENT, - policy.getConcurrentSuspensionLimit(clusterApi)); - } - @Test public void testTenantHostSuspensionLimit() { when(applicationApi.applicationId()).thenReturn(VespaModelUtil.TENANT_HOST_APPLICATION_ID); @@ -120,6 +120,7 @@ <module>standalone-container</module> <module>statistics</module> <module>storage</module> + <module>tenant-auth</module> <module>tenant-base</module> <module>tenant-cd</module> <module>testutil</module> diff --git a/searchcore/src/tests/proton/index/CMakeLists.txt b/searchcore/src/tests/proton/index/CMakeLists.txt index 6fffa47f1b9..ef40e291e18 100644 --- a/searchcore/src/tests/proton/index/CMakeLists.txt +++ b/searchcore/src/tests/proton/index/CMakeLists.txt @@ -8,6 +8,7 @@ vespa_add_executable(searchcore_indexmanager_test_app TEST searchcore_flushengine searchcore_pcommon searchcore_util + gtest ) vespa_add_executable(searchcore_fusionrunner_test_app TEST SOURCES diff --git a/searchcore/src/tests/proton/index/indexmanager_test.cpp b/searchcore/src/tests/proton/index/indexmanager_test.cpp index 4149d563bf9..d92cc62c5a1 100644 --- a/searchcore/src/tests/proton/index/indexmanager_test.cpp +++ b/searchcore/src/tests/proton/index/indexmanager_test.cpp @@ -1,5 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Unit tests for IndexManager. #include <vespa/document/fieldvalue/document.h> #include <vespa/document/fieldvalue/fieldvalue.h> @@ -21,8 +20,8 @@ #include <vespa/searchlib/queryeval/isourceselector.h> #include <vespa/searchlib/test/index/mock_field_length_inspector.h> #include <vespa/searchlib/util/dirtraverse.h> +#include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/io/fileutil.h> -#include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/util/blockingthreadstackexecutor.h> #include <vespa/vespalib/util/threadstackexecutor.h> #include <set> @@ -32,6 +31,8 @@ LOG_SETUP("indexmanager_test"); using document::Document; using document::FieldValue; +using proton::index::IndexConfig; +using proton::index::IndexManager; using search::SequencedTaskExecutor; using search::SerialNum; using search::TuneFileAttributes; @@ -40,10 +41,10 @@ using search::TuneFileIndexing; using search::datastore::EntryRef; using search::index::DocBuilder; using search::index::DummyFileHeaderContext; +using search::index::FieldLengthInfo; using search::index::Schema; using search::index::schema::DataType; using search::index::test::MockFieldLengthInspector; -using vespalib::makeLambdaTask; using search::memoryindex::CompactWordsStore; using search::memoryindex::FieldIndexCollection; using search::queryeval::Source; @@ -51,8 +52,7 @@ using std::set; using std::string; using vespalib::BlockingThreadStackExecutor; using vespalib::ThreadStackExecutor; -using proton::index::IndexManager; -using proton::index::IndexConfig; +using vespalib::makeLambdaTask; using namespace proton; using namespace searchcorespi; @@ -61,12 +61,12 @@ using namespace searchcorespi::index; namespace { class IndexManagerDummyReconfigurer : public searchcorespi::IIndexManager::Reconfigurer { - virtual bool - reconfigure(vespalib::Closure0<bool>::UP closure) override - { + + virtual bool reconfigure(vespalib::Closure0<bool>::UP closure) override { bool ret = true; - if (closure.get() != NULL) + if (closure.get() != nullptr) { ret = closure->call(); // Perform index manager reconfiguration now + } return ret; } @@ -97,7 +97,7 @@ Document::UP buildDocument(DocBuilder &doc_builder, int id, std::shared_ptr<search::IDestructorCallback> emptyDestructorCallback; -struct Fixture { +struct IndexManagerTest : public ::testing::Test { SerialNum _serial_num; IndexManagerDummyReconfigurer _reconfigurer; DummyFileHeaderContext _fileHeaderContext; @@ -107,7 +107,7 @@ struct Fixture { Schema _schema; DocBuilder _builder; - Fixture() + IndexManagerTest() : _serial_num(0), _reconfigurer(), _fileHeaderContext(), @@ -123,7 +123,7 @@ struct Fixture { resetIndexManager(); } - ~Fixture() { + ~IndexManagerTest() { _writeService.shutdown(); } @@ -159,19 +159,27 @@ struct Fixture { uint32_t expNumMemoryIndexes, SerialNum expLastiskIndexSerialNum, SerialNum expLastMemoryIndexSerialNum); + + IIndexCollection::SP get_source_collection() const { + return _index_manager->getMaintainer().getSourceCollection(); + } }; -void Fixture::flushIndexManager() { +void +IndexManagerTest::flushIndexManager() +{ vespalib::Executor::Task::UP task; SerialNum serialNum = _index_manager->getCurrentSerialNum(); auto &maintainer = _index_manager->getMaintainer(); - runAsMaster([&]() { task = maintainer.initFlush(serialNum, NULL); }); + runAsMaster([&]() { task = maintainer.initFlush(serialNum, nullptr); }); if (task.get()) { task->run(); } } -Document::UP Fixture::addDocument(uint32_t id) { +Document::UP +IndexManagerTest::addDocument(uint32_t id) +{ Document::UP doc = buildDocument(_builder, id, "foo"); SerialNum serialNum = ++_serial_num; runAsIndex([&]() { _index_manager->putDocument(id, *doc, serialNum); @@ -181,16 +189,18 @@ Document::UP Fixture::addDocument(uint32_t id) { return doc; } -void Fixture::resetIndexManager() { +void +IndexManagerTest::resetIndexManager() +{ _index_manager.reset(); _index_manager = std::make_unique<IndexManager>(index_dir, IndexConfig(), getSchema(), 1, _reconfigurer, _writeService, _writeService.getMasterExecutor(), TuneFileIndexManager(), TuneFileAttributes(),_fileHeaderContext); } - -void Fixture::assertStats(uint32_t expNumDiskIndexes, uint32_t expNumMemoryIndexes, - SerialNum expLastDiskIndexSerialNum, SerialNum expLastMemoryIndexSerialNum) +void +IndexManagerTest::assertStats(uint32_t expNumDiskIndexes, uint32_t expNumMemoryIndexes, + SerialNum expLastDiskIndexSerialNum, SerialNum expLastMemoryIndexSerialNum) { searchcorespi::IndexManagerStats stats(*_index_manager); SerialNum lastDiskIndexSerialNum = 0; @@ -203,36 +213,38 @@ void Fixture::assertStats(uint32_t expNumDiskIndexes, uint32_t expNumMemoryIndex if (!memoryIndexes.empty()) { lastMemoryIndexSerialNum = memoryIndexes.back().getSerialNum(); } - EXPECT_EQUAL(expNumDiskIndexes, diskIndexes.size()); - EXPECT_EQUAL(expNumMemoryIndexes, memoryIndexes.size()); - EXPECT_EQUAL(expLastDiskIndexSerialNum, lastDiskIndexSerialNum); - EXPECT_EQUAL(expLastMemoryIndexSerialNum, lastMemoryIndexSerialNum); + EXPECT_EQ(expNumDiskIndexes, diskIndexes.size()); + EXPECT_EQ(expNumMemoryIndexes, memoryIndexes.size()); + EXPECT_EQ(expLastDiskIndexSerialNum, lastDiskIndexSerialNum); + EXPECT_EQ(expLastMemoryIndexSerialNum, lastMemoryIndexSerialNum); } +TEST_F(IndexManagerTest, require_that_empty_memory_index_is_not_flushed) +{ + auto sources = get_source_collection(); + EXPECT_EQ(1u, sources->getSourceCount()); -TEST_F("requireThatEmptyMemoryIndexIsNotFlushed", Fixture) { - IIndexCollection::SP sources = f._index_manager->getMaintainer().getSourceCollection(); - EXPECT_EQUAL(1u, sources->getSourceCount()); - - f.flushIndexManager(); + flushIndexManager(); - sources = f._index_manager->getMaintainer().getSourceCollection(); - EXPECT_EQUAL(1u, sources->getSourceCount()); + sources = get_source_collection(); + EXPECT_EQ(1u, sources->getSourceCount()); } -TEST_F("requireThatEmptyMemoryIndexIsFlushedIfSourceSelectorChanged", Fixture) +TEST_F(IndexManagerTest, require_that_empty_memory_index_is_flushed_if_source_selector_changed) { - IIndexCollection::SP sources = f._index_manager->getMaintainer().getSourceCollection(); - EXPECT_EQUAL(1u, sources->getSourceCount()); + auto sources = get_source_collection(); + EXPECT_EQ(1u, sources->getSourceCount()); - f.removeDocument(docid, 42); - f.flushIndexManager(); + removeDocument(docid, 42); + flushIndexManager(); - sources = f._index_manager->getMaintainer().getSourceCollection(); - EXPECT_EQUAL(2u, sources->getSourceCount()); + sources = get_source_collection(); + EXPECT_EQ(2u, sources->getSourceCount()); } -set<uint32_t> readDiskIds(const string &dir, const string &type) { +set<uint32_t> +readDiskIds(const string &dir, const string &type) +{ set<uint32_t> ids; FastOS_DirectoryScan dir_scan(dir.c_str()); while (dir_scan.ReadNext()) { @@ -254,76 +266,77 @@ set<uint32_t> readDiskIds(const string &dir, const string &type) { return ids; } -TEST_F("requireThatMemoryIndexIsFlushed", Fixture) { +TEST_F(IndexManagerTest, require_that_memory_index_is_flushed) +{ FastOS_StatInfo stat; { - f.addDocument(docid); + addDocument(docid); - IIndexCollection::SP sources = - f._index_manager->getMaintainer().getSourceCollection(); - EXPECT_EQUAL(1u, sources->getSourceCount()); - EXPECT_EQUAL(1u, sources->getSourceId(0)); + auto sources = get_source_collection(); + EXPECT_EQ(1u, sources->getSourceCount()); + EXPECT_EQ(1u, sources->getSourceId(0)); - IndexFlushTarget target(f._index_manager->getMaintainer()); - EXPECT_EQUAL(0, target.getLastFlushTime().time()); + IndexFlushTarget target(_index_manager->getMaintainer()); + EXPECT_EQ(0, target.getLastFlushTime().time()); vespalib::Executor::Task::UP flushTask; - f.runAsMaster([&]() { flushTask = target.initFlush(1); }); + runAsMaster([&]() { flushTask = target.initFlush(1); }); flushTask->run(); EXPECT_TRUE(FastOS_File::Stat("test_data/index.flush.1", &stat)); - EXPECT_EQUAL(stat._modifiedTime, target.getLastFlushTime().time()); + EXPECT_EQ(stat._modifiedTime, target.getLastFlushTime().time()); - sources = f._index_manager->getMaintainer().getSourceCollection(); - EXPECT_EQUAL(2u, sources->getSourceCount()); - EXPECT_EQUAL(1u, sources->getSourceId(0)); - EXPECT_EQUAL(2u, sources->getSourceId(1)); + sources = get_source_collection(); + EXPECT_EQ(2u, sources->getSourceCount()); + EXPECT_EQ(1u, sources->getSourceId(0)); + EXPECT_EQ(2u, sources->getSourceId(1)); set<uint32_t> disk_ids = readDiskIds(index_dir, "flush"); ASSERT_TRUE(disk_ids.size() == 1); - EXPECT_EQUAL(1u, *disk_ids.begin()); + EXPECT_EQ(1u, *disk_ids.begin()); FlushStats stats = target.getLastFlushStats(); - EXPECT_EQUAL("test_data/index.flush.1", stats.getPath()); - EXPECT_EQUAL(7u, stats.getPathElementsToLog()); + EXPECT_EQ("test_data/index.flush.1", stats.getPath()); + EXPECT_EQ(7u, stats.getPathElementsToLog()); } { // verify last flush time when loading disk index - f.resetIndexManager(); - IndexFlushTarget target(f._index_manager->getMaintainer()); - EXPECT_EQUAL(stat._modifiedTime, target.getLastFlushTime().time()); + resetIndexManager(); + IndexFlushTarget target(_index_manager->getMaintainer()); + EXPECT_EQ(stat._modifiedTime, target.getLastFlushTime().time()); // updated serial number & flush time when nothing to flush FastOS_Thread::Sleep(8000); fastos::TimeStamp now = fastos::ClockSystem::now(); vespalib::Executor::Task::UP task; - f.runAsMaster([&]() { task = target.initFlush(2); }); - EXPECT_TRUE(task.get() == NULL); - EXPECT_EQUAL(2u, target.getFlushedSerialNum()); - EXPECT_LESS(stat._modifiedTime, target.getLastFlushTime().time()); - EXPECT_APPROX(now.time(), target.getLastFlushTime().time(), 8); + runAsMaster([&]() { task = target.initFlush(2); }); + EXPECT_TRUE(task.get() == nullptr); + EXPECT_EQ(2u, target.getFlushedSerialNum()); + EXPECT_LT(stat._modifiedTime, target.getLastFlushTime().time()); + EXPECT_NEAR(now.time(), target.getLastFlushTime().time(), 8); } } -TEST_F("requireThatMultipleFlushesGivesMultipleIndexes", Fixture) { +TEST_F(IndexManagerTest, require_that_multiple_flushes_gives_multiple_indexes) +{ size_t flush_count = 10; for (size_t i = 0; i < flush_count; ++i) { - f.addDocument(docid); - f.flushIndexManager(); + addDocument(docid); + flushIndexManager(); } set<uint32_t> disk_ids = readDiskIds(index_dir, "flush"); - EXPECT_EQUAL(flush_count, disk_ids.size()); + EXPECT_EQ(flush_count, disk_ids.size()); uint32_t i = 1; - for (set<uint32_t>::iterator it = disk_ids.begin(); it != disk_ids.end(); - ++it) { - EXPECT_EQUAL(i++, *it); + for (auto it = disk_ids.begin(); it != disk_ids.end(); ++it) { + EXPECT_EQ(i++, *it); } } -TEST_F("requireThatMaxFlushesSetsUrgent", Fixture) { +TEST_F(IndexManagerTest, require_that_max_flushes_sets_urgent) +{ size_t flush_count = 20; for (size_t i = 0; i < flush_count; ++i) { - f.addDocument(docid); - f.flushIndexManager(); + addDocument(docid); + flushIndexManager(); } - IndexFusionTarget target(f._index_manager->getMaintainer()); + IndexFusionTarget target(_index_manager->getMaintainer()); EXPECT_TRUE(target.needUrgentFlush()); } @@ -331,35 +344,39 @@ uint32_t getSource(const IIndexCollection &sources, uint32_t id) { return sources.getSourceSelector().createIterator()->getSource(id); } -TEST_F("requireThatPutDocumentUpdatesSelector", Fixture) { - f.addDocument(docid); - IIndexCollection::SP sources = f._index_manager->getMaintainer().getSourceCollection(); - EXPECT_EQUAL(1u, getSource(*sources, docid)); - f.flushIndexManager(); - f.addDocument(docid + 1); - sources = f._index_manager->getMaintainer().getSourceCollection(); - EXPECT_EQUAL(1u, getSource(*sources, docid)); - EXPECT_EQUAL(2u, getSource(*sources, docid + 1)); +TEST_F(IndexManagerTest, require_that_put_document_updates_selector) +{ + addDocument(docid); + auto sources = get_source_collection(); + EXPECT_EQ(1u, getSource(*sources, docid)); + flushIndexManager(); + addDocument(docid + 1); + sources = get_source_collection(); + EXPECT_EQ(1u, getSource(*sources, docid)); + EXPECT_EQ(2u, getSource(*sources, docid + 1)); } -TEST_F("requireThatRemoveDocumentUpdatesSelector", Fixture) { - Document::UP doc = f.addDocument(docid); - IIndexCollection::SP sources = f._index_manager->getMaintainer().getSourceCollection(); - EXPECT_EQUAL(1u, getSource(*sources, docid)); - f.flushIndexManager(); - f.removeDocument(docid, ++f._serial_num); - sources = f._index_manager->getMaintainer().getSourceCollection(); - EXPECT_EQUAL(2u, getSource(*sources, docid)); +TEST_F(IndexManagerTest, require_that_remove_document_updates_selector) +{ + Document::UP doc = addDocument(docid); + auto sources = get_source_collection(); + EXPECT_EQ(1u, getSource(*sources, docid)); + flushIndexManager(); + removeDocument(docid, ++_serial_num); + sources = get_source_collection(); + EXPECT_EQ(2u, getSource(*sources, docid)); } -TEST_F("requireThatSourceSelectorIsFlushed", Fixture) { - f.addDocument(docid); - f.flushIndexManager(); +TEST_F(IndexManagerTest, require_that_source_selector_is_flushed) +{ + addDocument(docid); + flushIndexManager(); FastOS_File file((index_dir + "/index.flush.1/selector.dat").c_str()); ASSERT_TRUE(file.OpenReadOnlyExisting()); } -TEST_F("requireThatFlushStatsAreCalculated", Fixture) { +TEST_F(IndexManagerTest, require_that_flush_stats_are_calculated) +{ Schema schema(getSchema()); FieldIndexCollection fic(schema, MockFieldLengthInspector()); SequencedTaskExecutor invertThreads(2); @@ -371,12 +388,12 @@ TEST_F("requireThatFlushStatsAreCalculated", Fixture) { uint64_t index_size = fic.getMemoryUsage().allocatedBytes() - fixed_index_size; /// Must account for both docid 0 being reserved and the extra after. uint64_t selector_size = (1) * sizeof(Source); - EXPECT_EQUAL(index_size, f._index_manager->getMaintainer().getFlushStats().memory_before_bytes - - f._index_manager->getMaintainer().getFlushStats().memory_after_bytes); - EXPECT_EQUAL(0u, f._index_manager->getMaintainer().getFlushStats().disk_write_bytes); - EXPECT_EQUAL(0u, f._index_manager->getMaintainer().getFlushStats().cpu_time_required); + EXPECT_EQ(index_size, _index_manager->getMaintainer().getFlushStats().memory_before_bytes - + _index_manager->getMaintainer().getFlushStats().memory_after_bytes); + EXPECT_EQ(0u, _index_manager->getMaintainer().getFlushStats().disk_write_bytes); + EXPECT_EQ(0u, _index_manager->getMaintainer().getFlushStats().cpu_time_required); - Document::UP doc = f.addDocument(docid); + Document::UP doc = addDocument(docid); inverter.invertDocument(docid, *doc); invertThreads.sync(); inverter.pushDocuments(std::shared_ptr<search::IDestructorCallback>()); @@ -385,17 +402,17 @@ TEST_F("requireThatFlushStatsAreCalculated", Fixture) { /// Must account for both docid 0 being reserved and the extra after. selector_size = (docid + 1) * sizeof(Source); - EXPECT_EQUAL(index_size, - f._index_manager->getMaintainer().getFlushStats().memory_before_bytes - - f._index_manager->getMaintainer().getFlushStats().memory_after_bytes); - EXPECT_EQUAL(selector_size + index_size, - f._index_manager->getMaintainer().getFlushStats().disk_write_bytes); - EXPECT_EQUAL(selector_size * (3+1) + index_size, - f._index_manager->getMaintainer().getFlushStats().cpu_time_required); - - doc = f.addDocument(docid + 10); + EXPECT_EQ(index_size, + _index_manager->getMaintainer().getFlushStats().memory_before_bytes - + _index_manager->getMaintainer().getFlushStats().memory_after_bytes); + EXPECT_EQ(selector_size + index_size, + _index_manager->getMaintainer().getFlushStats().disk_write_bytes); + EXPECT_EQ(selector_size * (3+1) + index_size, + _index_manager->getMaintainer().getFlushStats().cpu_time_required); + + doc = addDocument(docid + 10); inverter.invertDocument(docid + 10, *doc); - doc = f.addDocument(docid + 100); + doc = addDocument(docid + 100); inverter.invertDocument(docid + 100, *doc); invertThreads.sync(); inverter.pushDocuments(std::shared_ptr<search::IDestructorCallback>()); @@ -403,137 +420,144 @@ TEST_F("requireThatFlushStatsAreCalculated", Fixture) { index_size = fic.getMemoryUsage().allocatedBytes() - fixed_index_size; /// Must account for both docid 0 being reserved and the extra after. selector_size = (docid + 100 + 1) * sizeof(Source); - EXPECT_EQUAL(index_size, - f._index_manager->getMaintainer().getFlushStats().memory_before_bytes - - f._index_manager->getMaintainer().getFlushStats().memory_after_bytes); - EXPECT_EQUAL(selector_size + index_size, - f._index_manager->getMaintainer().getFlushStats().disk_write_bytes); - EXPECT_EQUAL(selector_size * (3+1) + index_size, - f._index_manager->getMaintainer().getFlushStats().cpu_time_required); + EXPECT_EQ(index_size, + _index_manager->getMaintainer().getFlushStats().memory_before_bytes - + _index_manager->getMaintainer().getFlushStats().memory_after_bytes); + EXPECT_EQ(selector_size + index_size, + _index_manager->getMaintainer().getFlushStats().disk_write_bytes); + EXPECT_EQ(selector_size * (3+1) + index_size, + _index_manager->getMaintainer().getFlushStats().cpu_time_required); } -TEST_F("requireThatFusionStatsAreCalculated", Fixture) { - f.addDocument(docid); - EXPECT_EQUAL(0u, f._index_manager->getMaintainer().getFusionStats().diskUsage); - f.flushIndexManager(); - ASSERT_TRUE(f._index_manager->getMaintainer().getFusionStats().diskUsage > 0); +TEST_F(IndexManagerTest, require_that_fusion_stats_are_calculated) +{ + addDocument(docid); + EXPECT_EQ(0u, _index_manager->getMaintainer().getFusionStats().diskUsage); + flushIndexManager(); + ASSERT_TRUE(_index_manager->getMaintainer().getFusionStats().diskUsage > 0); } -TEST_F("requireThatPutDocumentUpdatesSerialNum", Fixture) { - f._serial_num = 0; - EXPECT_EQUAL(0u, f._index_manager->getCurrentSerialNum()); - f.addDocument(docid); - EXPECT_EQUAL(1u, f._index_manager->getCurrentSerialNum()); +TEST_F(IndexManagerTest, require_that_put_document_updates_serial_num) +{ + _serial_num = 0; + EXPECT_EQ(0u, _index_manager->getCurrentSerialNum()); + addDocument(docid); + EXPECT_EQ(1u, _index_manager->getCurrentSerialNum()); } -TEST_F("requireThatRemoveDocumentUpdatesSerialNum", Fixture) { - f._serial_num = 0; - Document::UP doc = f.addDocument(docid); - EXPECT_EQUAL(1u, f._index_manager->getCurrentSerialNum()); - f.removeDocument(docid, ++f._serial_num); - EXPECT_EQUAL(2u, f._index_manager->getCurrentSerialNum()); +TEST_F(IndexManagerTest, require_that_remove_document_updates_serial_num) +{ + _serial_num = 0; + Document::UP doc = addDocument(docid); + EXPECT_EQ(1u, _index_manager->getCurrentSerialNum()); + removeDocument(docid, ++_serial_num); + EXPECT_EQ(2u, _index_manager->getCurrentSerialNum()); } -TEST_F("requireThatFlushUpdatesSerialNum", Fixture) { - f._serial_num = 0; - f.addDocument(docid); - EXPECT_EQUAL(1u, f._index_manager->getCurrentSerialNum()); - EXPECT_EQUAL(0u, f._index_manager->getFlushedSerialNum()); - f.flushIndexManager(); - EXPECT_EQUAL(1u, f._index_manager->getCurrentSerialNum()); - EXPECT_EQUAL(1u, f._index_manager->getFlushedSerialNum()); +TEST_F(IndexManagerTest, require_that_flush_updates_serial_num) +{ + _serial_num = 0; + addDocument(docid); + EXPECT_EQ(1u, _index_manager->getCurrentSerialNum()); + EXPECT_EQ(0u, _index_manager->getFlushedSerialNum()); + flushIndexManager(); + EXPECT_EQ(1u, _index_manager->getCurrentSerialNum()); + EXPECT_EQ(1u, _index_manager->getFlushedSerialNum()); } -TEST_F("requireThatFusionUpdatesIndexes", Fixture) { +TEST_F(IndexManagerTest, require_that_fusion_updates_indexes) +{ for (size_t i = 0; i < 10; ++i) { - f.addDocument(docid + i); - f.flushIndexManager(); + addDocument(docid + i); + flushIndexManager(); } uint32_t ids[] = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }; - IIndexCollection::SP - source_list(f._index_manager->getMaintainer().getSourceCollection()); - EXPECT_EQUAL(10u + 1, source_list->getSourceCount()); // disk + mem - EXPECT_EQUAL(ids[2], getSource(*source_list, docid + 2)); - EXPECT_EQUAL(ids[6], getSource(*source_list, docid + 6)); + auto sources = get_source_collection(); + EXPECT_EQ(10u + 1, sources->getSourceCount()); // disk + mem + EXPECT_EQ(ids[2], getSource(*sources, docid + 2)); + EXPECT_EQ(ids[6], getSource(*sources, docid + 6)); FusionSpec fusion_spec; fusion_spec.flush_ids.assign(ids, ids + 4); - f._index_manager->getMaintainer().runFusion(fusion_spec); + _index_manager->getMaintainer().runFusion(fusion_spec); set<uint32_t> fusion_ids = readDiskIds(index_dir, "fusion"); - EXPECT_EQUAL(1u, fusion_ids.size()); - EXPECT_EQUAL(ids[3], *fusion_ids.begin()); + EXPECT_EQ(1u, fusion_ids.size()); + EXPECT_EQ(ids[3], *fusion_ids.begin()); - source_list = f._index_manager->getMaintainer().getSourceCollection(); - EXPECT_EQUAL(10u + 1 - 4 + 1, source_list->getSourceCount()); - EXPECT_EQUAL(0u, getSource(*source_list, docid + 2)); - EXPECT_EQUAL(3u, getSource(*source_list, docid + 6)); + sources = get_source_collection(); + EXPECT_EQ(10u + 1 - 4 + 1, sources->getSourceCount()); + EXPECT_EQ(0u, getSource(*sources, docid + 2)); + EXPECT_EQ(3u, getSource(*sources, docid + 6)); } -TEST_F("requireThatFlushTriggersFusion", Fixture) { +TEST_F(IndexManagerTest, require_that_flush_triggers_fusion) +{ const uint32_t fusion_trigger = 5; - f.resetIndexManager(); + resetIndexManager(); for (size_t i = 1; i <= fusion_trigger; ++i) { - f.addDocument(docid); - f.flushIndexManager(); + addDocument(docid); + flushIndexManager(); } - IFlushTarget::SP target(new IndexFusionTarget(f._index_manager->getMaintainer())); + IFlushTarget::SP target(new IndexFusionTarget(_index_manager->getMaintainer())); target->initFlush(0)->run(); - f.addDocument(docid); - f.flushIndexManager(); + addDocument(docid); + flushIndexManager(); set<uint32_t> fusion_ids = readDiskIds(index_dir, "fusion"); - EXPECT_EQUAL(1u, fusion_ids.size()); - EXPECT_EQUAL(5u, *fusion_ids.begin()); + EXPECT_EQ(1u, fusion_ids.size()); + EXPECT_EQ(5u, *fusion_ids.begin()); set<uint32_t> flush_ids = readDiskIds(index_dir, "flush"); - EXPECT_EQUAL(1u, flush_ids.size()); - EXPECT_EQUAL(6u, *flush_ids.begin()); + EXPECT_EQ(1u, flush_ids.size()); + EXPECT_EQ(6u, *flush_ids.begin()); } -TEST_F("requireThatFusionTargetIsSetUp", Fixture) { - f.addDocument(docid); - f.flushIndexManager(); - f.addDocument(docid); - f.flushIndexManager(); - IFlushTarget::List lst(f._index_manager->getFlushTargets()); - EXPECT_EQUAL(2u, lst.size()); +TEST_F(IndexManagerTest, require_that_fusion_target_is_setUp) +{ + addDocument(docid); + flushIndexManager(); + addDocument(docid); + flushIndexManager(); + IFlushTarget::List lst(_index_manager->getFlushTargets()); + EXPECT_EQ(2u, lst.size()); IFlushTarget::SP target(lst.at(1)); - EXPECT_EQUAL("memoryindex.fusion", target->getName()); + EXPECT_EQ("memoryindex.fusion", target->getName()); EXPECT_FALSE(target->needUrgentFlush()); - f.addDocument(docid); - f.flushIndexManager(); - lst = f._index_manager->getFlushTargets(); - EXPECT_EQUAL(2u, lst.size()); + addDocument(docid); + flushIndexManager(); + lst = _index_manager->getFlushTargets(); + EXPECT_EQ(2u, lst.size()); target = lst.at(1); - EXPECT_EQUAL("memoryindex.fusion", target->getName()); + EXPECT_EQ("memoryindex.fusion", target->getName()); EXPECT_TRUE(target->needUrgentFlush()); } -TEST_F("requireThatFusionCleansUpOldIndexes", Fixture) { - f.addDocument(docid); - f.flushIndexManager(); +TEST_F(IndexManagerTest, require_that_fusion_cleans_up_old_indexes) +{ + addDocument(docid); + flushIndexManager(); // hold reference to index.flush.1 - IIndexCollection::SP fsc = f._index_manager->getMaintainer().getSourceCollection(); + auto fsc = get_source_collection(); - f.addDocument(docid + 1); - f.flushIndexManager(); + addDocument(docid + 1); + flushIndexManager(); set<uint32_t> flush_ids = readDiskIds(index_dir, "flush"); - EXPECT_EQUAL(2u, flush_ids.size()); + EXPECT_EQ(2u, flush_ids.size()); FusionSpec fusion_spec; fusion_spec.flush_ids.push_back(1); fusion_spec.flush_ids.push_back(2); - f._index_manager->getMaintainer().runFusion(fusion_spec); + _index_manager->getMaintainer().runFusion(fusion_spec); flush_ids = readDiskIds(index_dir, "flush"); - EXPECT_EQUAL(1u, flush_ids.size()); - EXPECT_EQUAL(1u, *flush_ids.begin()); + EXPECT_EQ(1u, flush_ids.size()); + EXPECT_EQ(1u, *flush_ids.begin()); fsc.reset(); - f._index_manager->getMaintainer().removeOldDiskIndexes(); + _index_manager->getMaintainer().removeOldDiskIndexes(); flush_ids = readDiskIds(index_dir, "flush"); - EXPECT_EQUAL(0u, flush_ids.size()); + EXPECT_EQ(0u, flush_ids.size()); } bool contains(const IIndexCollection &fsc, uint32_t id) { @@ -549,118 +573,123 @@ bool indexExists(const string &type, uint32_t id) { return disk_ids.find(id) != disk_ids.end(); } -TEST_F("requireThatDiskIndexesAreLoadedOnStartup", Fixture) { - f.addDocument(docid); - f.flushIndexManager(); - f._index_manager.reset(0); +TEST_F(IndexManagerTest, require_that_disk_indexes_are_loaded_on_startup) +{ + addDocument(docid); + flushIndexManager(); + _index_manager.reset(0); ASSERT_TRUE(indexExists("flush", 1)); - f.resetIndexManager(); + resetIndexManager(); - IIndexCollection::SP fsc = f._index_manager->getMaintainer().getSourceCollection(); - EXPECT_EQUAL(2u, fsc->getSourceCount()); + auto fsc = get_source_collection(); + EXPECT_EQ(2u, fsc->getSourceCount()); EXPECT_TRUE(contains(*fsc, 1u)); EXPECT_TRUE(contains(*fsc, 2u)); - EXPECT_EQUAL(1u, getSource(*fsc, docid)); + EXPECT_EQ(1u, getSource(*fsc, docid)); fsc.reset(); - f.addDocument(docid + 1); - f.flushIndexManager(); + addDocument(docid + 1); + flushIndexManager(); ASSERT_TRUE(indexExists("flush", 2)); FusionSpec fusion_spec; fusion_spec.flush_ids.push_back(1); fusion_spec.flush_ids.push_back(2); - f._index_manager->getMaintainer().runFusion(fusion_spec); - f._index_manager.reset(0); + _index_manager->getMaintainer().runFusion(fusion_spec); + _index_manager.reset(0); ASSERT_TRUE(!indexExists("flush", 1)); ASSERT_TRUE(!indexExists("flush", 2)); ASSERT_TRUE(indexExists("fusion", 2)); - f.resetIndexManager(); + resetIndexManager(); - fsc = f._index_manager->getMaintainer().getSourceCollection(); - EXPECT_EQUAL(2u, fsc->getSourceCount()); + fsc = get_source_collection(); + EXPECT_EQ(2u, fsc->getSourceCount()); EXPECT_TRUE(contains(*fsc, 0u)); EXPECT_TRUE(contains(*fsc, 1u)); - EXPECT_EQUAL(0u, getSource(*fsc, docid)); - EXPECT_EQUAL(0u, getSource(*fsc, docid + 1)); + EXPECT_EQ(0u, getSource(*fsc, docid)); + EXPECT_EQ(0u, getSource(*fsc, docid + 1)); /// Must account for both docid 0 being reserved and the extra after. - EXPECT_EQUAL(docid + 2, fsc->getSourceSelector().getDocIdLimit()); + EXPECT_EQ(docid + 2, fsc->getSourceSelector().getDocIdLimit()); fsc.reset(); - f.addDocument(docid + 2); - f.flushIndexManager(); - f._index_manager.reset(0); + addDocument(docid + 2); + flushIndexManager(); + _index_manager.reset(0); ASSERT_TRUE(indexExists("fusion", 2)); ASSERT_TRUE(indexExists("flush", 3)); - f.resetIndexManager(); + resetIndexManager(); - fsc = f._index_manager->getMaintainer().getSourceCollection(); - EXPECT_EQUAL(3u, fsc->getSourceCount()); + fsc = get_source_collection(); + EXPECT_EQ(3u, fsc->getSourceCount()); EXPECT_TRUE(contains(*fsc, 0u)); EXPECT_TRUE(contains(*fsc, 1u)); EXPECT_TRUE(contains(*fsc, 2u)); - EXPECT_EQUAL(0u, getSource(*fsc, docid)); - EXPECT_EQUAL(0u, getSource(*fsc, docid + 1)); - EXPECT_EQUAL(1u, getSource(*fsc, docid + 2)); + EXPECT_EQ(0u, getSource(*fsc, docid)); + EXPECT_EQ(0u, getSource(*fsc, docid + 1)); + EXPECT_EQ(1u, getSource(*fsc, docid + 2)); fsc.reset(); } -TEST_F("requireThatExistingIndexesAreToBeFusionedOnStartup", Fixture) { - f.addDocument(docid); - f.flushIndexManager(); - f.addDocument(docid + 1); - f.flushIndexManager(); - f.resetIndexManager(); +TEST_F(IndexManagerTest, require_that_existing_indexes_are_to_be_fusioned_on_startup) +{ + addDocument(docid); + flushIndexManager(); + addDocument(docid + 1); + flushIndexManager(); + resetIndexManager(); - IFlushTarget::SP target(new IndexFusionTarget(f._index_manager->getMaintainer())); + IFlushTarget::SP target(new IndexFusionTarget(_index_manager->getMaintainer())); target->initFlush(0)->run(); - f.addDocument(docid); - f.flushIndexManager(); + addDocument(docid); + flushIndexManager(); set<uint32_t> fusion_ids = readDiskIds(index_dir, "fusion"); - EXPECT_EQUAL(1u, fusion_ids.size()); - EXPECT_EQUAL(2u, *fusion_ids.begin()); + EXPECT_EQ(1u, fusion_ids.size()); + EXPECT_EQ(2u, *fusion_ids.begin()); } -TEST_F("requireThatSerialNumberIsWrittenOnFlush", Fixture) { - f.addDocument(docid); - f.flushIndexManager(); +TEST_F(IndexManagerTest, require_that_serial_number_is_written_on_flush) +{ + addDocument(docid); + flushIndexManager(); FastOS_File file((index_dir + "/index.flush.1/serial.dat").c_str()); EXPECT_TRUE(file.OpenReadOnly()); } -TEST_F("requireThatSerialNumberIsCopiedOnFusion", Fixture) { - f.addDocument(docid); - f.flushIndexManager(); - f.addDocument(docid); - f.flushIndexManager(); +TEST_F(IndexManagerTest, require_that_serial_number_is_copied_on_fusion) +{ + addDocument(docid); + flushIndexManager(); + addDocument(docid); + flushIndexManager(); FusionSpec fusion_spec; fusion_spec.flush_ids.push_back(1); fusion_spec.flush_ids.push_back(2); - f._index_manager->getMaintainer().runFusion(fusion_spec); + _index_manager->getMaintainer().runFusion(fusion_spec); FastOS_File file((index_dir + "/index.fusion.2/serial.dat").c_str()); EXPECT_TRUE(file.OpenReadOnly()); } -TEST_F("requireThatSerialNumberIsReadOnLoad", Fixture) { - f.addDocument(docid); - f.flushIndexManager(); - EXPECT_EQUAL(f._serial_num, f._index_manager->getFlushedSerialNum()); - f.resetIndexManager(); - EXPECT_EQUAL(f._serial_num, f._index_manager->getFlushedSerialNum()); - - f.addDocument(docid); - f.flushIndexManager(); - f.addDocument(docid); - f.flushIndexManager(); - search::SerialNum serial = f._serial_num; - f.addDocument(docid); - f.resetIndexManager(); - EXPECT_EQUAL(serial, f._index_manager->getFlushedSerialNum()); +TEST_F(IndexManagerTest, require_that_serial_number_is_read_on_load) +{ + addDocument(docid); + flushIndexManager(); + EXPECT_EQ(_serial_num, _index_manager->getFlushedSerialNum()); + resetIndexManager(); + EXPECT_EQ(_serial_num, _index_manager->getFlushedSerialNum()); + + addDocument(docid); + flushIndexManager(); + addDocument(docid); + flushIndexManager(); + search::SerialNum serial = _serial_num; + addDocument(docid); + resetIndexManager(); + EXPECT_EQ(serial, _index_manager->getFlushedSerialNum()); } void crippleFusion(uint32_t fusionId) { @@ -669,27 +698,28 @@ void crippleFusion(uint32_t fusionId) { FastOS_File(ost.str().data()).Delete(); } -TEST_F("requireThatFailedFusionIsRetried", Fixture) { - f.resetIndexManager(); +TEST_F(IndexManagerTest, require_that_failed_fusion_is_retried) +{ + resetIndexManager(); - f.addDocument(docid); - f.flushIndexManager(); - f.addDocument(docid); - f.flushIndexManager(); + addDocument(docid); + flushIndexManager(); + addDocument(docid); + flushIndexManager(); crippleFusion(2); - IndexFusionTarget target(f._index_manager->getMaintainer()); + IndexFusionTarget target(_index_manager->getMaintainer()); vespalib::Executor::Task::UP fusionTask = target.initFlush(1); fusionTask->run(); - FusionSpec spec = f._index_manager->getMaintainer().getFusionSpec(); + FusionSpec spec = _index_manager->getMaintainer().getFusionSpec(); set<uint32_t> fusion_ids = readDiskIds(index_dir, "fusion"); EXPECT_TRUE(fusion_ids.empty()); - EXPECT_EQUAL(0u, spec.last_fusion_id); - EXPECT_EQUAL(2u, spec.flush_ids.size()); - EXPECT_EQUAL(1u, spec.flush_ids[0]); - EXPECT_EQUAL(2u, spec.flush_ids[1]); + EXPECT_EQ(0u, spec.last_fusion_id); + EXPECT_EQ(2u, spec.flush_ids.size()); + EXPECT_EQ(1u, spec.flush_ids[0]); + EXPECT_EQ(2u, spec.flush_ids[1]); } namespace { @@ -697,49 +727,116 @@ namespace { void expectSchemaIndexFields(uint32_t expIndexFields) { Schema s; s.loadFromFile("test_data/index.flush.1/schema.txt"); - EXPECT_EQUAL(expIndexFields, s.getNumIndexFields()); + EXPECT_EQ(expIndexFields, s.getNumIndexFields()); } } -TEST_F("require that setSchema updates schema on disk, wiping removed fields", Fixture) +TEST_F(IndexManagerTest, require_that_setSchema_updates_schema_on_disk_wiping_removed_fields) { Schema empty_schema; - f.addDocument(docid); - f.flushIndexManager(); - TEST_DO(expectSchemaIndexFields(1)); - f.runAsMaster([&]() { f._index_manager->setSchema(empty_schema, ++f._serial_num); }); - TEST_DO(expectSchemaIndexFields(0)); + addDocument(docid); + flushIndexManager(); + expectSchemaIndexFields(1); + runAsMaster([&]() { _index_manager->setSchema(empty_schema, ++_serial_num); }); + expectSchemaIndexFields(0); } -TEST_F("require that indexes manager stats can be generated", Fixture) +TEST_F(IndexManagerTest, require_that_indexes_manager_stats_can_be_generated) { - TEST_DO(f.assertStats(0, 1, 0, 0)); - f.addDocument(1); - TEST_DO(f.assertStats(0, 1, 0, 1)); - f.flushIndexManager(); - TEST_DO(f.assertStats(1, 1, 1, 1)); - f.addDocument(2); - TEST_DO(f.assertStats(1, 1, 1, 2)); + assertStats(0, 1, 0, 0); + addDocument(1); + assertStats(0, 1, 0, 1); + flushIndexManager(); + assertStats(1, 1, 1, 1); + addDocument(2); + assertStats(1, 1, 1, 2); } -TEST_F("require that compactLidSpace works", Fixture) +TEST_F(IndexManagerTest, require_that_compact_lid_space_works) { Schema empty_schema; - f.addDocument(1); - f.addDocument(2); - f.removeDocument(2); - auto fsc = f._index_manager->getMaintainer().getSourceCollection(); - EXPECT_EQUAL(3u, fsc->getSourceSelector().getDocIdLimit()); - f.compactLidSpace(2); - EXPECT_EQUAL(2u, fsc->getSourceSelector().getDocIdLimit()); + addDocument(1); + addDocument(2); + removeDocument(2); + auto fsc = get_source_collection(); + EXPECT_EQ(3u, fsc->getSourceSelector().getDocIdLimit()); + compactLidSpace(2); + EXPECT_EQ(2u, fsc->getSourceSelector().getDocIdLimit()); +} + +template <typename IndexType> +IndexType* +as_index_type(const IIndexCollection& col, uint32_t source_id) +{ + auto& searchable = col.getSearchable(source_id); + auto* result = dynamic_cast<IndexType *>(&searchable); + assert(result != nullptr); + return result; +} + +IMemoryIndex* +as_memory_index(const IIndexCollection& col, uint32_t source_id) +{ + return as_index_type<IMemoryIndex>(col, source_id); +} + +IDiskIndex* +as_disk_index(const IIndexCollection& col, uint32_t source_id) +{ + return as_index_type<IDiskIndex>(col, source_id); +} + +void +expect_field_length_info(double exp_average, uint32_t exp_samples, const IndexSearchable& searchable) +{ + auto info = searchable.get_field_length_info(field_name); + EXPECT_DOUBLE_EQ(exp_average, info.get_average_field_length()); + EXPECT_EQ(exp_samples, info.get_num_samples()); +} + +TEST_F(IndexManagerTest, field_length_info_is_propagated_to_disk_index_and_next_memory_index_during_flush) +{ + addDocument(1); + addDocument(2); + + auto sources = get_source_collection(); + ASSERT_EQ(1, sources->getSourceCount()); + auto *first_index = as_memory_index(*sources, 0); + expect_field_length_info(1, 2, *first_index); + + flushIndexManager(); + + sources = get_source_collection(); + ASSERT_EQ(2, sources->getSourceCount()); + expect_field_length_info(1, 2, *as_disk_index(*sources, 0)); + auto *second_index = as_memory_index(*sources, 1); + EXPECT_NE(first_index, second_index); + expect_field_length_info(1, 2, *second_index); +} + +TEST_F(IndexManagerTest, field_length_info_is_loaded_from_disk_index_during_startup) +{ + addDocument(1); + addDocument(2); + flushIndexManager(); + resetIndexManager(); + + auto sources = get_source_collection(); + ASSERT_EQ(2, sources->getSourceCount()); + expect_field_length_info(1, 2, *as_disk_index(*sources, 0)); + expect_field_length_info(1, 2, *as_memory_index(*sources, 1)); } } // namespace -TEST_MAIN() { - TEST_DO(removeTestData()); +int +main(int argc, char* argv[]) +{ + removeTestData(); DummyFileHeaderContext::setCreator("indexmanager_test"); - TEST_RUN_ALL(); - TEST_DO(removeTestData()); + ::testing::InitGoogleTest(&argc, argv); + int result = RUN_ALL_TESTS(); + removeTestData(); + return result; } diff --git a/searchcore/src/vespa/searchcore/proton/index/diskindexwrapper.cpp b/searchcore/src/vespa/searchcore/proton/index/diskindexwrapper.cpp index d70c321722b..4cfaac9aac0 100644 --- a/searchcore/src/vespa/searchcore/proton/index/diskindexwrapper.cpp +++ b/searchcore/src/vespa/searchcore/proton/index/diskindexwrapper.cpp @@ -49,9 +49,7 @@ DiskIndexWrapper::accept(searchcorespi::IndexSearchableVisitor &visitor) const FieldLengthInfo DiskIndexWrapper::get_field_length_info(const vespalib::string& field_name) const { - // TODO: implement - (void) field_name; - return FieldLengthInfo(); + return _index.get_field_length_info(field_name); } } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/matching/blueprintbuilder.cpp b/searchcore/src/vespa/searchcore/proton/matching/blueprintbuilder.cpp index 268fe63ba4c..672e7f78784 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/blueprintbuilder.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/blueprintbuilder.cpp @@ -4,6 +4,7 @@ #include "blueprintbuilder.h" #include "termdatafromnode.h" #include "same_element_builder.h" +#include <vespa/searchcorespi/index/indexsearchable.h> #include <vespa/searchlib/query/tree/customtypevisitor.h> #include <vespa/searchlib/queryeval/leaf_blueprints.h> #include <vespa/searchlib/queryeval/intermediate_blueprints.h> diff --git a/searchcore/src/vespa/searchcore/proton/matching/fakesearchcontext.h b/searchcore/src/vespa/searchcore/proton/matching/fakesearchcontext.h index 02aedf15d6e..fe9c20112f4 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/fakesearchcontext.h +++ b/searchcore/src/vespa/searchcore/proton/matching/fakesearchcontext.h @@ -52,7 +52,7 @@ public: search::queryeval::ISourceSelector &selector() { return *_selector; } // Implements ISearchContext - search::queryeval::Searchable &getIndexes() override { + IndexSearchable &getIndexes() override { return *_indexes; } diff --git a/searchcore/src/vespa/searchcore/proton/matching/isearchcontext.h b/searchcore/src/vespa/searchcore/proton/matching/isearchcontext.h index 2965e3796bf..dc840dc79ff 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/isearchcontext.h +++ b/searchcore/src/vespa/searchcore/proton/matching/isearchcontext.h @@ -6,6 +6,8 @@ #include <memory> +namespace searchcorespi { class IndexSearchable; } + namespace proton::matching { /** @@ -31,13 +33,14 @@ public: ISearchContext & operator = (const ISearchContext &) = delete; typedef search::queryeval::Searchable Searchable; + using IndexSearchable = searchcorespi::IndexSearchable; /** * Obtain the index fields searchable. * * @return index fields searchable. **/ - virtual Searchable &getIndexes() = 0; + virtual IndexSearchable &getIndexes() = 0; /** * Obtain the attribute fields searchable. diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp b/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp index 4a944dc3214..5d1e2212c83 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp @@ -9,6 +9,7 @@ #include <vespa/log/log.h> LOG_SETUP(".proton.matching.match_tools"); #include <vespa/searchlib/query/tree/querytreecreator.h> +#include <vespa/searchcorespi/index/indexsearchable.h> using search::attribute::IAttributeContext; using search::queryeval::IRequestContext; @@ -158,7 +159,7 @@ MatchToolsFactory(QueryLimiter & queryLimiter, _hardDoom(hardDoom), _query(), _match_limiter(), - _queryEnv(indexEnv, attributeContext, rankProperties), + _queryEnv(indexEnv, attributeContext, rankProperties, searchContext.getIndexes()), _mdl(), _rankSetup(rankSetup), _featureOverrides(featureOverrides), diff --git a/searchcore/src/vespa/searchcore/proton/matching/queryenvironment.cpp b/searchcore/src/vespa/searchcore/proton/matching/queryenvironment.cpp index d4320c87ab2..ec48ee7164b 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/queryenvironment.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/queryenvironment.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "queryenvironment.h" +#include <vespa/searchlib/index/i_field_length_inspector.h> using search::attribute::IAttributeContext; using search::fef::IIndexEnvironment; @@ -11,12 +12,14 @@ namespace proton::matching { QueryEnvironment::QueryEnvironment(const IIndexEnvironment &indexEnv, const IAttributeContext &attrContext, - const Properties &properties) + const Properties &properties, + const search::index::IFieldLengthInspector &field_length_inspector) : _indexEnv(indexEnv), _attrContext(attrContext), _properties(properties), _locations(1), - _terms() + _terms(), + _field_length_inspector(field_length_inspector) { } @@ -53,6 +56,12 @@ QueryEnvironment::getAttributeContext() const return _attrContext; } +double +QueryEnvironment::get_average_field_length(const vespalib::string &field_name) const +{ + return _field_length_inspector.get_field_length_info(field_name).get_average_field_length(); +} + const search::fef::IIndexEnvironment & QueryEnvironment::getIndexEnvironment() const { diff --git a/searchcore/src/vespa/searchcore/proton/matching/queryenvironment.h b/searchcore/src/vespa/searchcore/proton/matching/queryenvironment.h index d79ba1796f7..8f958870d52 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/queryenvironment.h +++ b/searchcore/src/vespa/searchcore/proton/matching/queryenvironment.h @@ -6,6 +6,8 @@ #include <vespa/searchlib/fef/properties.h> #include <vespa/searchlib/fef/location.h> +namespace search::index { class IFieldLengthInspector; } + namespace proton::matching { /** @@ -19,6 +21,7 @@ private: search::fef::Properties _properties; std::vector<const search::fef::Location *> _locations; std::vector<const search::fef::ITermData *> _terms; + const search::index::IFieldLengthInspector &_field_length_inspector; QueryEnvironment(const QueryEnvironment &); QueryEnvironment &operator=(const QueryEnvironment &); @@ -33,7 +36,8 @@ public: **/ QueryEnvironment(const search::fef::IIndexEnvironment &indexEnv, const search::attribute::IAttributeContext &attrContext, - const search::fef::Properties &properties); + const search::fef::Properties &properties, + const search::index::IFieldLengthInspector &field_length_inspector); /** * Used to edit the list of terms by the one setting up this query @@ -71,6 +75,8 @@ public: // inherited from search::fef::IQueryEnvironment const search::attribute::IAttributeContext & getAttributeContext() const override; + double get_average_field_length(const vespalib::string &field_name) const override; + // inherited from search::fef::IQueryEnvironment const search::fef::IIndexEnvironment & getIndexEnvironment() const override; diff --git a/searchcore/src/vespa/searchcore/proton/matching/same_element_builder.cpp b/searchcore/src/vespa/searchcore/proton/matching/same_element_builder.cpp index d3a0ec4726f..16c86e8a4f5 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/same_element_builder.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/same_element_builder.cpp @@ -4,6 +4,7 @@ #include "querynodes.h" #include <vespa/searchlib/query/tree/customtypevisitor.h> #include <vespa/searchlib/queryeval/leaf_blueprints.h> +#include <vespa/searchcorespi/index/indexsearchable.h> using search::queryeval::Blueprint; using search::queryeval::EmptyBlueprint; diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp index d53adf19f76..9445a0a5206 100644 --- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp @@ -2,9 +2,10 @@ #include "disk_mem_usage_filter.h" #include "i_disk_mem_usage_listener.h" -#include <vespa/log/log.h> #include <vespa/searchcore/proton/common/hw_info.h> +#include <sstream> +#include <vespa/log/log.h> LOG_SETUP(".proton.server.disk_mem_usage_filter"); namespace proton { diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp index 772be9049db..06f19eb06cc 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp @@ -37,6 +37,7 @@ #include <vespa/searchlib/aggregation/forcelink.hpp> #include <vespa/searchlib/expression/forcelink.hpp> +#include <sstream> #include <vespa/log/log.h> LOG_SETUP(".proton.server.proton"); diff --git a/searchcore/src/vespa/searchcore/proton/server/searchcontext.cpp b/searchcore/src/vespa/searchcore/proton/server/searchcontext.cpp index ea09c60bd52..d9207ef70e1 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchcontext.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/searchcontext.cpp @@ -3,10 +3,11 @@ #include "searchcontext.h" using search::queryeval::Searchable; +using searchcorespi::IndexSearchable; namespace proton { -Searchable & +IndexSearchable & SearchContext::getIndexes() { return *_indexSearchable; @@ -23,7 +24,7 @@ uint32_t SearchContext::getDocIdLimit() return _docIdLimit; } -SearchContext::SearchContext(const Searchable::SP &indexSearchable, uint32_t docIdLimit) +SearchContext::SearchContext(const std::shared_ptr<IndexSearchable> &indexSearchable, uint32_t docIdLimit) : _indexSearchable(indexSearchable), _attributeBlueprintFactory(), _docIdLimit(docIdLimit) diff --git a/searchcore/src/vespa/searchcore/proton/server/searchcontext.h b/searchcore/src/vespa/searchcore/proton/server/searchcontext.h index 71475c3decd..b9ea6b334b3 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchcontext.h +++ b/searchcore/src/vespa/searchcore/proton/server/searchcontext.h @@ -16,16 +16,16 @@ class SearchContext : public matching::ISearchContext { private: /// Snapshot of the indexes used. - Searchable::SP _indexSearchable; + std::shared_ptr<IndexSearchable> _indexSearchable; search::AttributeBlueprintFactory _attributeBlueprintFactory; uint32_t _docIdLimit; - Searchable &getIndexes() override; + IndexSearchable &getIndexes() override; Searchable &getAttributes() override; uint32_t getDocIdLimit() override; public: - SearchContext(const Searchable::SP &indexSearchable, uint32_t docIdLimit); + SearchContext(const std::shared_ptr<IndexSearchable> &indexSearchable, uint32_t docIdLimit); }; } // namespace proton diff --git a/searchlib/src/tests/common/bitvector/bitvector_test.cpp b/searchlib/src/tests/common/bitvector/bitvector_test.cpp index 22a47952acb..e61c21bee1c 100644 --- a/searchlib/src/tests/common/bitvector/bitvector_test.cpp +++ b/searchlib/src/tests/common/bitvector/bitvector_test.cpp @@ -548,38 +548,39 @@ TEST("requireThatGrowWorks") v.setBit(103); EXPECT_EQUAL(200u, v.size()); + EXPECT_EQUAL(1023u, v.capacity()); v.invalidateCachedCount(); EXPECT_TRUE(assertBV("[7,39,71,103]", v)); EXPECT_EQUAL(4u, v.countTrueBits()); - EXPECT_TRUE(v.reserve(204)); + EXPECT_TRUE(v.reserve(1024)); EXPECT_EQUAL(200u, v.size()); - EXPECT_EQUAL(204u, v.capacity()); + EXPECT_EQUAL(2047u, v.capacity()); EXPECT_TRUE(assertBV("[7,39,71,103]", v)); EXPECT_EQUAL(4u, v.countTrueBits()); EXPECT_FALSE(v.extend(202)); EXPECT_EQUAL(202u, v.size()); - EXPECT_EQUAL(204u, v.capacity()); + EXPECT_EQUAL(2047u, v.capacity()); EXPECT_TRUE(assertBV("[7,39,71,103]", v)); EXPECT_EQUAL(4u, v.countTrueBits()); EXPECT_FALSE(v.shrink(200)); EXPECT_EQUAL(200u, v.size()); - EXPECT_EQUAL(204u, v.capacity()); + EXPECT_EQUAL(2047u, v.capacity()); EXPECT_TRUE(assertBV("[7,39,71,103]", v)); EXPECT_EQUAL(4u, v.countTrueBits()); - EXPECT_FALSE(v.reserve(204)); + EXPECT_FALSE(v.reserve(2047)); EXPECT_EQUAL(200u, v.size()); - EXPECT_EQUAL(204u, v.capacity()); + EXPECT_EQUAL(2047u, v.capacity()); EXPECT_TRUE(assertBV("[7,39,71,103]", v)); EXPECT_EQUAL(4u, v.countTrueBits()); EXPECT_FALSE(v.shrink(202)); EXPECT_EQUAL(202u, v.size()); - EXPECT_EQUAL(204u, v.capacity()); + EXPECT_EQUAL(2047u, v.capacity()); EXPECT_TRUE(assertBV("[7,39,71,103]", v)); EXPECT_EQUAL(4u, v.countTrueBits()); EXPECT_FALSE(v.shrink(100)); EXPECT_EQUAL(100u, v.size()); - EXPECT_EQUAL(204u, v.capacity()); + EXPECT_EQUAL(2047u, v.capacity()); EXPECT_TRUE(assertBV("[7,39,71]", v)); EXPECT_EQUAL(3u, v.countTrueBits()); g.transferHoldLists(1); diff --git a/searchlib/src/tests/features/bm25/bm25_test.cpp b/searchlib/src/tests/features/bm25/bm25_test.cpp index 84bafcfa0ed..1a3895d7e28 100644 --- a/searchlib/src/tests/features/bm25/bm25_test.cpp +++ b/searchlib/src/tests/features/bm25/bm25_test.cpp @@ -11,6 +11,7 @@ using namespace search::features; using namespace search::fef; +using namespace search::fef::objectstore; using CollectionType = FieldInfo::CollectionType; using StringVector = std::vector<vespalib::string>; @@ -40,12 +41,13 @@ struct Bm25BlueprintTest : public ::testing::Test { EXPECT_FALSE(blueprint->setup(index_env, params)); } - void expect_setup_succeed(const StringVector& params) { + Blueprint::SP expect_setup_succeed(const StringVector& params) { auto blueprint = make_blueprint(); test::DummyDependencyHandler deps(*blueprint); EXPECT_TRUE(blueprint->setup(index_env, params)); EXPECT_EQ(0, deps.input.size()); EXPECT_EQ(StringVector({"score"}), deps.output); + return blueprint; } }; @@ -70,6 +72,15 @@ TEST_F(Bm25BlueprintTest, blueprint_setup_succeeds_for_index_field) expect_setup_succeed({"iws"}); } +TEST_F(Bm25BlueprintTest, blueprint_can_prepare_shared_state_with_average_field_length) +{ + auto blueprint = expect_setup_succeed({"is"}); + test::QueryEnvironment query_env; + query_env.get_avg_field_lengths()["is"] = 10; + ObjectStore store; + blueprint->prepareSharedState(query_env, store); + EXPECT_DOUBLE_EQ(10, as_value<double>(*store.get("bm25.afl.is"))); +} struct Bm25ExecutorTest : public ::testing::Test { BlueprintFactory factory; @@ -87,9 +98,10 @@ struct Bm25ExecutorTest : public ::testing::Test { test.getQueryEnv().getBuilder().addIndexNode({"foo"}); test.getQueryEnv().getBuilder().addIndexNode({"foo"}); test.getQueryEnv().getBuilder().addIndexNode({"bar"}); - + test.getQueryEnv().getBuilder().set_avg_field_length("foo", 10); + } + void setup() { EXPECT_TRUE(test.setup()); - match_data = test.createMatchDataBuilder(); clear_term(0, 0); clear_term(1, 0); @@ -111,19 +123,21 @@ struct Bm25ExecutorTest : public ::testing::Test { tfmd->setFieldLength(field_length); } - feature_t get_score(feature_t num_occs, feature_t field_length) const { - return (num_occs * 2.2) / (num_occs + (1.2 * (0.25 + 0.75 * field_length / 10.0))); + feature_t get_score(feature_t num_occs, feature_t field_length, double avg_field_length = 10) const { + return (num_occs * 2.2) / (num_occs + (1.2 * (0.25 + 0.75 * field_length / avg_field_length))); } }; TEST_F(Bm25ExecutorTest, score_is_calculated_for_a_single_term) { + setup(); prepare_term(0, 0, 3, 20); EXPECT_TRUE(execute(get_score(3.0, 20))); } TEST_F(Bm25ExecutorTest, score_is_calculated_for_multiple_terms) { + setup(); prepare_term(0, 0, 3, 20); prepare_term(1, 0, 7, 5); EXPECT_TRUE(execute(get_score(3.0, 20) + get_score(7.0, 5.0))); @@ -131,6 +145,7 @@ TEST_F(Bm25ExecutorTest, score_is_calculated_for_multiple_terms) TEST_F(Bm25ExecutorTest, term_that_does_not_match_document_is_ignored) { + setup(); prepare_term(0, 0, 3, 20); prepare_term(1, 0, 7, 5, 123); EXPECT_TRUE(execute(get_score(3.0, 20))); @@ -138,8 +153,17 @@ TEST_F(Bm25ExecutorTest, term_that_does_not_match_document_is_ignored) TEST_F(Bm25ExecutorTest, term_searching_another_field_is_ignored) { + setup(); prepare_term(2, 1, 3, 20); EXPECT_TRUE(execute(0.0)); } +TEST_F(Bm25ExecutorTest, uses_average_field_length_from_shared_state_if_found) +{ + test.getQueryEnv().getObjectStore().add("bm25.afl.foo", std::make_unique<AnyWrapper<double>>(15)); + setup(); + prepare_term(0, 0, 3, 20); + EXPECT_TRUE(execute(get_score(3.0, 20, 15))); +} + GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/vespa/searchlib/attribute/singleboolattribute.cpp b/searchlib/src/vespa/searchlib/attribute/singleboolattribute.cpp index 3e260e8453a..75a0e4b8c71 100644 --- a/searchlib/src/vespa/searchlib/attribute/singleboolattribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/singleboolattribute.cpp @@ -25,24 +25,28 @@ SingleBoolAttribute::~SingleBoolAttribute() getGenerationHolder().clearHoldLists(); } -bool -SingleBoolAttribute::addDoc(DocId & doc) { - size_t needSize = getNumDocs() + 1; - bool incGen = false; - if (_bv.capacity() < needSize) { +void +SingleBoolAttribute::ensureRoom(DocId docIdLimit) { + if (_bv.capacity() < docIdLimit) { const GrowStrategy & gs = this->getConfig().getGrowStrategy(); - uint32_t newSize = needSize + (needSize * gs.getDocsGrowFactor()) + gs.getDocsGrowDelta(); - incGen = _bv.reserve(newSize); + uint32_t newSize = docIdLimit + (docIdLimit * gs.getDocsGrowFactor()) + gs.getDocsGrowDelta(); + bool incGen = _bv.reserve(newSize); + if (incGen) { + incGeneration(); + } } - incGen = _bv.extend(needSize) || incGen; +} + +bool +SingleBoolAttribute::addDoc(DocId & doc) { + DocId docIdLimit = getNumDocs()+1; + ensureRoom(docIdLimit); + bool incGen = _bv.extend(docIdLimit); + assert( ! incGen); incNumDocs(); doc = getNumDocs() - 1; updateUncommittedDocIdLimit(doc); - if (incGen) { - incGeneration(); - } else { - removeAllOldGenerations(); - } + removeAllOldGenerations(); return true; } @@ -85,7 +89,7 @@ SingleBoolAttribute::onCommit() { void SingleBoolAttribute::onAddDocs(DocId docIdLimit) { - _bv.reserve(docIdLimit); + ensureRoom(docIdLimit); } void diff --git a/searchlib/src/vespa/searchlib/attribute/singleboolattribute.h b/searchlib/src/vespa/searchlib/attribute/singleboolattribute.h index 789948838cb..20ec0b6d077 100644 --- a/searchlib/src/vespa/searchlib/attribute/singleboolattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/singleboolattribute.h @@ -101,6 +101,7 @@ protected: return false; } private: + void ensureRoom(DocId docIdLimit); int8_t getFromEnum(EnumHandle) const override { return 0; } diff --git a/searchlib/src/vespa/searchlib/common/allocatedbitvector.cpp b/searchlib/src/vespa/searchlib/common/allocatedbitvector.cpp index dcee48aed1a..3de0c1f1320 100644 --- a/searchlib/src/vespa/searchlib/common/allocatedbitvector.cpp +++ b/searchlib/src/vespa/searchlib/common/allocatedbitvector.cpp @@ -9,14 +9,22 @@ using vespalib::GenerationHeldBase; using vespalib::GenerationHeldAlloc; using vespalib::GenerationHolder; -////////////////////////////////////////////////////////////////////// -// Parameterized Constructor -////////////////////////////////////////////////////////////////////// +namespace { + +size_t computeCapacity(size_t capacity, size_t allocatedBytes) { + size_t possibleCapacity = (allocatedBytes * 8) - 1; + assert(possibleCapacity >= capacity); + return possibleCapacity; +} + +} + AllocatedBitVector::AllocatedBitVector(Index numberOfElements) : BitVector(), _capacityBits(numberOfElements), _alloc(allocatePaddedAndAligned(numberOfElements)) { + _capacityBits = computeCapacity(_capacityBits, _alloc.size()); init(_alloc.get(), 0, numberOfElements); clear(); } @@ -33,6 +41,7 @@ AllocatedBitVector::AllocatedBitVector(Index numberOfElements, Index capacityBit _capacityBits(capacityBits), _alloc(allocatePaddedAndAligned(0, numberOfElements, capacityBits)) { + _capacityBits = computeCapacity(_capacityBits, _alloc.size()); init(_alloc.get(), 0, numberOfElements); clear(); if (rhsSize > 0) { @@ -58,6 +67,7 @@ AllocatedBitVector::AllocatedBitVector(const BitVector & rhs, Index capacity_) : _capacityBits(capacity_), _alloc(allocatePaddedAndAligned(0, rhs.size(), capacity_)) { + _capacityBits = computeCapacity(_capacityBits, _alloc.size()); memcpy(_alloc.get(), rhs.getStart(), rhs.sizeBytes()); init(_alloc.get(), 0, rhs.size()); } @@ -65,7 +75,7 @@ AllocatedBitVector::AllocatedBitVector(const BitVector & rhs, Index capacity_) : ////////////////////////////////////////////////////////////////////// // Destructor ////////////////////////////////////////////////////////////////////// -AllocatedBitVector::~AllocatedBitVector() { } +AllocatedBitVector::~AllocatedBitVector() = default; void AllocatedBitVector::cleanup() @@ -78,8 +88,8 @@ AllocatedBitVector::cleanup() void AllocatedBitVector::resize(Index newLength) { - _capacityBits = newLength; - _alloc = allocatePaddedAndAligned(_capacityBits); + _alloc = allocatePaddedAndAligned(newLength); + _capacityBits = computeCapacity(newLength, _alloc.size()); init(_alloc.get(), 0, newLength); clear(); } diff --git a/searchlib/src/vespa/searchlib/common/allocatedbitvector.h b/searchlib/src/vespa/searchlib/common/allocatedbitvector.h index 6de255c48c9..c52c52354a1 100644 --- a/searchlib/src/vespa/searchlib/common/allocatedbitvector.h +++ b/searchlib/src/vespa/searchlib/common/allocatedbitvector.h @@ -32,7 +32,7 @@ public: AllocatedBitVector(Index numberOfElements, Alloc buffer, size_t offset); /** - * Creates a new bitvector with room for numberOfElements bits. + * Creates a new bitvector with size of numberOfElements bits and at least a capacity of capacity. * Copies what it can from the original vector. This is used for extending vector. */ AllocatedBitVector(Index numberOfElements, Index capacity, const void * rhsBuf, size_t rhsSize); diff --git a/searchlib/src/vespa/searchlib/common/bitvector.cpp b/searchlib/src/vespa/searchlib/common/bitvector.cpp index 2c45bc8f69a..7296842f2c1 100644 --- a/searchlib/src/vespa/searchlib/common/bitvector.cpp +++ b/searchlib/src/vespa/searchlib/common/bitvector.cpp @@ -52,7 +52,7 @@ BitVector::allocatePaddedAndAligned(Index start, Index end, Index capacity) assert(alloc.size()/sizeof(Word) >= words); // Clear padding size_t usedBytes = numBytes(end - start); - memset(static_cast<char *>(alloc.get()) + usedBytes, 0, sz - usedBytes); + memset(static_cast<char *>(alloc.get()) + usedBytes, 0, alloc.size() - usedBytes); return alloc; } diff --git a/searchlib/src/vespa/searchlib/common/growablebitvector.h b/searchlib/src/vespa/searchlib/common/growablebitvector.h index ff5d878063d..e13e3b42e3d 100644 --- a/searchlib/src/vespa/searchlib/common/growablebitvector.h +++ b/searchlib/src/vespa/searchlib/common/growablebitvector.h @@ -9,8 +9,7 @@ namespace search { class GrowableBitVector : public AllocatedBitVector { public: - GrowableBitVector(Index newSize, Index newCapacity, - GenerationHolder &generationHolder); + GrowableBitVector(Index newSize, Index newCapacity, GenerationHolder &generationHolder); /** Will return true if a a buffer is held */ bool reserve(Index newCapacity); diff --git a/searchlib/src/vespa/searchlib/common/partialbitvector.cpp b/searchlib/src/vespa/searchlib/common/partialbitvector.cpp index e57396c0dfa..e1dc144541e 100644 --- a/searchlib/src/vespa/searchlib/common/partialbitvector.cpp +++ b/searchlib/src/vespa/searchlib/common/partialbitvector.cpp @@ -29,6 +29,6 @@ PartialBitVector::PartialBitVector(const BitVector & org, Index start, Index end setBit(size()); } -PartialBitVector::~PartialBitVector() { } +PartialBitVector::~PartialBitVector() = default; } // namespace search diff --git a/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp b/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp index f17f9459ff9..46fcdafc585 100644 --- a/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp +++ b/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp @@ -91,7 +91,7 @@ void LogDataStore::updateSerialNum() { LockGuard guard(_updateLock); - if (getPrevActive(guard) != NULL) { + if (getPrevActive(guard) != nullptr) { if (getActive(guard).getSerialNum() < getPrevActive(guard)->getLastPersistedSerialNum()) { getActive(guard).setSerialNum(getPrevActive(guard)->getLastPersistedSerialNum()); @@ -234,7 +234,7 @@ LogDataStore::lastSyncToken() const uint64_t lastSerial(getActive(guard).getLastPersistedSerialNum()); if (lastSerial == 0) { const FileChunk * prev = getPrevActive(guard); - if (prev != NULL) { + if (prev != nullptr) { lastSerial = prev->getLastPersistedSerialNum(); } } @@ -274,7 +274,7 @@ LogDataStore::remove(uint64_t serialNum, uint32_t lid) if (lm.valid()) { _fileChunks[lm.getFileId()]->remove(lid, lm.size()); } - lm = getActive(guard).append(serialNum, lid, NULL, 0); + lm = getActive(guard).append(serialNum, lid, nullptr, 0); assert( lm.empty() ); _lidInfo[lid] = lm; } @@ -327,7 +327,7 @@ LogDataStore::getMaxCompactGain() const void LogDataStore::flush(uint64_t syncToken) { - WriteableFileChunk * active = NULL; + WriteableFileChunk * active = nullptr; std::unique_ptr<FileChunkHolder> activeHolder; assert(syncToken == _initFlushSyncToken); { @@ -604,7 +604,7 @@ LogDataStore::getDiskBloat() const /// Do not count the holes in the last file as bloat if (i != _active) { const FileChunk * chunk = _fileChunks[i.getId()].get(); - if (chunk != NULL) { + if (chunk != nullptr) { sz += chunk->getDiskBloat(); } } @@ -916,7 +916,7 @@ LogDataStore::scanDir(const vespalib::string &dir, const vespalib::string &suffi if (file.size() > suffix.size() && file.find(suffix.c_str()) == file.size() - suffix.size()) { vespalib::string base(file.substr(0, file.find(suffix.c_str()))); - char *err(NULL); + char *err(nullptr); errno = 0; NameId baseId(strtoul(base.c_str(), &err, 10)); if ((errno == 0) && (err[0] == '\0')) { diff --git a/searchlib/src/vespa/searchlib/docstore/logdatastore.h b/searchlib/src/vespa/searchlib/docstore/logdatastore.h index c4d1e8bbdb4..4ab747d115d 100644 --- a/searchlib/src/vespa/searchlib/docstore/logdatastore.h +++ b/searchlib/src/vespa/searchlib/docstore/logdatastore.h @@ -89,7 +89,7 @@ public: const search::common::FileHeaderContext &fileHeaderContext, transactionlog::SyncProxy &tlSyncer, const IBucketizer::SP & bucketizer, bool readOnly = false); - ~LogDataStore(); + ~LogDataStore() override; // Implements IDataStore API ssize_t read(uint32_t lid, vespalib::DataBuffer & buffer) const override; @@ -220,7 +220,7 @@ private: const FileChunk * getPrevActive(const LockGuard & guard) const { assert(guard.locks(_updateLock)); (void) guard; - return ( !_prevActive.isActive() ) ? _fileChunks[_prevActive.getId()].get() : NULL; + return ( !_prevActive.isActive() ) ? _fileChunks[_prevActive.getId()].get() : nullptr; } void setActive(const LockGuard & guard, FileId fileId) { assert(guard.locks(_updateLock)); diff --git a/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp b/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp index 91f5c37b817..50517cf09e2 100644 --- a/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp +++ b/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp @@ -3,7 +3,7 @@ #include "writeablefilechunk.h" #include "data_store_file_chunk_stats.h" #include "summaryexceptions.h" -#include <vespa/vespalib/util/closuretask.h> +#include <vespa/vespalib/util/lambdatask.h> #include <vespa/vespalib/util/array.hpp> #include <vespa/vespalib/data/fileheader.h> #include <vespa/vespalib/data/databuffer.h> @@ -14,8 +14,7 @@ #include <vespa/log/log.h> LOG_SETUP(".search.writeablefilechunk"); -using vespalib::makeTask; -using vespalib::makeClosure; +using vespalib::makeLambdaTask; using vespalib::FileHeader; using vespalib::make_string; using vespalib::LockGuard; @@ -45,7 +44,6 @@ class PendingChunk uint64_t _dataOffset; uint32_t _dataLen; public: - typedef std::shared_ptr<PendingChunk> SP; PendingChunk(uint64_t lastSerial, uint64_t dataOffset, uint32_t dataLen); ~PendingChunk(); vespalib::nbostream & getSerializedIdx() { return _idx; } @@ -59,7 +57,6 @@ public: class ProcessedChunk { public: - typedef std::unique_ptr<ProcessedChunk> UP; ProcessedChunk(uint32_t chunkId, uint32_t alignment) : _chunkId(chunkId), _payLoad(0), @@ -77,7 +74,7 @@ private: }; WriteableFileChunk:: -WriteableFileChunk(vespalib::ThreadExecutor &executor, +WriteableFileChunk(vespalib::Executor &executor, FileId fileId, NameId nameId, const vespalib::string &baseName, SerialNum initialSerialNum, @@ -155,6 +152,7 @@ WriteableFileChunk::openIdx() { } return file; } + WriteableFileChunk::~WriteableFileChunk() { if (!frozen()) { @@ -177,7 +175,7 @@ WriteableFileChunk::updateLidMap(const LockGuard &guard, ISetLid &ds, uint64_t s { size_t sz = FileChunk::updateLidMap(guard, ds, serialNum, docIdLimit); _nextChunkId = _chunkInfo.size(); - _active.reset( new Chunk(_nextChunkId++, Chunk::Config(_config.getMaxChunkBytes()))); + _active = std::make_unique<Chunk>(_nextChunkId++, Chunk::Config(_config.getMaxChunkBytes())); _serialNum = getLastPersistedSerialNum(); _firstChunkIdToBeWritten = _active->getId(); setDiskFootprint(0); @@ -188,7 +186,7 @@ WriteableFileChunk::updateLidMap(const LockGuard &guard, ISetLid &ds, uint64_t s void WriteableFileChunk::restart(uint32_t nextChunkId) { - _executor.execute(makeTask(makeClosure(this, &WriteableFileChunk::fileWriter, nextChunkId))); + _executor.execute(makeLambdaTask([this, nextChunkId] {fileWriter(nextChunkId);})); } namespace { @@ -219,7 +217,7 @@ WriteableFileChunk::read(LidInfoWithLidV::const_iterator begin, size_t count, IB const LidInfoWithLid & li = *(begin + i); uint32_t chunk = li.getChunkId(); if ((chunk >= _chunkInfo.size()) || !_chunkInfo[chunk].valid()) { - ChunkMap::const_iterator found = _chunkMap.find(chunk); + auto found = _chunkMap.find(chunk); vespalib::ConstBufferRef buffer; if (found != _chunkMap.end()) { buffer = found->second->getLid(li.getLid()); @@ -234,8 +232,8 @@ WriteableFileChunk::read(LidInfoWithLidV::const_iterator begin, size_t count, IB } } for (auto & it : chunksOnFile) { - LidInfoWithLidV::const_iterator first = find_first(begin, it.first); - LidInfoWithLidV::const_iterator last = seek_past(first, begin + count, it.first); + auto first = find_first(begin, it.first); + auto last = seek_past(first, begin + count, it.first); FileChunk::read(first, last - first, it.second, visitor); } } else { @@ -250,7 +248,7 @@ WriteableFileChunk::read(uint32_t lid, SubChunkId chunkId, vespalib::DataBuffer if (!frozen()) { LockGuard guard(_lock); if ((chunkId >= _chunkInfo.size()) || !_chunkInfo[chunkId].valid()) { - ChunkMap::const_iterator found = _chunkMap.find(chunkId); + auto found = _chunkMap.find(chunkId); if (found != _chunkMap.end()) { return found->second->read(lid, buffer); } else { @@ -268,13 +266,13 @@ WriteableFileChunk::read(uint32_t lid, SubChunkId chunkId, vespalib::DataBuffer void WriteableFileChunk::internalFlush(uint32_t chunkId, uint64_t serialNum) { - Chunk * active(NULL); + Chunk * active(nullptr); { LockGuard guard(_lock); active = _chunkMap[chunkId].get(); } - ProcessedChunk::UP tmp(new ProcessedChunk(chunkId, _alignment)); + auto tmp = std::make_unique<ProcessedChunk>(chunkId, _alignment); if (_alignment > 1) { tmp->getBuf().ensureFree(active->getMaxPackSize(_config.getCompression()) + _alignment - 1); } @@ -293,12 +291,12 @@ WriteableFileChunk::internalFlush(uint32_t chunkId, uint64_t serialNum) } void -WriteableFileChunk::enque(ProcessedChunk::UP tmp) +WriteableFileChunk::enque(ProcessedChunkUP tmp) { LOG(debug, "enqueing %p", tmp.get()); MonitorGuard guard(_writeMonitor); _writeQ.push_back(std::move(tmp)); - if (_writeTaskIsRunning == false) { + if ( ! _writeTaskIsRunning) { _writeTaskIsRunning = true; uint32_t nextChunkId = _firstChunkIdToBeWritten; guard.signal(); @@ -359,12 +357,12 @@ WriteableFileChunk::insertChunks(ProcessedChunkMap & orderedChunks, ProcessedChu { (void) nextChunkId; for (auto &chunk : newChunks) { - if (chunk.get() != 0) { + if (chunk) { assert(chunk->getChunkId() >= nextChunkId); assert(orderedChunks.find(chunk->getChunkId()) == orderedChunks.end()); orderedChunks[chunk->getChunkId()] = std::move(chunk); } else { - orderedChunks[std::numeric_limits<uint32_t>::max()] = ProcessedChunk::UP(); + orderedChunks[std::numeric_limits<uint32_t>::max()] = ProcessedChunkUP(); } } } @@ -375,7 +373,7 @@ WriteableFileChunk::fetchNextChain(ProcessedChunkMap & orderedChunks, const uint ProcessedChunkQ chunks; while (!orderedChunks.empty() && ((orderedChunks.begin()->first == (firstChunkId+chunks.size())) || - (orderedChunks.begin()->second.get() == NULL))) + !orderedChunks.begin()->second)) { chunks.push_back(std::move(orderedChunks.begin()->second)); orderedChunks.erase(orderedChunks.begin()); @@ -393,8 +391,7 @@ WriteableFileChunk::computeChunkMeta(const LockGuard & guard, const ChunkMeta cmeta(offset, tmp.getPayLoad(), active.getLastSerial(), active.count()); assert((size_t(tmp.getBuf().getData())%_alignment) == 0); assert((dataLen%_alignment) == 0); - PendingChunk::SP pcsp; - pcsp.reset(new PendingChunk(active.getLastSerial(), offset, dataLen)); + auto pcsp = std::make_shared<PendingChunk>(active.getLastSerial(), offset, dataLen); PendingChunk &pc(*pcsp.get()); nbostream &os(pc.getSerializedIdx()); cmeta.serialize(os); @@ -424,8 +421,7 @@ WriteableFileChunk::computeChunkMeta(ProcessedChunkQ & chunks, size_t startPos, LockGuard guard(_lock); if (!_pendingChunks.empty()) { - const PendingChunk::SP pcsp(_pendingChunks.back()); - const PendingChunk &pc(*pcsp.get()); + const PendingChunk & pc = *_pendingChunks.back(); assert(pc.getLastSerial() >= lastSerial); lastSerial = pc.getLastSerial(); } @@ -454,7 +450,7 @@ WriteableFileChunk::writeData(const ProcessedChunkQ & chunks, size_t sz) { vespalib::DataBuffer buf(0ul, _alignment); buf.ensureFree(sz); - for (const ProcessedChunk::UP & chunk : chunks) { + for (const auto & chunk : chunks) { buf.writeBytes(chunk->getBuf().getData(), chunk->getBuf().getDataLen()); } @@ -540,15 +536,15 @@ WriteableFileChunk::freeze() { if (!frozen()) { waitForAllChunksFlushedToDisk(); - enque(ProcessedChunk::UP()); - _executor.sync(); + enque(ProcessedChunkUP()); { MonitorGuard guard(_writeMonitor); while (_writeTaskIsRunning) { guard.wait(10); } - assert(_writeQ.empty()); } + assert(_writeQ.empty()); + assert(_chunkMap.empty()); { MonitorGuard guard(_lock); setDiskFootprint(getDiskFootprint(guard)); @@ -632,7 +628,7 @@ int32_t WriteableFileChunk::flushLastIfNonEmpty(bool force) chunkId = _active->getId(); _chunkMap[chunkId] = std::move(_active); assert(_nextChunkId < LidInfo::getChunkIdLimit()); - _active.reset(new Chunk(_nextChunkId++, Chunk::Config(_config.getMaxChunkBytes()))); + _active = std::make_unique<Chunk>(_nextChunkId++, Chunk::Config(_config.getMaxChunkBytes())); } return chunkId; } @@ -643,10 +639,7 @@ WriteableFileChunk::flush(bool block, uint64_t syncToken) int32_t chunkId = flushLastIfNonEmpty(syncToken > _serialNum); if (chunkId >= 0) { setSerialNum(syncToken); - _executor.execute(makeTask(makeClosure(this, - &WriteableFileChunk::internalFlush, - static_cast<uint32_t>(chunkId), - _serialNum))); + _executor.execute(makeLambdaTask([this, chunkId, serialNum=_serialNum] { internalFlush(chunkId, serialNum); })); } else { if (block) { MonitorGuard guard(_lock); @@ -656,7 +649,6 @@ WriteableFileChunk::flush(bool block, uint64_t syncToken) } } if (block) { - _executor.sync(); waitForChunkFlushedToDisk(chunkId); } } @@ -693,10 +685,7 @@ WriteableFileChunk::waitForAllChunksFlushedToDisk() const } LidInfo -WriteableFileChunk::append(uint64_t serialNum, - uint32_t lid, - const void * buffer, - size_t len) +WriteableFileChunk::append(uint64_t serialNum, uint32_t lid, const void * buffer, size_t len) { assert( !frozen() ); if ( ! _active->hasRoom(len)) { @@ -818,8 +807,7 @@ WriteableFileChunk::needFlushPendingChunks(const MonitorGuard & guard, uint64_t assert(guard.monitors(_lock)); if (_pendingChunks.empty()) return false; - const PendingChunk::SP pcsp(_pendingChunks.front()); - const PendingChunk &pc(*pcsp.get()); + const PendingChunk & pc = *_pendingChunks.front(); if (pc.getLastSerial() > serialNum) return false; bool datWritten = datFileLen >= pc.getDataOffset() + pc.getDataLen(); @@ -868,8 +856,7 @@ WriteableFileChunk::unconditionallyFlushPendingChunks(const vespalib::LockGuard for (;;) { if (!needFlushPendingChunks(guard, serialNum, datFileLen)) break; - PendingChunk::SP pcsp; - pcsp.swap(_pendingChunks.front()); + std::shared_ptr<PendingChunk> pcsp = std::move(_pendingChunks.front()); _pendingChunks.pop_front(); const PendingChunk &pc(*pcsp.get()); assert(_pendingIdx >= pc.getIdxLen()); diff --git a/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.h b/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.h index 4a2ebfc42df..2c300bc9035 100644 --- a/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.h +++ b/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.h @@ -3,7 +3,7 @@ #pragma once #include "filechunk.h" -#include <vespa/vespalib/util/threadexecutor.h> +#include <vespa/vespalib/util/executor.h> #include <vespa/searchlib/transactionlog/syncproxy.h> #include <vespa/fastos/file.h> #include <map> @@ -42,7 +42,7 @@ public: public: typedef std::unique_ptr<WriteableFileChunk> UP; - WriteableFileChunk(vespalib::ThreadExecutor & executor, FileId fileId, NameId nameId, + WriteableFileChunk(vespalib::Executor & executor, FileId fileId, NameId nameId, const vespalib::string & baseName, uint64_t initialSerialNum, uint32_t docIdLimit, const Config & config, const TuneFileSummary &tune, const common::FileHeaderContext &fileHeaderContext, @@ -128,7 +128,7 @@ private: bool _writeTaskIsRunning; vespalib::Monitor _writeMonitor; ProcessedChunkQ _writeQ; - vespalib::ThreadExecutor & _executor; + vespalib::Executor & _executor; ProcessedChunkMap _orderedChunks; BucketDensityComputer _bucketMap; }; diff --git a/searchlib/src/vespa/searchlib/features/attributefeature.cpp b/searchlib/src/vespa/searchlib/features/attributefeature.cpp index 56d02ce6d4e..1e18a2d3af8 100644 --- a/searchlib/src/vespa/searchlib/features/attributefeature.cpp +++ b/searchlib/src/vespa/searchlib/features/attributefeature.cpp @@ -147,7 +147,7 @@ public: * @param attribute The attribute vector to use. * @param idx The index used for an array attribute. */ - AttributeExecutor(const search::attribute::IAttributeVector * attribute, uint32_t idx); + AttributeExecutor(const attribute::IAttributeVector * attribute, uint32_t idx); void execute(uint32_t docId) override; }; @@ -172,7 +172,7 @@ public: * @param key The key to find a corresponding weight for. * @param useKey Whether we should consider the key. */ - WeightedSetAttributeExecutor(const search::attribute::IAttributeVector * attribute, T key, bool useKey); + WeightedSetAttributeExecutor(const attribute::IAttributeVector * attribute, T key, bool useKey); void execute(uint32_t docId) override; }; @@ -183,7 +183,7 @@ SingleAttributeExecutor<T>::execute(uint32_t docId) typename T::LoadedValueType v = _attribute.getFast(docId); // value outputs().set_number(0, __builtin_expect(attribute::isUndefined(v), false) - ? attribute::getUndefined<search::feature_t>() + ? attribute::getUndefined<feature_t>() : util::getAsFeature(v)); outputs().set_number(1, 0.0f); // weight outputs().set_number(2, 0.0f); // contains @@ -267,8 +267,9 @@ WeightedSetAttributeExecutor<BT, T>::execute(uint32_t docId) AttributeBlueprint::AttributeBlueprint() : - search::fef::Blueprint("attribute"), + fef::Blueprint("attribute"), _attrName(), + _attrKey(), _extra(), _tensorType(ValueType::double_type()) { @@ -277,18 +278,19 @@ AttributeBlueprint::AttributeBlueprint() : AttributeBlueprint::~AttributeBlueprint() = default; void -AttributeBlueprint::visitDumpFeatures(const search::fef::IIndexEnvironment &, - search::fef::IDumpFeatureVisitor &) const +AttributeBlueprint::visitDumpFeatures(const fef::IIndexEnvironment &, + fef::IDumpFeatureVisitor &) const { } bool -AttributeBlueprint::setup(const search::fef::IIndexEnvironment & env, - const search::fef::ParameterList & params) +AttributeBlueprint::setup(const fef::IIndexEnvironment & env, + const fef::ParameterList & params) { // params[0] = attribute name // params[1] = index (array attribute) or key (weighted set attribute) _attrName = params[0].getValue(); + _attrKey = createAttributeKey(_attrName); if (params.size() == 2) { _extra = params[1].getValue(); } @@ -315,10 +317,10 @@ AttributeBlueprint::setup(const search::fef::IIndexEnvironment & env, return !_tensorType.is_error(); } -search::fef::Blueprint::UP +fef::Blueprint::UP AttributeBlueprint::createInstance() const { - return search::fef::Blueprint::UP(new AttributeBlueprint()); + return std::make_unique<AttributeBlueprint>(); } #define CREATE_AND_RETURN_IF_SINGLE_NUMERIC(a, T) \ @@ -328,7 +330,7 @@ AttributeBlueprint::createInstance() const namespace { -search::fef::FeatureExecutor & +fef::FeatureExecutor & createAttributeExecutor(const IAttributeVector *attribute, const vespalib::string &attrName, const vespalib::string &extraParam, vespalib::Stash &stash) { if (attribute == NULL) { @@ -375,7 +377,7 @@ createAttributeExecutor(const IAttributeVector *attribute, const vespalib::strin } } -search::fef::FeatureExecutor & +fef::FeatureExecutor & createTensorAttributeExecutor(const IAttributeVector *attribute, const vespalib::string &attrName, const ValueType &tensorType, vespalib::Stash &stash) @@ -385,8 +387,8 @@ createTensorAttributeExecutor(const IAttributeVector *attribute, const vespalib: " Returning empty tensor.", attrName.c_str()); return ConstantTensorExecutor::createEmpty(tensorType, stash); } - if (attribute->getCollectionType() != search::attribute::CollectionType::SINGLE || - attribute->getBasicType() != search::attribute::BasicType::TENSOR) { + if (attribute->getCollectionType() != attribute::CollectionType::SINGLE || + attribute->getBasicType() != attribute::BasicType::TENSOR) { LOG(warning, "The attribute vector '%s' is NOT of type tensor." " Returning empty tensor.", attribute->getName().c_str()); return ConstantTensorExecutor::createEmpty(tensorType, stash); @@ -413,10 +415,16 @@ createTensorAttributeExecutor(const IAttributeVector *attribute, const vespalib: } -search::fef::FeatureExecutor & -AttributeBlueprint::createExecutor(const search::fef::IQueryEnvironment &env, vespalib::Stash &stash) const +void +AttributeBlueprint::prepareSharedState(const fef::IQueryEnvironment & env, fef::IObjectStore & store) const +{ + lookupAndStoreAttribute(_attrKey, _attrName, env, store); +} + +fef::FeatureExecutor & +AttributeBlueprint::createExecutor(const fef::IQueryEnvironment &env, vespalib::Stash &stash) const { - const IAttributeVector *attribute = env.getAttributeContext().getAttribute(_attrName); + const IAttributeVector * attribute = lookupAttribute(_attrKey, _attrName, env); if (_tensorType.is_tensor()) { return createTensorAttributeExecutor(attribute, _attrName, _tensorType, stash); } else { diff --git a/searchlib/src/vespa/searchlib/features/attributefeature.h b/searchlib/src/vespa/searchlib/features/attributefeature.h index c115c4acaea..47597823f08 100644 --- a/searchlib/src/vespa/searchlib/features/attributefeature.h +++ b/searchlib/src/vespa/searchlib/features/attributefeature.h @@ -16,6 +16,7 @@ namespace search::features { class AttributeBlueprint : public fef::Blueprint { private: vespalib::string _attrName; // the name of the attribute vector + vespalib::string _attrKey; // Used for looking up the attribute in the ObjectStore. vespalib::string _extra; // the index or key vespalib::eval::ValueType _tensorType; @@ -25,6 +26,7 @@ public: void visitDumpFeatures(const fef::IIndexEnvironment & env, fef::IDumpFeatureVisitor & visitor) const override; fef::Blueprint::UP createInstance() const override; + void prepareSharedState(const fef::IQueryEnvironment & queryEnv, fef::IObjectStore & objectStore) const override; fef::FeatureExecutor &createExecutor(const fef::IQueryEnvironment &env, vespalib::Stash &stash) const override; fef::ParameterDescriptions getDescriptions() const override; bool setup(const fef::IIndexEnvironment & env, const fef::ParameterList & params) override; diff --git a/searchlib/src/vespa/searchlib/features/bm25_feature.cpp b/searchlib/src/vespa/searchlib/features/bm25_feature.cpp index a9430db09c3..0be3c2876f7 100644 --- a/searchlib/src/vespa/searchlib/features/bm25_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/bm25_feature.cpp @@ -3,26 +3,29 @@ #include "bm25_feature.h" #include <vespa/searchlib/fef/itermdata.h> #include <vespa/searchlib/fef/itermfielddata.h> +#include <vespa/searchlib/fef/objectstore.h> #include <memory> namespace search::features { +using fef::AnyWrapper; using fef::Blueprint; using fef::FeatureExecutor; using fef::FieldInfo; using fef::ITermData; using fef::ITermFieldData; using fef::MatchDataDetails; +using fef::objectstore::as_value; Bm25Executor::Bm25Executor(const fef::FieldInfo& field, - const fef::IQueryEnvironment& env) + const fef::IQueryEnvironment& env, + double avg_field_length) : FeatureExecutor(), _terms(), - _avg_field_length(10), + _avg_field_length(avg_field_length), _k1_param(1.2), _b_param(0.75) { - // TODO: Don't use hard coded avg_field_length // TODO: Add support for setting k1 and b for (size_t i = 0; i < env.getNumTerms(); ++i) { const ITermData* term = env.getTerm(i); @@ -93,10 +96,33 @@ Bm25Blueprint::setup(const fef::IIndexEnvironment& env, const fef::ParameterList return (_field != nullptr); } +namespace { + +vespalib::string +make_avg_field_length_key(const vespalib::string& base_name, const vespalib::string& field_name) +{ + return base_name + ".afl." + field_name; +} + +} + +void +Bm25Blueprint::prepareSharedState(const fef::IQueryEnvironment& env, fef::IObjectStore& store) const +{ + vespalib::string key = make_avg_field_length_key(getBaseName(), _field->name()); + if (store.get(key) == nullptr) { + store.add(key, std::make_unique<AnyWrapper<double>>(env.get_average_field_length(_field->name()))); + } +} + fef::FeatureExecutor& Bm25Blueprint::createExecutor(const fef::IQueryEnvironment& env, vespalib::Stash& stash) const { - return stash.create<Bm25Executor>(*_field, env); + const auto* lookup_result = env.getObjectStore().get(make_avg_field_length_key(getBaseName(), _field->name())); + double avg_field_length = lookup_result != nullptr ? + as_value<double>(*lookup_result) : + env.get_average_field_length(_field->name()); + return stash.create<Bm25Executor>(*_field, env, avg_field_length); } } diff --git a/searchlib/src/vespa/searchlib/features/bm25_feature.h b/searchlib/src/vespa/searchlib/features/bm25_feature.h index 457cfea4c87..4b1576b57b3 100644 --- a/searchlib/src/vespa/searchlib/features/bm25_feature.h +++ b/searchlib/src/vespa/searchlib/features/bm25_feature.h @@ -30,7 +30,8 @@ private: public: Bm25Executor(const fef::FieldInfo& field, - const fef::IQueryEnvironment& env); + const fef::IQueryEnvironment& env, + double avg_field_length); void handle_bind_match_data(const fef::MatchData& match_data) override; void execute(uint32_t docId) override; @@ -53,6 +54,7 @@ public: return fef::ParameterDescriptions().desc().indexField(fef::ParameterCollection::ANY); } bool setup(const fef::IIndexEnvironment& env, const fef::ParameterList& params) override; + void prepareSharedState(const fef::IQueryEnvironment& env, fef::IObjectStore& store) const override; fef::FeatureExecutor& createExecutor(const fef::IQueryEnvironment& env, vespalib::Stash& stash) const override; }; diff --git a/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp b/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp index 642870e94f8..1f51ee5cef6 100644 --- a/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp @@ -852,7 +852,8 @@ createQueryVector(const IQueryEnvironment & env, const IAttributeVector * attrib void DotProductBlueprint::prepareSharedState(const IQueryEnvironment & env, IObjectStore & store) const { - const IAttributeVector * attribute = env.getAttributeContext().getAttribute(getAttribute(env)); + vespalib::string attributeKey = make_attribute_key(getBaseName(), _defaultAttribute); + const IAttributeVector * attribute = lookupAndStoreAttribute(attributeKey, getAttribute(env), env, store); if (attribute == nullptr) return; vespalib::string queryVectorKey = make_queryvector_key(getBaseName(), _queryVector); @@ -864,17 +865,13 @@ DotProductBlueprint::prepareSharedState(const IQueryEnvironment & env, IObjectSt } } - attribute = upgradeIfNecessary(attribute, env); - - vespalib::string attributeKey = make_attribute_key(getBaseName(), _defaultAttribute); - if (store.get(attributeKey) == nullptr) { - store.add(attributeKey, std::make_unique<fef::AnyWrapper<const IAttributeVector *>>(attribute)); - } + upgradeIfNecessary(attribute, env); } FeatureExecutor & DotProductBlueprint::createExecutor(const IQueryEnvironment & env, vespalib::Stash &stash) const { + // Doing it "manually" here to avoid looking up attribute override unless needed. const fef::Anything * attributeArg = env.getObjectStore().get(make_attribute_key(getBaseName(), _defaultAttribute)); const IAttributeVector * attribute = (attributeArg != nullptr) ? static_cast<const fef::AnyWrapper<const IAttributeVector *> *>(attributeArg)->getValue() diff --git a/searchlib/src/vespa/searchlib/features/queryfeature.cpp b/searchlib/src/vespa/searchlib/features/queryfeature.cpp index b9041901ced..b927188c1aa 100644 --- a/searchlib/src/vespa/searchlib/features/queryfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/queryfeature.cpp @@ -15,6 +15,7 @@ #include <vespa/eval/tensor/tensor.h> #include <vespa/eval/eval/value_type.h> #include <vespa/vespalib/locale/c.h> +#include <cerrno> #include <vespa/log/log.h> LOG_SETUP(".features.queryfeature"); diff --git a/searchlib/src/vespa/searchlib/fef/blueprint.cpp b/searchlib/src/vespa/searchlib/fef/blueprint.cpp index b4def052379..d7c5cb665ed 100644 --- a/searchlib/src/vespa/searchlib/fef/blueprint.cpp +++ b/searchlib/src/vespa/searchlib/fef/blueprint.cpp @@ -69,4 +69,35 @@ Blueprint::setup(const IIndexEnvironment &indexEnv, return false; } +const attribute::IAttributeVector * +Blueprint::lookupAndStoreAttribute(const vespalib::string & key, vespalib::stringref attrName, + const IQueryEnvironment & env, IObjectStore & store) +{ + const Anything * obj = store.get(key); + if (obj == nullptr) { + const IAttributeVector * attribute = env.getAttributeContext().getAttribute(attrName); + store.add(key, std::make_unique<AnyWrapper<const IAttributeVector *>>(attribute)); + return attribute; + } + return static_cast<const AnyWrapper<const IAttributeVector *> *>(obj)->getValue(); +} + +const attribute::IAttributeVector * +Blueprint::lookupAttribute(const vespalib::string & key, vespalib::stringref attrName, const IQueryEnvironment & env) +{ + const Anything * attributeArg = env.getObjectStore().get(key); + const IAttributeVector * attribute = (attributeArg != nullptr) + ? static_cast<const AnyWrapper<const IAttributeVector *> *>(attributeArg)->getValue() + : nullptr; + if (attribute == nullptr) { + attribute = env.getAttributeContext().getAttribute(attrName); + } + return attribute;; +} + +vespalib::string +Blueprint::createAttributeKey(vespalib::stringref attrName) { + return "fef.attribute.key." + attrName; +} + } diff --git a/searchlib/src/vespa/searchlib/fef/blueprint.h b/searchlib/src/vespa/searchlib/fef/blueprint.h index 53c06dbd4cc..dd622ea36d9 100644 --- a/searchlib/src/vespa/searchlib/fef/blueprint.h +++ b/searchlib/src/vespa/searchlib/fef/blueprint.h @@ -70,6 +70,7 @@ private: DependencyHandler *_dependency_handler; protected: + using IAttributeVector = attribute::IAttributeVector; /** * Define an input feature for this blueprint. This method should * be invoked by the @ref setup method. Note that the order in @@ -100,6 +101,19 @@ protected: void describeOutput(vespalib::stringref outName, vespalib::stringref desc, const FeatureType &type = FeatureType::number()); + /** + * Used to store a reference to the attribute during prepareSharedState + * for later use in createExecutor + **/ + static const IAttributeVector * + lookupAndStoreAttribute(const vespalib::string & key, vespalib::stringref attrName, + const IQueryEnvironment & env, IObjectStore & objectStore); + /** + * Used to lookup attribute from the most efficient source. + **/ + static const IAttributeVector * + lookupAttribute(const vespalib::string & key, vespalib::stringref attrName, const IQueryEnvironment & env); + static vespalib::string createAttributeKey(vespalib::stringref attrName); public: /** * Create an empty blueprint. Blueprints in their initial state diff --git a/searchlib/src/vespa/searchlib/fef/iqueryenvironment.h b/searchlib/src/vespa/searchlib/fef/iqueryenvironment.h index a7f268e5c6b..041e9ec67bc 100644 --- a/searchlib/src/vespa/searchlib/fef/iqueryenvironment.h +++ b/searchlib/src/vespa/searchlib/fef/iqueryenvironment.h @@ -70,6 +70,15 @@ public: virtual const search::attribute::IAttributeContext & getAttributeContext() const = 0; /** + * Returns the average field length for the given field. + * + * @param field_name field name + * + * @return average field length + **/ + virtual double get_average_field_length(const vespalib::string &field_name) const = 0; + + /** * Returns a const view of the index environment. * * @return index environment diff --git a/searchlib/src/vespa/searchlib/fef/objectstore.h b/searchlib/src/vespa/searchlib/fef/objectstore.h index 49176afa3c9..2debcd277e9 100644 --- a/searchlib/src/vespa/searchlib/fef/objectstore.h +++ b/searchlib/src/vespa/searchlib/fef/objectstore.h @@ -2,9 +2,13 @@ #pragma once #include <vespa/vespalib/stllike/hash_map.h> +#include <cassert> namespace search::fef { +/** + * Top level interface for things to store in an IObjectStore. + */ class Anything { public: @@ -12,6 +16,9 @@ public: virtual ~Anything() { } }; +/** + * Implementation of the Anything interface that wraps a value of the given type. + */ template<typename T> class AnyWrapper : public Anything { @@ -22,6 +29,9 @@ private: T _value; }; +/** + * Interface for a key value store of Anything instances. + */ class IObjectStore { public: @@ -30,6 +40,9 @@ public: virtual const Anything * get(const vespalib::string & key) const = 0; }; +/** + * Object store implementation on top of a hash map. + */ class ObjectStore : public IObjectStore { public: @@ -42,4 +55,20 @@ private: ObjectMap _objectMap; }; +namespace objectstore { + +/** + * Utility function that gets the value stored in an Anything instance (via AnyWrapper). + */ +template<typename T> +const T & +as_value(const Anything &val) { + using WrapperType = AnyWrapper<T>; + const auto *wrapper = dynamic_cast<const WrapperType *>(&val); + assert(wrapper != nullptr); + return wrapper->getValue(); +} + +} + } diff --git a/searchlib/src/vespa/searchlib/fef/phrasesplitter.h b/searchlib/src/vespa/searchlib/fef/phrasesplitter.h index 4c7ca4b67d7..4e46c9eaa7c 100644 --- a/searchlib/src/vespa/searchlib/fef/phrasesplitter.h +++ b/searchlib/src/vespa/searchlib/fef/phrasesplitter.h @@ -113,6 +113,7 @@ public: const Properties & getProperties() const override { return _queryEnv.getProperties(); } const Location & getLocation() const override { return _queryEnv.getLocation(); } const attribute::IAttributeContext & getAttributeContext() const override { return _queryEnv.getAttributeContext(); } + double get_average_field_length(const vespalib::string &field_name) const override { return _queryEnv.get_average_field_length(field_name); } const IIndexEnvironment & getIndexEnvironment() const override { return _queryEnv.getIndexEnvironment(); } void bind_match_data(const fef::MatchData &md) { _matchData = &md; } }; diff --git a/searchlib/src/vespa/searchlib/fef/test/queryenvironment.cpp b/searchlib/src/vespa/searchlib/fef/test/queryenvironment.cpp index ee305dcff55..4697675c071 100644 --- a/searchlib/src/vespa/searchlib/fef/test/queryenvironment.cpp +++ b/searchlib/src/vespa/searchlib/fef/test/queryenvironment.cpp @@ -2,21 +2,17 @@ #include "queryenvironment.h" -namespace search { -namespace fef { -namespace test { +namespace search::fef::test { QueryEnvironment::QueryEnvironment(IndexEnvironment *env) : _indexEnv(env), _terms(), _properties(), _location(), - _attrCtx((env == NULL) ? attribute::IAttributeContext::UP() : env->getAttributeMap().createContext()) + _attrCtx((env == nullptr) ? attribute::IAttributeContext::UP() : env->getAttributeMap().createContext()) { } QueryEnvironment::~QueryEnvironment() { } -} // namespace test -} // namespace fef -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/fef/test/queryenvironment.h b/searchlib/src/vespa/searchlib/fef/test/queryenvironment.h index 0179b5020e6..40898281794 100644 --- a/searchlib/src/vespa/searchlib/fef/test/queryenvironment.h +++ b/searchlib/src/vespa/searchlib/fef/test/queryenvironment.h @@ -6,10 +6,9 @@ #include <vespa/searchlib/fef/iqueryenvironment.h> #include <vespa/searchlib/fef/location.h> #include <vespa/searchlib/fef/simpletermdata.h> +#include <unordered_map> -namespace search { -namespace fef { -namespace test { +namespace search::fef::test { /** * Implementation of the IQueryEnvironment interface used for testing. @@ -25,6 +24,7 @@ private: Properties _properties; Location _location; search::attribute::IAttributeContext::UP _attrCtx; + std::unordered_map<std::string, double> _avg_field_lengths; public: /** @@ -40,6 +40,13 @@ public: const ITermData *getTerm(uint32_t idx) const override { return idx < _terms.size() ? &_terms[idx] : NULL; } const Location & getLocation() const override { return _location; } const search::attribute::IAttributeContext &getAttributeContext() const override { return *_attrCtx; } + double get_average_field_length(const vespalib::string& field_name) const override { + auto itr = _avg_field_lengths.find(field_name); + if (itr != _avg_field_lengths.end()) { + return itr->second; + } + return 1.0; + } const IIndexEnvironment &getIndexEnvironment() const override { assert(_indexEnv != NULL); return *_indexEnv; } /** Returns a reference to the index environment of this. */ @@ -76,9 +83,9 @@ public: /** Returns a reference to the location of this. */ Location & getLocation() { return _location; } + + std::unordered_map<std::string, double>& get_avg_field_lengths() { return _avg_field_lengths; } }; -} // namespace test -} // namespace fef -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/fef/test/queryenvironmentbuilder.cpp b/searchlib/src/vespa/searchlib/fef/test/queryenvironmentbuilder.cpp index 2d9fb998869..67a2eaf5677 100644 --- a/searchlib/src/vespa/searchlib/fef/test/queryenvironmentbuilder.cpp +++ b/searchlib/src/vespa/searchlib/fef/test/queryenvironmentbuilder.cpp @@ -2,16 +2,13 @@ #include "queryenvironmentbuilder.h" -namespace search { -namespace fef { -namespace test { +namespace search::fef::test { QueryEnvironmentBuilder::QueryEnvironmentBuilder(QueryEnvironment &env, MatchDataLayout &layout) : _queryEnv(env), _layout(layout) { - // empty } QueryEnvironmentBuilder::~QueryEnvironmentBuilder() { } @@ -39,8 +36,8 @@ QueryEnvironmentBuilder::addIndexNode(const std::vector<vespalib::string> &field td.setWeight(search::query::Weight(100)); for (uint32_t i = 0; i < fieldNames.size(); ++i) { const FieldInfo *info = _queryEnv.getIndexEnv()->getFieldByName(fieldNames[i]); - if (info == NULL || info->type() != FieldType::INDEX) { - return NULL; + if (info == nullptr || info->type() != FieldType::INDEX) { + return nullptr; } SimpleTermFieldData &tfd = td.addField(info->id()); tfd.setHandle(_layout.allocTermField(tfd.getFieldId())); @@ -52,8 +49,8 @@ SimpleTermData * QueryEnvironmentBuilder::addAttributeNode(const vespalib::string &attrName) { const FieldInfo *info = _queryEnv.getIndexEnv()->getFieldByName(attrName); - if (info == NULL || info->type() != FieldType::ATTRIBUTE) { - return NULL; + if (info == nullptr || info->type() != FieldType::ATTRIBUTE) { + return nullptr; } _queryEnv.getTerms().push_back(SimpleTermData()); SimpleTermData &td = _queryEnv.getTerms().back(); @@ -63,6 +60,11 @@ QueryEnvironmentBuilder::addAttributeNode(const vespalib::string &attrName) return &td; } -} // namespace test -} // namespace fef -} // namespace search +QueryEnvironmentBuilder& +QueryEnvironmentBuilder::set_avg_field_length(const vespalib::string& field_name, double avg_field_length) +{ + _queryEnv.get_avg_field_lengths()[field_name] = avg_field_length; + return *this; +} + +} diff --git a/searchlib/src/vespa/searchlib/fef/test/queryenvironmentbuilder.h b/searchlib/src/vespa/searchlib/fef/test/queryenvironmentbuilder.h index 98aed323f9a..36a63b2a9a2 100644 --- a/searchlib/src/vespa/searchlib/fef/test/queryenvironmentbuilder.h +++ b/searchlib/src/vespa/searchlib/fef/test/queryenvironmentbuilder.h @@ -57,6 +57,8 @@ public: /** Returns a const reference to the match data layout of this. */ const MatchDataLayout &getLayout() const { return _layout; } + QueryEnvironmentBuilder& set_avg_field_length(const vespalib::string& field_name, double avg_field_length); + private: QueryEnvironmentBuilder(const QueryEnvironmentBuilder &); // hide QueryEnvironmentBuilder & operator=(const QueryEnvironmentBuilder &); // hide diff --git a/searchlib/src/vespa/searchlib/util/url.cpp b/searchlib/src/vespa/searchlib/util/url.cpp index 638f22fc8b7..496a19d153f 100644 --- a/searchlib/src/vespa/searchlib/util/url.cpp +++ b/searchlib/src/vespa/searchlib/util/url.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "url.h" +#include <algorithm> #include <vespa/log/log.h> LOG_SETUP(".searchlib.util.url"); diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java index eb90b2f56d7..2b72a775b24 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java @@ -9,10 +9,7 @@ import com.yahoo.config.model.api.SuperModelListener; import com.yahoo.config.model.api.SuperModelProvider; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.NodeType; -import com.yahoo.vespa.flags.BooleanFlag; import com.yahoo.vespa.flags.FlagSource; -import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.service.monitor.DuperModelInfraApi; import com.yahoo.vespa.service.monitor.InfraApplicationApi; @@ -40,14 +37,12 @@ public class DuperModelManager implements DuperModelInfraApi { static final TenantHostApplication tenantHostApplication = new TenantHostApplication(); private final Map<ApplicationId, InfraApplication> supportedInfraApplications; - private final Map<ApplicationId, InfraApplication> supportedMinusTenantHostInfraApplications; private final Object monitor = new Object(); private final DuperModel duperModel; // The set of active infrastructure ApplicationInfo. Not all are necessarily in the DuperModel for historical reasons. private final Set<ApplicationId> activeInfraInfos = new HashSet<>(10); - private final BooleanFlag tenantHostApplicationEnabled; @Inject public DuperModelManager(ConfigserverConfig configServerConfig, FlagSource flagSource, SuperModelProvider superModelProvider) { @@ -59,7 +54,6 @@ public class DuperModelManager implements DuperModelInfraApi { /** For testing */ DuperModelManager(boolean multitenant, boolean isController, SuperModelProvider superModelProvider, DuperModel duperModel, FlagSource flagSource) { this.duperModel = duperModel; - this.tenantHostApplicationEnabled = Flags.ENABLE_TENANT_HOST_APP.bindTo(flagSource); if (multitenant) { supportedInfraApplications = @@ -70,9 +64,6 @@ public class DuperModelManager implements DuperModelInfraApi { } else { supportedInfraApplications = Map.of(); } - supportedMinusTenantHostInfraApplications = supportedInfraApplications.entrySet().stream() - .filter(app -> app.getValue().getCapacity().type() != NodeType.host) - .collect(Collectors.toUnmodifiableMap(Map.Entry::getKey, Map.Entry::getValue)); superModelProvider.registerListener(new SuperModelListener() { @Override @@ -103,23 +94,16 @@ public class DuperModelManager implements DuperModelInfraApi { @Override public List<InfraApplicationApi> getSupportedInfraApplications() { - return new ArrayList<>(getSupportedApps().values()); + return new ArrayList<>(supportedInfraApplications.values()); } @Override public Optional<InfraApplicationApi> getInfraApplication(ApplicationId applicationId) { - return Optional.ofNullable(getSupportedApps().get(applicationId)); - } - - private Map<ApplicationId, InfraApplication> getSupportedApps() { - return tenantHostApplicationEnabled.value() ? supportedInfraApplications : supportedMinusTenantHostInfraApplications; + return Optional.ofNullable(supportedInfraApplications.get(applicationId)); } /** * Returns true if application is considered an infrastructure application by the DuperModel. - * - * <p>Note: Unless enable-tenant-host-app flag is enabled, the tenant host "application" is NOT considered an - * infrastructure application: It is just a cluster in the {@link ZoneApplication zone application}. */ public boolean isSupportedInfraApplication(ApplicationId applicationId) { return supportedInfraApplications.containsKey(applicationId); diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/ZoneApplication.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/ZoneApplication.java deleted file mode 100644 index bcf5f096e7f..00000000000 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/ZoneApplication.java +++ /dev/null @@ -1,108 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.service.duper; - -import com.yahoo.config.model.api.ServiceInfo; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ApplicationName; -import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.TenantName; -import com.yahoo.vespa.applicationmodel.ClusterId; -import com.yahoo.vespa.applicationmodel.ServiceType; -import com.yahoo.vespa.service.model.ApplicationInstanceGenerator; - -import java.util.Objects; - -/** - * @author hakon - * - * TODO: This does not extend InfraApplication because - * 1) It is not deployed same as the other HostedVespaApplications - * 2) ZoneApplication has multiple clusters - */ -public class ZoneApplication { - - private ZoneApplication() {} - - private static final ApplicationId ZONE_APPLICATION_ID = InfraApplication - .createHostedVespaApplicationId("routing"); - private static final ClusterId NODE_ADMIN_CLUSTER_ID = new ClusterId("node-admin"); - private static final ClusterId ROUTING_CLUSTER_ID = new ClusterId("routing"); - - public static ApplicationId getApplicationId() { - return ZONE_APPLICATION_ID; - } - - public static TenantName getTenantName() { - return ZONE_APPLICATION_ID.tenant(); - } - - public static ApplicationName getApplicationName() { - return ZONE_APPLICATION_ID.application(); - } - - public static NodeType getNodeAdminNodeType() { - return NodeType.host; - } - - public static ClusterId getNodeAdminClusterId() { - return NODE_ADMIN_CLUSTER_ID; - } - - public static ClusterSpec.Type getNodeAdminClusterSpecType() { - return ClusterSpec.Type.container; - } - - public static ClusterSpec.Id getNodeAdminClusterSpecId() { - return new ClusterSpec.Id(getNodeAdminClusterId().s()); - } - - public static ServiceType getNodeAdminServiceType() { - return ServiceType.CONTAINER; - } - - public static int getNodeAdminHealthPort() { - return HostAdminApplication.HOST_ADMIN_HEALT_PORT; - } - - public static NodeType getRoutingNodeType() { - return NodeType.proxy; - } - - public static ClusterId getRoutingClusterId() { - return ROUTING_CLUSTER_ID; - } - - public static ClusterSpec.Type getRoutingClusterSpecType() { - return ClusterSpec.Type.container; - } - - public static ClusterSpec.Id getRoutingClusterSpecId() { - return new ClusterSpec.Id(getRoutingClusterId().s()); - } - - public static ServiceType getRoutingServiceType() { - return ServiceType.CONTAINER; - } - - public static int getRoutingHealthPort() { - return 4088; - } - - public static boolean isNodeAdminService(ApplicationId applicationId, - ClusterId clusterId, - ServiceType serviceType) { - return Objects.equals(applicationId, getApplicationId()) && - Objects.equals(serviceType, getNodeAdminServiceType()) && - Objects.equals(clusterId, getNodeAdminClusterId()); - } - - /** Whether a {@link ServiceInfo} belongs to the zone application's node-admin cluster. */ - public static boolean isNodeAdminServiceInfo(ApplicationId applicationId, ServiceInfo serviceInfo) { - return isNodeAdminService( - applicationId, - ApplicationInstanceGenerator.getClusterId(serviceInfo), - ApplicationInstanceGenerator.toServiceType(serviceInfo)); - } - -} diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java index 7601cfd2e95..3cc7010e209 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java @@ -10,7 +10,6 @@ import com.yahoo.vespa.applicationmodel.ServiceStatus; import com.yahoo.vespa.applicationmodel.ServiceStatusInfo; import com.yahoo.vespa.applicationmodel.ServiceType; import com.yahoo.vespa.service.duper.DuperModelManager; -import com.yahoo.vespa.service.duper.ZoneApplication; import com.yahoo.vespa.service.executor.RunletExecutorImpl; import com.yahoo.vespa.service.manager.HealthMonitorApi; import com.yahoo.vespa.service.manager.MonitorManager; @@ -77,7 +76,7 @@ public class HealthMonitorManager implements MonitorManager, HealthMonitorApi { @Override public void applicationActivated(ApplicationInfo application) { - if (wouldMonitor(application.getApplicationId())) { + if (duperModel.isSupportedInfraApplication(application.getApplicationId())) { healthMonitors .computeIfAbsent(application.getApplicationId(), applicationHealthMonitorFactory::create) .monitor(application); @@ -103,24 +102,9 @@ public class HealthMonitorManager implements MonitorManager, HealthMonitorApi { return new ServiceStatusInfo(ServiceStatus.NOT_CHECKED); } - if (applicationId.equals(ZoneApplication.getApplicationId())) { - // New: The zone app is health monitored (monitor != null), possibly even the routing cluster - // which is a normal jdisc container (unnecessary but harmless), but the node-admin cluster - // are tenant Docker hosts running host admin that are monitored via /state/v1/health. - if (ZoneApplication.isNodeAdminService(applicationId, clusterId, serviceType)) { - return monitor.getStatus(applicationId, clusterId, serviceType, configId); - } else { - return new ServiceStatusInfo(ServiceStatus.NOT_CHECKED); - } - } - return monitor.getStatus(applicationId, clusterId, serviceType, configId); } - private boolean wouldMonitor(ApplicationId id) { - return duperModel.isSupportedInfraApplication(id) || id.equals(ZoneApplication.getApplicationId()); - } - @Override public List<ApplicationId> getMonitoredApplicationIds() { return Collections.list(healthMonitors.keys()); diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthModel.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthModel.java index 8e3780744f6..0408e0134ea 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthModel.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthModel.java @@ -6,14 +6,11 @@ import com.yahoo.config.model.api.HostInfo; import com.yahoo.config.model.api.PortInfo; import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.provision.HostName; -import com.yahoo.vespa.service.duper.HostAdminApplication; -import com.yahoo.vespa.service.duper.ZoneApplication; import com.yahoo.vespa.service.executor.RunletExecutor; import com.yahoo.vespa.service.model.ApplicationInstanceGenerator; import com.yahoo.vespa.service.monitor.ServiceId; import java.time.Duration; -import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.List; @@ -28,7 +25,7 @@ public class StateV1HealthModel implements AutoCloseable { private static final String PORT_TAG_HTTP = "HTTP"; /** Port tags implying /state/v1/health is served on HTTP. */ - public static final List<String> HTTP_HEALTH_PORT_TAGS = Arrays.asList(PORT_TAG_HTTP, PORT_TAG_STATE); + public static final List<String> HTTP_HEALTH_PORT_TAGS = List.of(PORT_TAG_HTTP, PORT_TAG_STATE); private final Duration targetHealthStaleness; private final Duration requestTimeout; private final Duration connectionKeepAlive; @@ -47,32 +44,16 @@ public class StateV1HealthModel implements AutoCloseable { Map<ServiceId, HealthEndpoint> extractHealthEndpoints(ApplicationInfo application) { Map<ServiceId, HealthEndpoint> endpoints = new HashMap<>(); - boolean isZoneApplication = application.getApplicationId().equals(ZoneApplication.getApplicationId()); - for (HostInfo hostInfo : application.getModel().getHosts()) { HostName hostname = HostName.from(hostInfo.getHostname()); for (ServiceInfo serviceInfo : hostInfo.getServices()) { - - boolean isNodeAdmin = false; - if (isZoneApplication) { - if (ZoneApplication.isNodeAdminServiceInfo(application.getApplicationId(), serviceInfo)) { - isNodeAdmin = true; - } else { - // Only the node admin/host admin cluster of the zone application should be monitored - // TODO: Move the node admin cluster out to a separate infrastructure application - continue; - } - } - ServiceId serviceId = ApplicationInstanceGenerator.getServiceId(application, serviceInfo); for (PortInfo portInfo : serviceInfo.getPorts()) { if (portTaggedWith(portInfo, HTTP_HEALTH_PORT_TAGS)) { - // The host-admin-in-zone-application is one big hack. - int port = isNodeAdmin ? HostAdminApplication.HOST_ADMIN_HEALT_PORT : portInfo.getPort(); StateV1HealthEndpoint endpoint = new StateV1HealthEndpoint( serviceId, hostname, - port, + portInfo.getPort(), targetHealthStaleness, requestTimeout, connectionKeepAlive, diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ApplicationInstanceGenerator.java b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ApplicationInstanceGenerator.java index e535aff8b46..5cc2d538c24 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ApplicationInstanceGenerator.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ApplicationInstanceGenerator.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.service.model; import com.yahoo.config.model.api.ApplicationInfo; import com.yahoo.config.model.api.HostInfo; import com.yahoo.config.model.api.ServiceInfo; -import com.yahoo.config.model.api.container.ContainerServiceType; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.applicationmodel.ApplicationInstance; @@ -19,7 +18,6 @@ import com.yahoo.vespa.applicationmodel.ServiceStatusInfo; import com.yahoo.vespa.applicationmodel.ServiceType; import com.yahoo.vespa.applicationmodel.TenantId; import com.yahoo.vespa.service.duper.ConfigServerApplication; -import com.yahoo.vespa.service.duper.ZoneApplication; import com.yahoo.vespa.service.monitor.ServiceId; import com.yahoo.vespa.service.monitor.ServiceStatusProvider; @@ -56,20 +54,9 @@ public class ApplicationInstanceGenerator { for (HostInfo host : applicationInfo.getModel().getHosts()) { HostName hostName = new HostName(host.getHostname()); - boolean isTenantHost = - applicationInfo.getApplicationId().equals(ZoneApplication.getApplicationId()) && - host.getServices().stream().anyMatch(serviceInfo -> - ZoneApplication.isNodeAdminServiceInfo(applicationInfo.getApplicationId(), serviceInfo)); - for (ServiceInfo serviceInfo : host.getServices()) { ServiceClusterKey serviceClusterKey = toServiceClusterKey(serviceInfo); - if (isTenantHost && !ZoneApplication.isNodeAdminServiceInfo(applicationInfo.getApplicationId(), serviceInfo)) { - // A tenant host only runs the host-admin service, even though the model contains a bunch of - // standard services like config-sentinel and metrics proxy. - continue; - } - ServiceInstance serviceInstance = toServiceInstance( applicationInfo.getApplicationId(), @@ -78,9 +65,7 @@ public class ApplicationInstanceGenerator { hostName, serviceStatusProvider); - if (!groupedServiceInstances.containsKey(serviceClusterKey)) { - groupedServiceInstances.put(serviceClusterKey, new HashSet<>()); - } + groupedServiceInstances.putIfAbsent(serviceClusterKey, new HashSet<>()); groupedServiceInstances.get(serviceClusterKey).add(serviceInstance); } } diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/duper/TestZoneApplication.java b/service-monitor/src/test/java/com/yahoo/vespa/service/duper/TestZoneApplication.java deleted file mode 100644 index 773643c1d09..00000000000 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/duper/TestZoneApplication.java +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.service.duper; - -import com.yahoo.config.model.api.ApplicationInfo; -import com.yahoo.config.model.api.HostInfo; -import com.yahoo.config.provision.HostName; - -import java.util.ArrayList; -import java.util.List; -import java.util.Objects; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -/** - * @author hakonhall - */ -public class TestZoneApplication { - - private final List<HostName> nodeAdminHostnames; - private final List<HostName> routingHostnames; - - private TestZoneApplication(List<HostName> nodeAdminHostnames, List<HostName> routingHostnames) { - this.nodeAdminHostnames = nodeAdminHostnames; - this.routingHostnames = routingHostnames; - } - - public ApplicationInfo makeApplicationInfo() { - // Make a test ApplicationInfo by: - // 1. Make an ApplicationInfo as-if the node-admin cluster of the zone application were the only cluster. - // Make sure to get the correct tenant name, application name, cluster id, service type, hostnames, - // services, and ports. This should be easy with the help of InfraApplication. - ApplicationInfo nodeAdminPart = new NodeAdminPartOfZoneApplication().makeApplicationInfo(nodeAdminHostnames); - - // 2. Make an ApplicationInfo as-if the routing cluster of the zone application were the only cluster. - // Don't care if the application is not perfect. - ApplicationInfo routingPart = new RoutingPartOfZoneApplication().makeApplicationInfo(routingHostnames); - - // 3. Take HostInfo from (1) and (2) to make a single ApplicationInfo. - List<HostInfo> allHostInfos = new ArrayList<>(); - allHostInfos.addAll(nodeAdminPart.getModel().getHosts()); - allHostInfos.addAll(routingPart.getModel().getHosts()); - - return new ApplicationInfo(nodeAdminPart.getApplicationId(), 0, new HostsModel(allHostInfos)); - } - - public static class Builder { - private List<HostName> nodeAdminHostnames = null; - private List<HostName> routingHostnames = null; - - public Builder addNodeAdminCluster(String... hostnames) { - this.nodeAdminHostnames = Stream.of(hostnames).map(HostName::from).collect(Collectors.toList()); - return this; - } - - public Builder addRoutingCluster(String... hostnames) { - this.routingHostnames = Stream.of(hostnames).map(HostName::from).collect(Collectors.toList()); - return this; - } - - public TestZoneApplication build() { - return new TestZoneApplication(Objects.requireNonNull(nodeAdminHostnames), Objects.requireNonNull(routingHostnames)); - } - } - - private static class NodeAdminPartOfZoneApplication extends InfraApplication { - public NodeAdminPartOfZoneApplication() { - super(ZoneApplication.getApplicationName().value(), - ZoneApplication.getNodeAdminNodeType(), - ZoneApplication.getNodeAdminClusterSpecType(), - ZoneApplication.getNodeAdminClusterSpecId(), - ZoneApplication.getNodeAdminServiceType(), - ZoneApplication.getNodeAdminHealthPort()); - } - } - - /** - * This InfraApplication is bogus (containing host admin instead of jdisc container), but the tests are - * not supposed to explore this cluster. - */ - private static class RoutingPartOfZoneApplication extends InfraApplication { - public RoutingPartOfZoneApplication() { - super(ZoneApplication.getApplicationName().value(), - ZoneApplication.getRoutingNodeType(), - ZoneApplication.getRoutingClusterSpecType(), - ZoneApplication.getRoutingClusterSpecId(), - ZoneApplication.getRoutingServiceType(), - ZoneApplication.getRoutingHealthPort()); - } - } -} diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/health/HealthMonitorManagerTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/health/HealthMonitorManagerTest.java index 89bcda05074..008a271f905 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/health/HealthMonitorManagerTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/health/HealthMonitorManagerTest.java @@ -3,15 +3,12 @@ package com.yahoo.vespa.service.health; import com.yahoo.config.model.api.ApplicationInfo; import com.yahoo.config.provision.HostName; -import com.yahoo.vespa.applicationmodel.ConfigId; import com.yahoo.vespa.applicationmodel.ServiceStatus; import com.yahoo.vespa.applicationmodel.ServiceStatusInfo; import com.yahoo.vespa.service.duper.ControllerHostApplication; import com.yahoo.vespa.service.duper.DuperModelManager; import com.yahoo.vespa.service.duper.InfraApplication; import com.yahoo.vespa.service.duper.ProxyHostApplication; -import com.yahoo.vespa.service.duper.TestZoneApplication; -import com.yahoo.vespa.service.duper.ZoneApplication; import com.yahoo.vespa.service.monitor.ConfigserverUtil; import org.junit.Before; import org.junit.Test; @@ -22,7 +19,6 @@ import java.util.stream.Stream; import static org.junit.Assert.assertEquals; import static org.mockito.Matchers.any; -import static org.mockito.Matchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -49,62 +45,6 @@ public class HealthMonitorManagerTest { } @Test - public void verifyZoneApplicationIsMonitored() { - ApplicationInfo zoneApplicationInfo = new TestZoneApplication.Builder() - .addNodeAdminCluster("h1", "h2") - .addRoutingCluster("r1") - .build() - .makeApplicationInfo(); - - verify(monitorFactory, times(0)).create(zoneApplicationInfo.getApplicationId()); - verify(monitor, times(0)).monitor(any()); - manager.applicationActivated(zoneApplicationInfo); - verify(monitorFactory).create(zoneApplicationInfo.getApplicationId()); - verify(monitor).monitor(any()); - - when(monitor.getStatus(any(), any(), any(), any())).thenReturn(new ServiceStatusInfo(ServiceStatus.DOWN)); - verifyNodeAdminGetStatus(0); - assertEquals(ServiceStatus.DOWN, getNodeAdminStatus()); - verifyNodeAdminGetStatus(1); - - verifyRoutingGetStatus(0); - assertEquals(ServiceStatus.NOT_CHECKED, getRoutingStatus()); - verifyRoutingGetStatus(0); - } - - private void verifyNodeAdminGetStatus(int invocations) { - verify(monitor, times(invocations)).getStatus( - eq(ZoneApplication.getApplicationId()), - eq(ZoneApplication.getNodeAdminClusterId()), - any(), - any()); - } - - private void verifyRoutingGetStatus(int invocations) { - verify(monitor, times(invocations)).getStatus( - eq(ZoneApplication.getApplicationId()), - eq(ZoneApplication.getRoutingClusterId()), - any(), - any()); - } - - private ServiceStatus getNodeAdminStatus() { - return manager.getStatus( - ZoneApplication.getApplicationId(), - ZoneApplication.getNodeAdminClusterId(), - ZoneApplication.getNodeAdminServiceType(), - new ConfigId("foo")).serviceStatus(); - } - - private ServiceStatus getRoutingStatus() { - return manager.getStatus( - ZoneApplication.getApplicationId(), - ZoneApplication.getRoutingClusterId(), - ZoneApplication.getRoutingServiceType(), - new ConfigId("bar")).serviceStatus(); - } - - @Test public void infrastructureApplication() { ProxyHostApplication proxyHostApplication = new ProxyHostApplication(); when(duperModel.isSupportedInfraApplication(proxyHostApplication.getApplicationId())).thenReturn(true); diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthModelTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthModelTest.java index 3fce1cca899..a7f632a2084 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthModelTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthModelTest.java @@ -10,8 +10,6 @@ import com.yahoo.vespa.applicationmodel.ConfigId; import com.yahoo.vespa.applicationmodel.ServiceStatus; import com.yahoo.vespa.applicationmodel.ServiceType; import com.yahoo.vespa.service.duper.ProxyHostApplication; -import com.yahoo.vespa.service.duper.TestZoneApplication; -import com.yahoo.vespa.service.duper.ZoneApplication; import com.yahoo.vespa.service.executor.Cancellable; import com.yahoo.vespa.service.executor.RunletExecutor; import com.yahoo.vespa.service.monitor.ServiceId; @@ -71,24 +69,6 @@ public class StateV1HealthModelTest { } @Test - public void testMonitoringTenantHostHealth() { - ApplicationInfo zoneApplicationInfo = new TestZoneApplication.Builder() - .addNodeAdminCluster("h1") - .addRoutingCluster("r1") - .build() - .makeApplicationInfo(); - - Map<ServiceId, HealthEndpoint> endpoints = model.extractHealthEndpoints(zoneApplicationInfo); - assertEquals(1, endpoints.size()); - HealthEndpoint endpoint = endpoints.values().iterator().next(); - assertEquals("http://h1:8080/state/v1/health", endpoint.description()); - ServiceId serviceId = endpoint.getServiceId(); - assertEquals(ZoneApplication.getApplicationId(), serviceId.getApplicationId()); - assertEquals(ZoneApplication.getNodeAdminClusterId(), serviceId.getClusterId()); - assertEquals(ZoneApplication.getNodeAdminServiceType(), serviceId.getServiceType()); - } - - @Test public void caseInsensitiveTagMatching() { PortInfo portInfo = mock(PortInfo.class); when(portInfo.getTags()).thenReturn(List.of("http", "STATE", "foo")); diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/manager/UnionMonitorManagerTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/manager/UnionMonitorManagerTest.java index 5cfe70fae5f..f6ef3977a56 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/manager/UnionMonitorManagerTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/manager/UnionMonitorManagerTest.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.service.manager; import com.yahoo.vespa.applicationmodel.ConfigId; import com.yahoo.vespa.applicationmodel.ServiceStatus; import com.yahoo.vespa.applicationmodel.ServiceStatusInfo; -import com.yahoo.vespa.service.duper.ZoneApplication; +import com.yahoo.vespa.service.duper.ConfigServerHostApplication; import com.yahoo.vespa.service.health.HealthMonitorManager; import com.yahoo.vespa.service.slobrok.SlobrokMonitorManagerImpl; import org.junit.Test; @@ -18,6 +18,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; public class UnionMonitorManagerTest { + private final ConfigServerHostApplication application = new ConfigServerHostApplication(); private final SlobrokMonitorManagerImpl slobrokMonitorManager = mock(SlobrokMonitorManagerImpl.class); private final HealthMonitorManager healthMonitorManager = mock(HealthMonitorManager.class); @@ -38,9 +39,9 @@ public class UnionMonitorManagerTest { when(healthMonitorManager.getStatus(any(), any(), any(), any())).thenReturn(new ServiceStatusInfo(healthStatus)); when(slobrokMonitorManager.getStatus(any(), any(), any(), any())).thenReturn(new ServiceStatusInfo(slobrokStatus)); ServiceStatus status = manager.getStatus( - ZoneApplication.getApplicationId(), - ZoneApplication.getNodeAdminClusterId(), - ZoneApplication.getNodeAdminServiceType(), new ConfigId("config-id")).serviceStatus(); + application.getApplicationId(), + application.getClusterId(), + application.getServiceType(), new ConfigId("config-id")).serviceStatus(); assertSame(expectedStatus, status); } }
\ No newline at end of file diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/model/ApplicationInstanceGeneratorTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/model/ApplicationInstanceGeneratorTest.java index e182c9d6468..4810f29b28f 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/model/ApplicationInstanceGeneratorTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/model/ApplicationInstanceGeneratorTest.java @@ -2,32 +2,19 @@ package com.yahoo.vespa.service.model; import com.yahoo.config.model.api.ApplicationInfo; -import com.yahoo.config.model.api.HostInfo; -import com.yahoo.config.model.api.Model; -import com.yahoo.config.model.api.ServiceInfo; -import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.applicationmodel.ApplicationInstance; -import com.yahoo.vespa.applicationmodel.ClusterId; -import com.yahoo.vespa.applicationmodel.ServiceCluster; import com.yahoo.vespa.applicationmodel.ServiceStatus; import com.yahoo.vespa.applicationmodel.ServiceStatusInfo; import com.yahoo.vespa.service.duper.ConfigServerApplication; -import com.yahoo.vespa.service.duper.ZoneApplication; import com.yahoo.vespa.service.monitor.ServiceStatusProvider; import org.junit.Test; import java.util.List; -import java.util.Map; -import java.util.Set; import java.util.stream.Collectors; -import java.util.stream.Stream; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; -import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.mockito.Matchers.any; import static org.mockito.Mockito.mock; @@ -37,10 +24,7 @@ public class ApplicationInstanceGeneratorTest { private static final String configServer1 = "cfg1.yahoo.com"; private static final String configServer2 = "cfg2.yahoo.com"; private static final String configServer3 = "cfg3.yahoo.com"; - private static final List<String> configServerList = Stream.of( - configServer1, - configServer2, - configServer3).collect(Collectors.toList()); + private static final List<String> configServerList = List.of(configServer1, configServer2, configServer3); private static final ConfigServerApplication configServerApplication = new ConfigServerApplication(); private final ServiceStatusProvider statusProvider = mock(ServiceStatusProvider.class); @@ -84,63 +68,4 @@ public class ApplicationInstanceGeneratorTest { .hostName() .toString())); } - - @Test - public void verifyOnlyNodeAdminServiceIsLeft() { - when(statusProvider.getStatus(any(), any(), any(), any())).thenReturn(new ServiceStatusInfo(ServiceStatus.NOT_CHECKED)); - - String host1 = "host1"; - String host2 = "host2"; - - List<ServiceInfo> serviceInfos1 = List.of( - makeServiceInfo("metrics", "metricsproxy-container", host1) - ); - - List<ServiceInfo> serviceInfos2 = List.of( - makeServiceInfo("metrics", "metricsproxy-container", host2), - makeServiceInfo(ZoneApplication.getNodeAdminClusterId().s(), - ZoneApplication.getNodeAdminServiceType().s(), host2) - ); - - List<HostInfo> hostInfos = List.of( - new HostInfo(host1, serviceInfos1), - new HostInfo(host2, serviceInfos2) - ); - - Model model = mock(Model.class); - when(model.getHosts()).thenReturn(hostInfos); - - ApplicationInfo applicationInfo = new ApplicationInfo(ZoneApplication.getApplicationId(), 0, model); - - Zone zone = mock(Zone.class); - when(zone.environment()).thenReturn(Environment.prod); - when(zone.region()).thenReturn(RegionName.from("us-east-1")); - - ApplicationInstanceGenerator generator = new ApplicationInstanceGenerator(applicationInfo, zone); - ApplicationInstance applicationInstance = generator.makeApplicationInstance(statusProvider); - - Map<ClusterId, List<ServiceCluster>> serviceClusters = - applicationInstance.serviceClusters().stream().collect(Collectors.groupingBy(ServiceCluster::clusterId)); - assertEquals(2, serviceClusters.size()); - List<ServiceCluster> nodeAdminClusters = serviceClusters.get(ZoneApplication.getNodeAdminClusterId()); - assertNotNull(nodeAdminClusters); - assertEquals(1, nodeAdminClusters.size()); - ServiceCluster nodeAdminCluster = nodeAdminClusters.iterator().next(); - assertEquals(1, nodeAdminCluster.serviceInstances().size()); - assertEquals(host2, nodeAdminCluster.serviceInstances().iterator().next().hostName().s()); - - List<ServiceCluster> metricsClusters = serviceClusters.get(new ClusterId("metrics")); - assertNotNull(metricsClusters); - assertEquals(1, metricsClusters.size()); - ServiceCluster metricsCluster = metricsClusters.iterator().next(); - - // The metrics service on the node admin host is ignored - assertEquals(1, metricsCluster.serviceInstances().size()); - assertEquals(host1, metricsCluster.serviceInstances().iterator().next().hostName().s()); - } - - private ServiceInfo makeServiceInfo(String clusterId, String serviceType, String hostname) { - var properties = Map.of(ApplicationInstanceGenerator.CLUSTER_ID_PROPERTY_NAME, clusterId); - return new ServiceInfo("servicename", serviceType, List.of(), properties, "configid", hostname); - } }
\ No newline at end of file diff --git a/staging_vespalib/src/vespa/vespalib/util/rusage.cpp b/staging_vespalib/src/vespa/vespalib/util/rusage.cpp index 645be2937d6..62d0158f784 100644 --- a/staging_vespalib/src/vespa/vespalib/util/rusage.cpp +++ b/staging_vespalib/src/vespa/vespalib/util/rusage.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "rusage.h" #include <stdexcept> +#include <cerrno> #include <vespa/vespalib/util/stringfmt.h> namespace vespalib { diff --git a/storage/src/tests/CMakeLists.txt b/storage/src/tests/CMakeLists.txt index 68ed987599a..7340668f70f 100644 --- a/storage/src/tests/CMakeLists.txt +++ b/storage/src/tests/CMakeLists.txt @@ -8,7 +8,6 @@ vespa_add_executable(storage_testrunner_app TEST DEPENDS storage_teststorageserver storage_testvisiting - storage_testbucketdb storage_testcommon storage_testhostreporter storage_testdistributor diff --git a/storage/src/tests/bucketdb/CMakeLists.txt b/storage/src/tests/bucketdb/CMakeLists.txt index 714faf34de5..2468e587aff 100644 --- a/storage/src/tests/bucketdb/CMakeLists.txt +++ b/storage/src/tests/bucketdb/CMakeLists.txt @@ -1,10 +1,10 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -# TODO: Remove test library when all tests have been migrated to gtest. -vespa_add_library(storage_testbucketdb TEST +vespa_add_executable(storage_bucketdb_gtest_runner_app TEST SOURCES bucketinfotest.cpp bucketmanagertest.cpp + gtest_runner.cpp initializertest.cpp judyarraytest.cpp judymultimaptest.cpp @@ -12,14 +12,6 @@ vespa_add_library(storage_testbucketdb TEST DEPENDS storage storage_testcommon -) - -vespa_add_executable(storage_bucketdb_gtest_runner_app TEST - SOURCES - gtest_runner.cpp - DEPENDS - storage - storage_testcommon gtest ) diff --git a/storage/src/tests/bucketdb/bucketinfotest.cpp b/storage/src/tests/bucketdb/bucketinfotest.cpp index 0298c50866c..fe922b5d6bd 100644 --- a/storage/src/tests/bucketdb/bucketinfotest.cpp +++ b/storage/src/tests/bucketdb/bucketinfotest.cpp @@ -1,47 +1,13 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vdstestlib/cppunit/macros.h> -#include <boost/assign.hpp> -#include <boost/random.hpp> -#include <cppunit/extensions/HelperMacros.h> -#include <map> -#include <vector> #include <vespa/vespalib/text/stringtokenizer.h> #include <vespa/storage/bucketdb/bucketinfo.h> +#include <vespa/vespalib/gtest/gtest.h> +#include <vector> + +using namespace ::testing; -namespace storage { - -namespace distributor { - -struct BucketInfoTest : public CppUnit::TestFixture { - void testBucketInfoEntriesWithNewestTimestampsAreKept(); - void testOrder(); - void testHasInvalidCopy(); - void testAddNodeSetsTrustedWhenConsistent(); - void testTrustedResetWhenCopiesBecomeInconsistent(); - void testTrustedResetWhenTrustedCopiesGoOutOfSync(); - void testTrustedNotResetWhenNonTrustedCopiesStillOutOfSync(); - void add_nodes_can_immediately_update_trusted_flag(); - void add_nodes_can_defer_update_of_trusted_flag(); - void remove_node_can_immediately_update_trusted_flag(); - void remove_node_can_defer_update_of_trusted_flag(); - - CPPUNIT_TEST_SUITE(BucketInfoTest); - CPPUNIT_TEST(testBucketInfoEntriesWithNewestTimestampsAreKept); - CPPUNIT_TEST(testOrder); - CPPUNIT_TEST(testHasInvalidCopy); - CPPUNIT_TEST(testAddNodeSetsTrustedWhenConsistent); - CPPUNIT_TEST_IGNORED(testTrustedResetWhenCopiesBecomeInconsistent); - CPPUNIT_TEST(testTrustedResetWhenTrustedCopiesGoOutOfSync); - CPPUNIT_TEST(testTrustedNotResetWhenNonTrustedCopiesStillOutOfSync); - CPPUNIT_TEST(add_nodes_can_immediately_update_trusted_flag); - CPPUNIT_TEST(add_nodes_can_defer_update_of_trusted_flag); - CPPUNIT_TEST(remove_node_can_immediately_update_trusted_flag); - CPPUNIT_TEST(remove_node_can_defer_update_of_trusted_flag); - CPPUNIT_TEST_SUITE_END(); -}; - -CPPUNIT_TEST_SUITE_REGISTRATION(BucketInfoTest); +namespace storage::distributor { BucketInfo getBucketInfo(std::string nodeList, std::string order) { @@ -83,68 +49,55 @@ nodeList(const BucketInfo& info) { // in the meantime from having their updates lost when we perform a batch // insert. This also applies for when we postpone db updates in persistence // message tracker until we've received a reply from all copies. -void -BucketInfoTest::testBucketInfoEntriesWithNewestTimestampsAreKept() -{ +TEST(BucketInfoTest, bucket_info_entries_with_newest_timestamps_are_kept) { BucketInfo bi; std::vector<uint16_t> idealState; idealState.push_back(0); bi.addNode(BucketCopy(5, 0, api::BucketInfo(1,1,1)), idealState); - CPPUNIT_ASSERT_EQUAL(api::BucketInfo(1,1,1), - bi.getNode(0)->getBucketInfo()); + EXPECT_EQ(api::BucketInfo(1,1,1), bi.getNode(0)->getBucketInfo()); bi.addNode(BucketCopy(5, 0, api::BucketInfo(2,2,2)), idealState); - CPPUNIT_ASSERT_EQUAL(api::BucketInfo(1,1,1), - bi.getNode(0)->getBucketInfo()); + EXPECT_EQ(api::BucketInfo(1,1,1), bi.getNode(0)->getBucketInfo()); bi.addNode(BucketCopy(4, 0, api::BucketInfo(3,3,3)), idealState); - CPPUNIT_ASSERT_EQUAL(api::BucketInfo(1,1,1), - bi.getNode(0)->getBucketInfo()); + EXPECT_EQ(api::BucketInfo(1,1,1), bi.getNode(0)->getBucketInfo()); bi.addNode(BucketCopy(7, 0, api::BucketInfo(4,4,4)), idealState); - CPPUNIT_ASSERT_EQUAL(api::BucketInfo(4,4,4), - bi.getNode(0)->getBucketInfo()); + EXPECT_EQ(api::BucketInfo(4,4,4), bi.getNode(0)->getBucketInfo()); bi.addNode(BucketCopy(2, 1, api::BucketInfo(4,4,4)), idealState); - CPPUNIT_ASSERT_EQUAL(api::BucketInfo(4,4,4), - bi.getNode(1)->getBucketInfo()); + EXPECT_EQ(api::BucketInfo(4,4,4), bi.getNode(1)->getBucketInfo()); } -void -BucketInfoTest::testOrder() { - - CPPUNIT_ASSERT_EQUAL(std::string("2,0,1"), nodeList(getBucketInfo("0,1,2", "2,0,1"))); - CPPUNIT_ASSERT_EQUAL(std::string("2,0,1"), nodeList(getBucketInfo("1,0,2", "2,0,1"))); - CPPUNIT_ASSERT_EQUAL(std::string("1,0,2"), nodeList(getBucketInfo("1,2,0", "1"))); - CPPUNIT_ASSERT_EQUAL(std::string("2,1,0,3,4"), nodeList(getBucketInfo("0,1,2,3,4", "2,1"))); +TEST(BucketInfoTest, node_ordering_is_preserved) { + EXPECT_EQ("2,0,1", nodeList(getBucketInfo("0,1,2", "2,0,1"))); + EXPECT_EQ("2,0,1", nodeList(getBucketInfo("1,0,2", "2,0,1"))); + EXPECT_EQ("1,0,2", nodeList(getBucketInfo("1,2,0", "1"))); + EXPECT_EQ("2,1,0,3,4", nodeList(getBucketInfo("0,1,2,3,4", "2,1"))); } -void -BucketInfoTest::testHasInvalidCopy() -{ +TEST(BucketInfoTest, can_query_for_replica_with_invalid_info) { std::vector<uint16_t> order; BucketInfo info; info.addNode(BucketCopy(0, 0, api::BucketInfo(10, 100, 1000)), order); info.addNode(BucketCopy(0, 1, api::BucketInfo(10, 100, 1000)), order); - CPPUNIT_ASSERT(!info.hasInvalidCopy()); + EXPECT_FALSE(info.hasInvalidCopy()); info.addNode(BucketCopy(0, 2, api::BucketInfo()), order); - CPPUNIT_ASSERT(info.hasInvalidCopy()); + EXPECT_TRUE(info.hasInvalidCopy()); } -void -BucketInfoTest::testAddNodeSetsTrustedWhenConsistent() -{ +TEST(BucketInfoTest, add_node_sets_trusted_when_consistent) { std::vector<uint16_t> order; { BucketInfo info; info.addNode(BucketCopy(0, 0, api::BucketInfo(0x1, 2, 144)).setTrusted(), order); info.addNode(BucketCopy(0, 1, api::BucketInfo(0x1, 2, 144)), order); - CPPUNIT_ASSERT(info.getNode(1)->trusted()); + EXPECT_TRUE(info.getNode(1)->trusted()); } { @@ -155,91 +108,78 @@ BucketInfoTest::testAddNodeSetsTrustedWhenConsistent() BucketCopy copy(1, 1, api::BucketInfo(0x1, 1, 2)); info.updateNode(copy); - CPPUNIT_ASSERT(info.getNode(1)->trusted()); - CPPUNIT_ASSERT(!info.getNode(2)->trusted()); + EXPECT_TRUE(info.getNode(1)->trusted()); + EXPECT_FALSE(info.getNode(2)->trusted()); } } -void -BucketInfoTest::testTrustedResetWhenCopiesBecomeInconsistent() -{ - CPPUNIT_FAIL("TODO: test this!"); -} - -void -BucketInfoTest::testTrustedResetWhenTrustedCopiesGoOutOfSync() -{ +TEST(BucketInfoTest, testTrustedResetWhenTrustedCopiesGoOutOfSync) { std::vector<uint16_t> order; BucketInfo info; info.addNode(BucketCopy(0, 0, api::BucketInfo(10, 100, 1000)).setTrusted(), order); info.addNode(BucketCopy(0, 1, api::BucketInfo(10, 100, 1000)), order); - CPPUNIT_ASSERT(info.getNode(0)->trusted()); - CPPUNIT_ASSERT(info.getNode(1)->trusted()); + EXPECT_TRUE(info.getNode(0)->trusted()); + EXPECT_TRUE(info.getNode(1)->trusted()); info.updateNode(BucketCopy(0, 1, api::BucketInfo(20, 200, 2000)).setTrusted()); - CPPUNIT_ASSERT(!info.getNode(0)->trusted()); - CPPUNIT_ASSERT(!info.getNode(1)->trusted()); + EXPECT_FALSE(info.getNode(0)->trusted()); + EXPECT_FALSE(info.getNode(1)->trusted()); } -void -BucketInfoTest::testTrustedNotResetWhenNonTrustedCopiesStillOutOfSync() -{ +TEST(BucketInfoTest, trusted_not_reset_when_non_trusted_copies_still_out_of_sync) { std::vector<uint16_t> order; BucketInfo info; info.addNode(BucketCopy(0, 0, api::BucketInfo(10, 100, 1000)).setTrusted(), order); info.addNode(BucketCopy(0, 1, api::BucketInfo(20, 200, 2000)), order); info.addNode(BucketCopy(0, 2, api::BucketInfo(30, 300, 3000)), order); - CPPUNIT_ASSERT(info.getNode(0)->trusted()); - CPPUNIT_ASSERT(!info.getNode(1)->trusted()); - CPPUNIT_ASSERT(!info.getNode(2)->trusted()); + EXPECT_TRUE(info.getNode(0)->trusted()); + EXPECT_FALSE(info.getNode(1)->trusted()); + EXPECT_FALSE(info.getNode(2)->trusted()); info.updateNode(BucketCopy(0, 1, api::BucketInfo(21, 201, 2001))); - CPPUNIT_ASSERT(info.getNode(0)->trusted()); - CPPUNIT_ASSERT(!info.getNode(1)->trusted()); - CPPUNIT_ASSERT(!info.getNode(2)->trusted()); + EXPECT_TRUE(info.getNode(0)->trusted()); + EXPECT_FALSE(info.getNode(1)->trusted()); + EXPECT_FALSE(info.getNode(2)->trusted()); } -void BucketInfoTest::add_nodes_can_immediately_update_trusted_flag() { +TEST(BucketInfoTest, add_nodes_can_immediately_update_trusted_flag) { BucketInfo info; std::vector<uint16_t> order; info.addNodes({BucketCopy(0, 0, api::BucketInfo(10, 100, 1000))}, order, TrustedUpdate::UPDATE); // Only one replica, so implicitly trusted iff trusted flag update is invoked. - CPPUNIT_ASSERT(info.getNode(0)->trusted()); + EXPECT_TRUE(info.getNode(0)->trusted()); } -void BucketInfoTest::add_nodes_can_defer_update_of_trusted_flag() { +TEST(BucketInfoTest, add_nodes_can_defer_update_of_trusted_flag) { BucketInfo info; std::vector<uint16_t> order; info.addNodes({BucketCopy(0, 0, api::BucketInfo(10, 100, 1000))}, order, TrustedUpdate::DEFER); - CPPUNIT_ASSERT(!info.getNode(0)->trusted()); + EXPECT_FALSE(info.getNode(0)->trusted()); } -void BucketInfoTest::remove_node_can_immediately_update_trusted_flag() { +TEST(BucketInfoTest, remove_node_can_immediately_update_trusted_flag) { BucketInfo info; std::vector<uint16_t> order; info.addNodes({BucketCopy(0, 0, api::BucketInfo(10, 100, 1000)), BucketCopy(0, 1, api::BucketInfo(20, 200, 2000))}, order, TrustedUpdate::UPDATE); - CPPUNIT_ASSERT(!info.getNode(0)->trusted()); + EXPECT_FALSE(info.getNode(0)->trusted()); info.removeNode(1, TrustedUpdate::UPDATE); // Only one replica remaining after remove, so implicitly trusted iff trusted flag update is invoked. - CPPUNIT_ASSERT(info.getNode(0)->trusted()); + EXPECT_TRUE(info.getNode(0)->trusted()); } -void BucketInfoTest::remove_node_can_defer_update_of_trusted_flag() { +TEST(BucketInfoTest, remove_node_can_defer_update_of_trusted_flag) { BucketInfo info; std::vector<uint16_t> order; info.addNodes({BucketCopy(0, 0, api::BucketInfo(10, 100, 1000)), BucketCopy(0, 1, api::BucketInfo(20, 200, 2000))}, order, TrustedUpdate::UPDATE); info.removeNode(1, TrustedUpdate::DEFER); - CPPUNIT_ASSERT(!info.getNode(0)->trusted()); -} - + EXPECT_FALSE(info.getNode(0)->trusted()); } -} // storage - +} // storage::distributor diff --git a/storage/src/tests/bucketdb/bucketmanagertest.cpp b/storage/src/tests/bucketdb/bucketmanagertest.cpp index 09fe310e97e..1f72347b7ed 100644 --- a/storage/src/tests/bucketdb/bucketmanagertest.cpp +++ b/storage/src/tests/bucketdb/bucketmanagertest.cpp @@ -1,12 +1,13 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/config/helper/configgetter.h> -#include <cppunit/extensions/HelperMacros.h> +#include <vespa/config/helper/configgetter.hpp> #include <vespa/document/config/config-documenttypes.h> #include <vespa/document/datatype/documenttype.h> #include <vespa/document/fieldvalue/document.h> #include <vespa/document/update/documentupdate.h> #include <vespa/document/repo/documenttyperepo.h> +#include <vespa/document/test/make_document_bucket.h> +#include <vespa/document/test/make_bucket_space.h> #include <vespa/storage/bucketdb/bucketmanager.h> #include <vespa/storage/common/global_bucket_space_distribution_converter.h> #include <vespa/storage/persistence/filestorage/filestormanager.h> @@ -16,13 +17,10 @@ #include <tests/common/teststorageapp.h> #include <tests/common/dummystoragelink.h> #include <tests/common/testhelper.h> -#include <vespa/document/test/make_document_bucket.h> -#include <vespa/document/test/make_bucket_space.h> #include <vespa/vdslib/state/random.h> #include <vespa/vespalib/io/fileutil.h> -#include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/util/stringfmt.h> -#include <vespa/config/helper/configgetter.hpp> +#include <vespa/vespalib/gtest/gtest.h> #include <future> #include <vespa/log/log.h> @@ -35,6 +33,7 @@ using document::DocumentType; using document::DocumentTypeRepo; using document::test::makeDocumentBucket; using document::test::makeBucketSpace; +using namespace ::testing; namespace storage { @@ -57,41 +56,8 @@ std::ostream& operator<<(std::ostream& out, const TestBucketInfo& info) { class ConcurrentOperationFixture; struct TestParams; -struct BucketManagerTest : public CppUnit::TestFixture { +struct BucketManagerTest : public Test { public: - CPPUNIT_TEST_SUITE(BucketManagerTest); - CPPUNIT_TEST(testRequestBucketInfoWithList); - CPPUNIT_TEST(testDistributionBitGenerationEmpty); - CPPUNIT_TEST(testDistributionBitChangeOnCreateBucket); - CPPUNIT_TEST(testMinUsedBitsFromComponentIsHonored); - CPPUNIT_TEST(testRemoveLastModifiedOK); - CPPUNIT_TEST(testRemoveLastModifiedFailed); - CPPUNIT_TEST(testSwallowNotifyBucketChangeReply); - CPPUNIT_TEST(testMetricsGeneration); - CPPUNIT_TEST(metrics_are_tracked_per_bucket_space); - CPPUNIT_TEST(testSplitReplyOrderedAfterBucketReply); - CPPUNIT_TEST(testJoinReplyOrderedAfterBucketReply); - CPPUNIT_TEST(testDeleteReplyOrderedAfterBucketReply); - CPPUNIT_TEST(testOnlyEnqueueWhenProcessingRequest); - CPPUNIT_TEST(testOrderRepliesAfterBucketSpecificRequest); - CPPUNIT_TEST(testQueuedRepliesOnlyDispatchedWhenAllProcessingDone); - CPPUNIT_TEST(testMutationRepliesForSplitBucketAreEnqueued); - CPPUNIT_TEST(testMutationRepliesForDeletedBucketAreEnqueued); - CPPUNIT_TEST(testMutationRepliesForJoinedBucketAreEnqueued); - CPPUNIT_TEST(testConflictingPutRepliesAreEnqueued); - CPPUNIT_TEST(testConflictingUpdateRepliesAreEnqueued); - CPPUNIT_TEST(testRemappedMutationIsCheckedAgainstOriginalBucket); - CPPUNIT_TEST(testBucketConflictSetIsClearedBetweenBlockingRequests); - CPPUNIT_TEST(testConflictSetOnlyClearedAfterAllBucketRequestsDone); - CPPUNIT_TEST(testRejectRequestWithMismatchingDistributionHash); - CPPUNIT_TEST(testDbNotIteratedWhenAllRequestsRejected); - CPPUNIT_TEST(fall_back_to_legacy_global_distribution_hash_on_mismatch); - - // FIXME(vekterli): test is not deterministic and enjoys failing - // sporadically when running under Valgrind. See bug 5932891. - CPPUNIT_TEST_IGNORED(testRequestBucketInfoWithState); - CPPUNIT_TEST_SUITE_END(); - std::unique_ptr<TestServiceLayerApp> _node; std::unique_ptr<DummyStorageLink> _top; BucketManager *_manager; @@ -101,12 +67,13 @@ public: uint32_t _emptyBuckets; document::Document::SP _document; + ~BucketManagerTest(); + void setupTestEnvironment(bool fakePersistenceLayer = true, bool noDelete = false); void addBucketsToDB(uint32_t count); bool wasBlockedDueToLastModified(api::StorageMessage* msg, uint64_t lastModified); - bool wasBlockedDueToLastModified(api::StorageMessage::SP msg); void insertSingleBucket(const document::BucketId& bucket, const api::BucketInfo& info); void waitUntilRequestsAreProcessing(size_t nRequests = 1); @@ -127,53 +94,30 @@ public: void assertRequestWithBadHashIsRejected( ConcurrentOperationFixture& fixture); +protected: + void update_min_used_bits() { + _manager->updateMinUsedBits(); + } + void trigger_metric_manager_update() { + vespalib::Monitor l; + _manager->updateMetrics(BucketManager::MetricLockGuard(l)); + } - void testRequestBucketInfoWithState(); - void testRequestBucketInfoWithList(); - void testDistributionBitGenerationEmpty(); - void testDistributionBitChangeOnCreateBucket(); - void testMinUsedBitsFromComponentIsHonored(); - - void testRemoveLastModifiedOK(); - void testRemoveLastModifiedFailed(); - - void testSwallowNotifyBucketChangeReply(); - void testMetricsGeneration(); - void metrics_are_tracked_per_bucket_space(); - void testSplitReplyOrderedAfterBucketReply(); - void testJoinReplyOrderedAfterBucketReply(); - void testDeleteReplyOrderedAfterBucketReply(); - void testOnlyEnqueueWhenProcessingRequest(); - void testOrderRepliesAfterBucketSpecificRequest(); - void testQueuedRepliesOnlyDispatchedWhenAllProcessingDone(); - void testMutationRepliesForSplitBucketAreEnqueued(); - void testMutationRepliesForDeletedBucketAreEnqueued(); - void testMutationRepliesForJoinedBucketAreEnqueued(); - void testConflictingPutRepliesAreEnqueued(); - void testConflictingUpdateRepliesAreEnqueued(); - void testRemappedMutationIsCheckedAgainstOriginalBucket(); - void testBucketConflictSetIsClearedBetweenBlockingRequests(); - void testConflictSetOnlyClearedAfterAllBucketRequestsDone(); - void testRejectRequestWithMismatchingDistributionHash(); - void testDbNotIteratedWhenAllRequestsRejected(); - void fall_back_to_legacy_global_distribution_hash_on_mismatch(); + const BucketManagerMetrics& bucket_manager_metrics() const { + return *_manager->_metrics; + } public: - static constexpr uint32_t DIR_SPREAD = 3; static constexpr uint32_t MESSAGE_WAIT_TIME = 60*2; - - void setUp() override { + void SetUp() override { _emptyBuckets = 0; } - void tearDown() override { - } - friend class ConcurrentOperationFixture; }; -CPPUNIT_TEST_SUITE_REGISTRATION(BucketManagerTest); +BucketManagerTest::~BucketManagerTest() = default; #define ASSERT_DUMMYLINK_REPLY_COUNT(link, count) \ if (link->getNumReplies() != count) { \ @@ -183,7 +127,7 @@ CPPUNIT_TEST_SUITE_REGISTRATION(BucketManagerTest); for (uint32_t i=0; i<link->getNumReplies(); ++i) { \ ost << link->getReply(i)->getType() << "\n"; \ } \ - CPPUNIT_FAIL(ost.str()); \ + FAIL() << ost.str(); \ } std::string getMkDirDisk(const std::string & rootFolder, int disk) { @@ -203,34 +147,34 @@ void BucketManagerTest::setupTestEnvironment(bool fakePersistenceLayer, assert(system(getMkDirDisk(rootFolder, 0).c_str()) == 0); assert(system(getMkDirDisk(rootFolder, 1).c_str()) == 0); - std::shared_ptr<const DocumentTypeRepo> repo(new DocumentTypeRepo( + auto repo = std::make_shared<const DocumentTypeRepo>( *ConfigGetter<DocumenttypesConfig>::getConfig( - "config-doctypes", FileSpec(TEST_PATH("config-doctypes.cfg"))))); - _top.reset(new DummyStorageLink); - _node.reset(new TestServiceLayerApp( - DiskCount(2), NodeIndex(0), config.getConfigId())); + "config-doctypes", FileSpec("../config-doctypes.cfg"))); + _top = std::make_unique<DummyStorageLink>(); + _node = std::make_unique<TestServiceLayerApp>( + DiskCount(2), NodeIndex(0), config.getConfigId()); _node->setTypeRepo(repo); _node->setupDummyPersistence(); - // Set up the 3 links - StorageLink::UP manager(new BucketManager("", _node->getComponentRegister())); - _manager = (BucketManager*) manager.get(); + // Set up the 3 links + auto manager = std::make_unique<BucketManager>("", _node->getComponentRegister()); + _manager = manager.get(); _top->push_back(std::move(manager)); if (fakePersistenceLayer) { - StorageLink::UP bottom(new DummyStorageLink); - _bottom = (DummyStorageLink*) bottom.get(); + auto bottom = std::make_unique<DummyStorageLink>(); + _bottom = bottom.get(); _top->push_back(std::move(bottom)); } else { - StorageLink::UP bottom(new FileStorManager( + auto bottom = std::make_unique<FileStorManager>( config.getConfigId(), _node->getPartitions(), - _node->getPersistenceProvider(), _node->getComponentRegister())); - _filestorManager = (FileStorManager*) bottom.get(); + _node->getPersistenceProvider(), _node->getComponentRegister()); + _filestorManager = bottom.get(); _top->push_back(std::move(bottom)); } - // Generate a doc to use for testing.. + // Generate a doc to use for testing.. const DocumentType &type(*_node->getTypeRepo() ->getDocumentType("text/html")); - _document.reset(new document::Document(type, document::DocumentId( - document::DocIdString("test", "ntnu")))); + _document = std::make_shared<document::Document>( + type, document::DocumentId(document::DocIdString("test", "ntnu"))); } void BucketManagerTest::addBucketsToDB(uint32_t count) @@ -241,7 +185,7 @@ void BucketManagerTest::addBucketsToDB(uint32_t count) while (_bucketInfo.size() < count) { document::BucketId id(16, randomizer.nextUint32()); id = id.stripUnused(); - if (_bucketInfo.size() == 0) { + if (_bucketInfo.empty()) { id = _node->getBucketIdFactory().getBucketId( _document->getId()).stripUnused(); } @@ -261,15 +205,13 @@ void BucketManagerTest::addBucketsToDB(uint32_t count) info.count = 0; info.crc = 0; ++_emptyBuckets; - for (std::map<document::BucketId, TestBucketInfo>::iterator it - = _bucketInfo.begin(); it != _bucketInfo.end(); ++it) - { + for (const auto& bi : _bucketInfo) { bucketdb::StorageBucketInfo entry; - entry.disk = it->second.partition; - entry.setBucketInfo(api::BucketInfo(it->second.crc, - it->second.count, - it->second.size)); - _node->getStorageBucketDatabase().insert(it->first, entry, "foo"); + entry.disk = bi.second.partition; + entry.setBucketInfo(api::BucketInfo(bi.second.crc, + bi.second.count, + bi.second.size)); + _node->getStorageBucketDatabase().insert(bi.first, entry, "foo"); } } @@ -293,27 +235,25 @@ BucketManagerTest::wasBlockedDueToLastModified(api::StorageMessage* msg, _top->sendDown(api::StorageMessage::SP(msg)); if (_top->getNumReplies() == 1) { - CPPUNIT_ASSERT_EQUAL(0, (int)_bottom->getNumCommands()); - CPPUNIT_ASSERT(!static_cast<api::StorageReply&>( - *_top->getReply(0)).getResult().success()); + assert(_bottom->getNumCommands() == 0); + assert(!dynamic_cast<api::StorageReply&>(*_top->getReply(0)).getResult().success()); return true; } else { - CPPUNIT_ASSERT_EQUAL(0, (int)_top->getNumReplies()); + assert(_top->getNumReplies() == 0); // Check that bucket database now has the operation's timestamp as last modified. { StorBucketDatabase::WrappedEntry entry( _node->getStorageBucketDatabase().get(id, "foo")); - CPPUNIT_ASSERT_EQUAL(lastModified, entry->info.getLastModified()); + assert(entry->info.getLastModified() == lastModified); } return false; } } -void BucketManagerTest::testRemoveLastModifiedOK() -{ - CPPUNIT_ASSERT(!wasBlockedDueToLastModified( +TEST_F(BucketManagerTest, remove_last_modified_ok) { + EXPECT_FALSE(wasBlockedDueToLastModified( new api::RemoveCommand(makeDocumentBucket(document::BucketId(16, 1)), document::DocumentId("userdoc:m:1:foo"), api::Timestamp(1235)), @@ -321,45 +261,37 @@ void BucketManagerTest::testRemoveLastModifiedOK() } -void BucketManagerTest::testRemoveLastModifiedFailed() -{ - CPPUNIT_ASSERT(wasBlockedDueToLastModified( +TEST_F(BucketManagerTest, remove_last_modified_failed) { + EXPECT_TRUE(wasBlockedDueToLastModified( new api::RemoveCommand(makeDocumentBucket(document::BucketId(16, 1)), document::DocumentId("userdoc:m:1:foo"), api::Timestamp(1233)), 1233)); } -void BucketManagerTest::testDistributionBitGenerationEmpty() -{ - TestName("BucketManagerTest::testDistributionBitGenerationEmpty()"); +TEST_F(BucketManagerTest, distribution_bit_generation_empty) { setupTestEnvironment(); _manager->doneInit(); - vespalib::Monitor l; - _manager->updateMetrics(BucketManager::MetricLockGuard(l)); - CPPUNIT_ASSERT_EQUAL(58u, _node->getStateUpdater().getReportedNodeState()->getMinUsedBits()); + trigger_metric_manager_update(); + EXPECT_EQ(58u, _node->getStateUpdater().getReportedNodeState()->getMinUsedBits()); } -void BucketManagerTest::testDistributionBitChangeOnCreateBucket() -{ - TestName("BucketManagerTest::testDistributionBitChangeOnCreateBucket()"); +TEST_F(BucketManagerTest, distribution_bit_change_on_create_bucket){ setupTestEnvironment(); addBucketsToDB(30); _top->open(); _node->getDoneInitializeHandler().notifyDoneInitializing(); _manager->doneInit(); - _manager->updateMinUsedBits(); - CPPUNIT_ASSERT_EQUAL(16u, _node->getStateUpdater().getReportedNodeState()->getMinUsedBits()); + update_min_used_bits(); + EXPECT_EQ(16u, _node->getStateUpdater().getReportedNodeState()->getMinUsedBits()); std::shared_ptr<api::CreateBucketCommand> cmd( new api::CreateBucketCommand(makeDocumentBucket(document::BucketId(4, 5678)))); _top->sendDown(cmd); - CPPUNIT_ASSERT_EQUAL(4u, _node->getStateUpdater().getReportedNodeState()->getMinUsedBits()); + EXPECT_EQ(4u, _node->getStateUpdater().getReportedNodeState()->getMinUsedBits()); } -void BucketManagerTest::testMinUsedBitsFromComponentIsHonored() -{ - TestName("BucketManagerTest::testMinUsedBitsFromComponentIsHonored()"); +TEST_F(BucketManagerTest, Min_Used_Bits_From_Component_Is_Honored) { setupTestEnvironment(); // Let these differ in order to test state update behavior. _node->getComponentRegister().getMinUsedBitsTracker().setMinUsedBits(10); @@ -377,40 +309,21 @@ void BucketManagerTest::testMinUsedBitsFromComponentIsHonored() std::shared_ptr<api::CreateBucketCommand> cmd( new api::CreateBucketCommand(makeDocumentBucket(document::BucketId(12, 5678)))); _top->sendDown(cmd); - CPPUNIT_ASSERT_EQUAL(13u, _node->getStateUpdater().getReportedNodeState()->getMinUsedBits()); + EXPECT_EQ(13u, _node->getStateUpdater().getReportedNodeState()->getMinUsedBits()); } -void BucketManagerTest::testRequestBucketInfoWithState() -{ - TestName("BucketManagerTest::testRequestBucketInfoWithState()"); - // Test prior to building bucket cache +// FIXME: non-deterministic test +TEST_F(BucketManagerTest, IGNORED_request_bucket_info_with_state) { + // Test prior to building bucket cache setupTestEnvironment(); addBucketsToDB(30); - /* Currently this is just queued up - { - std::shared_ptr<api::RequestBucketInfoCommand> cmd( - new api::RequestBucketInfoCommand( - 0, lib::ClusterState("distributor:3 .2.s:d storage:1"))); - _top->sendDown(cmd); - _top->waitForMessages(1, 5); - CPPUNIT_ASSERT_EQUAL((size_t) 1, _top->getNumReplies()); - std::shared_ptr<api::RequestBucketInfoReply> reply( - std::dynamic_pointer_cast<api::RequestBucketInfoReply>( - _top->getReply(0))); - _top->reset(); - CPPUNIT_ASSERT(reply.get()); - CPPUNIT_ASSERT_EQUAL(api::ReturnCode(api::ReturnCode::NOT_READY), - reply->getResult()); - } */ + std::vector<lib::ClusterState> states; - states.push_back(lib::ClusterState("version:0")); - states.push_back(lib::ClusterState("version:1 distributor:1 storage:1")); - states.push_back(lib::ClusterState( - "version:2 distributor:3 .1.s:i .2.s:d storage:4")); - states.push_back(lib::ClusterState( - "version:3 distributor:3 .1.s:i .2.s:d storage:4 .3.s:d")); - states.push_back(lib::ClusterState( - "version:4 distributor:3 .1.s:i .2.s:d storage:4")); + states.emplace_back("version:0"); + states.emplace_back("version:1 distributor:1 storage:1"); + states.emplace_back("version:2 distributor:3 .1.s:i .2.s:d storage:4"); + states.emplace_back("version:3 distributor:3 .1.s:i .2.s:d storage:4 .3.s:d"); + states.emplace_back("version:4 distributor:3 .1.s:i .2.s:d storage:4"); _node->setClusterState(states.back()); for (uint32_t i=0; i<states.size(); ++i) { @@ -419,11 +332,11 @@ void BucketManagerTest::testRequestBucketInfoWithState() _manager->onDown(cmd); } - // Send a request bucket info command that will be outdated and failed. + // Send a request bucket info command that will be outdated and failed. std::shared_ptr<api::RequestBucketInfoCommand> cmd1( new api::RequestBucketInfoCommand(makeBucketSpace(), 0, states[1])); - // Send two request bucket info commands that will be processed together - // when the bucket manager is idle, as states are equivalent + // Send two request bucket info commands that will be processed together + // when the bucket manager is idle, as states are equivalent std::shared_ptr<api::RequestBucketInfoCommand> cmd2( new api::RequestBucketInfoCommand(makeBucketSpace(), 0, states[2])); std::shared_ptr<api::RequestBucketInfoCommand> cmd3( @@ -457,104 +370,29 @@ void BucketManagerTest::testRequestBucketInfoWithState() std::shared_ptr<api::RequestBucketInfoReply> reply3( replies[cmd3->getMsgId()]); _top->reset(); - CPPUNIT_ASSERT(reply1.get()); - CPPUNIT_ASSERT(reply2.get()); - CPPUNIT_ASSERT(reply3.get()); - CPPUNIT_ASSERT_EQUAL(api::ReturnCode(api::ReturnCode::REJECTED, + ASSERT_TRUE(reply1.get()); + ASSERT_TRUE(reply2.get()); + ASSERT_TRUE(reply3.get()); + EXPECT_EQ(api::ReturnCode(api::ReturnCode::REJECTED, "Ignoring bucket info request for cluster state version 1 as " "versions from version 2 differs from this state."), reply1->getResult()); - CPPUNIT_ASSERT_EQUAL(api::ReturnCode(api::ReturnCode::REJECTED, + EXPECT_EQ(api::ReturnCode(api::ReturnCode::REJECTED, "There is already a newer bucket info request for " "this node from distributor 0"), reply2->getResult()); - CPPUNIT_ASSERT_EQUAL(api::ReturnCode(api::ReturnCode::OK), + EXPECT_EQ(api::ReturnCode(api::ReturnCode::OK), reply3->getResult()); api::RequestBucketInfoReply::Entry entry; - CPPUNIT_ASSERT_EQUAL((size_t) 18, reply3->getBucketInfo().size()); + ASSERT_EQ(18u, reply3->getBucketInfo().size()); entry = api::RequestBucketInfoReply::Entry( document::BucketId(16, 0xe8c8), api::BucketInfo(0x79d04f78, 11153, 1851385240u)); - CPPUNIT_ASSERT_EQUAL(entry, reply3->getBucketInfo()[0]); + EXPECT_EQ(entry, reply3->getBucketInfo()[0]); } } -namespace { - struct PopenWrapper { - FILE* _file; - std::vector<char> _buffer; - uint32_t _index; - uint32_t _size; - bool _eof; - - PopenWrapper(const std::string& cmd) - : _buffer(65536, '\0'), _index(0), _size(0), _eof(false) - { - _file = popen(cmd.c_str(), "r"); - if (_file == 0) { - throw vespalib::Exception("Failed to run '" + cmd - + "' in popen: " + strerror(errno), VESPA_STRLOC); - } - } - - const char* getNextLine() { - if (_eof && _size == 0) return 0; - // Check if we have a newline waiting - char* newline = strchr(&_buffer[_index], '\n'); - // If not try to get one - if (_eof) { - newline = &_buffer[_index + _size]; - } else if (newline == 0) { - // If we index is passed half the buffer, reposition - if (_index > _buffer.size() / 2) { - memcpy(&_buffer[0], &_buffer[_index], _size); - _index = 0; - } - // Verify we have space to write to - if (_index + _size >= _buffer.size()) { - throw vespalib::Exception("No newline could be find in " - "half the buffer size. Wrapper not designed to " - "handle that long lines (1)", VESPA_STRLOC); - } - // Fill up buffer - size_t bytesRead = fread(&_buffer[_index + _size], - 1, _buffer.size() - _index - _size - 1, - _file); - if (bytesRead == 0) { - if (!feof(_file)) { - throw vespalib::Exception("Failed to run fgets: " - + std::string(strerror(errno)), VESPA_STRLOC); - } else { - _eof = true; - } - } else { - _size += bytesRead; - } - newline = strchr(&_buffer[_index], '\n'); - if (newline == 0) { - if (_eof) { - if (_size == 0) return 0; - } else { - throw vespalib::Exception("No newline could be find in " - "half the buffer size. Wrapper not designed to " - "handle that long lines (2)", VESPA_STRLOC); - } - } - } - *newline = '\0'; - ++newline; - const char* line = &_buffer[_index]; - uint32_t strlen = (newline - line); - _index += strlen; - _size -= strlen; - return line; - } - }; -} - -void BucketManagerTest::testRequestBucketInfoWithList() -{ - TestName("BucketManagerTest::testRequestBucketInfoWithList()"); +TEST_F(BucketManagerTest, request_bucket_info_with_list) { setupTestEnvironment(); addBucketsToDB(30); _top->open(); @@ -562,39 +400,26 @@ void BucketManagerTest::testRequestBucketInfoWithList() _top->doneInit(); { std::vector<document::BucketId> bids; - bids.push_back(document::BucketId(16, 0xe8c8)); + bids.emplace_back(16, 0xe8c8); - std::shared_ptr<api::RequestBucketInfoCommand> cmd( - new api::RequestBucketInfoCommand(makeBucketSpace(), bids)); + auto cmd = std::make_shared<api::RequestBucketInfoCommand>(makeBucketSpace(), bids); _top->sendDown(cmd); _top->waitForMessages(1, 5); ASSERT_DUMMYLINK_REPLY_COUNT(_top, 1); - std::shared_ptr<api::RequestBucketInfoReply> reply( - std::dynamic_pointer_cast<api::RequestBucketInfoReply>( - _top->getReply(0))); + auto reply = std::dynamic_pointer_cast<api::RequestBucketInfoReply>(_top->getReply(0)); _top->reset(); - CPPUNIT_ASSERT(reply.get()); - CPPUNIT_ASSERT_EQUAL(api::ReturnCode(api::ReturnCode::OK), - reply->getResult()); - if (reply->getBucketInfo().size() > 1) { - std::cerr << "Too many replies found\n"; - for (uint32_t i=0; i<reply->getBucketInfo().size(); ++i) { - std::cerr << reply->getBucketInfo()[i] << "\n"; - } - } - CPPUNIT_ASSERT_EQUAL((size_t) 1, reply->getBucketInfo().size()); + ASSERT_TRUE(reply.get()); + EXPECT_EQ(api::ReturnCode(api::ReturnCode::OK), reply->getResult()); + ASSERT_EQ(1u, reply->getBucketInfo().size()); api::RequestBucketInfoReply::Entry entry( document::BucketId(16, 0xe8c8), api::BucketInfo(0x79d04f78, 11153, 1851385240u)); - CPPUNIT_ASSERT_EQUAL(entry, reply->getBucketInfo()[0]); + EXPECT_EQ(entry, reply->getBucketInfo()[0]); } } -void -BucketManagerTest::testSwallowNotifyBucketChangeReply() -{ - TestName("BucketManagerTest::testSwallowNotifyBucketChangeReply()"); +TEST_F(BucketManagerTest, swallow_notify_bucket_change_reply) { setupTestEnvironment(); addBucketsToDB(30); _top->open(); @@ -603,17 +428,14 @@ BucketManagerTest::testSwallowNotifyBucketChangeReply() api::NotifyBucketChangeCommand cmd(makeDocumentBucket(document::BucketId(1, 16)), api::BucketInfo()); - std::shared_ptr<api::NotifyBucketChangeReply> reply( - new api::NotifyBucketChangeReply(cmd)); + auto reply = std::make_shared<api::NotifyBucketChangeReply>(cmd); _top->sendDown(reply); // Should not leave the bucket manager. - CPPUNIT_ASSERT_EQUAL(0, (int)_bottom->getNumCommands()); + EXPECT_EQ(0u, _bottom->getNumCommands()); } -void -BucketManagerTest::testMetricsGeneration() -{ +TEST_F(BucketManagerTest, metrics_generation) { setupTestEnvironment(); _top->open(); // Add 3 buckets; 2 ready, 1 active. 300 docs total, 600 bytes total. @@ -633,19 +455,18 @@ BucketManagerTest::testMetricsGeneration() } _node->getDoneInitializeHandler().notifyDoneInitializing(); _top->doneInit(); - vespalib::Monitor l; - _manager->updateMetrics(BucketManager::MetricLockGuard(l)); - - CPPUNIT_ASSERT_EQUAL(size_t(2), _manager->_metrics->disks.size()); - const DataStoredMetrics& m(*_manager->_metrics->disks[0]); - CPPUNIT_ASSERT_EQUAL(int64_t(3), m.buckets.getLast()); - CPPUNIT_ASSERT_EQUAL(int64_t(300), m.docs.getLast()); - CPPUNIT_ASSERT_EQUAL(int64_t(600), m.bytes.getLast()); - CPPUNIT_ASSERT_EQUAL(int64_t(1), m.active.getLast()); - CPPUNIT_ASSERT_EQUAL(int64_t(2), m.ready.getLast()); + trigger_metric_manager_update(); + + ASSERT_EQ(2u, bucket_manager_metrics().disks.size()); + const DataStoredMetrics& m(*bucket_manager_metrics().disks[0]); + EXPECT_EQ(3, m.buckets.getLast()); + EXPECT_EQ(300, m.docs.getLast()); + EXPECT_EQ(600, m.bytes.getLast()); + EXPECT_EQ(1, m.active.getLast()); + EXPECT_EQ(2, m.ready.getLast()); } -void BucketManagerTest::metrics_are_tracked_per_bucket_space() { +TEST_F(BucketManagerTest, metrics_are_tracked_per_bucket_space) { setupTestEnvironment(); _top->open(); auto& repo = _node->getComponentRegister().getBucketSpaceRepo(); @@ -669,25 +490,24 @@ void BucketManagerTest::metrics_are_tracked_per_bucket_space() { } _node->getDoneInitializeHandler().notifyDoneInitializing(); _top->doneInit(); - vespalib::Monitor l; - _manager->updateMetrics(BucketManager::MetricLockGuard(l)); + trigger_metric_manager_update(); - auto& spaces = _manager->_metrics->bucket_spaces; + auto& spaces = bucket_manager_metrics().bucket_spaces; auto default_m = spaces.find(document::FixedBucketSpaces::default_space()); - CPPUNIT_ASSERT(default_m != spaces.end()); - CPPUNIT_ASSERT_EQUAL(int64_t(1), default_m->second->buckets_total.getLast()); - CPPUNIT_ASSERT_EQUAL(int64_t(100), default_m->second->docs.getLast()); - CPPUNIT_ASSERT_EQUAL(int64_t(200), default_m->second->bytes.getLast()); - CPPUNIT_ASSERT_EQUAL(int64_t(0), default_m->second->active_buckets.getLast()); - CPPUNIT_ASSERT_EQUAL(int64_t(1), default_m->second->ready_buckets.getLast()); + ASSERT_TRUE(default_m != spaces.end()); + EXPECT_EQ(1, default_m->second->buckets_total.getLast()); + EXPECT_EQ(100, default_m->second->docs.getLast()); + EXPECT_EQ(200, default_m->second->bytes.getLast()); + EXPECT_EQ(0, default_m->second->active_buckets.getLast()); + EXPECT_EQ(1, default_m->second->ready_buckets.getLast()); auto global_m = spaces.find(document::FixedBucketSpaces::global_space()); - CPPUNIT_ASSERT(global_m != spaces.end()); - CPPUNIT_ASSERT_EQUAL(int64_t(1), global_m->second->buckets_total.getLast()); - CPPUNIT_ASSERT_EQUAL(int64_t(150), global_m->second->docs.getLast()); - CPPUNIT_ASSERT_EQUAL(int64_t(300), global_m->second->bytes.getLast()); - CPPUNIT_ASSERT_EQUAL(int64_t(1), global_m->second->active_buckets.getLast()); - CPPUNIT_ASSERT_EQUAL(int64_t(0), global_m->second->ready_buckets.getLast()); + ASSERT_TRUE(global_m != spaces.end()); + EXPECT_EQ(1, global_m->second->buckets_total.getLast()); + EXPECT_EQ(150, global_m->second->docs.getLast()); + EXPECT_EQ(300, global_m->second->bytes.getLast()); + EXPECT_EQ(1, global_m->second->active_buckets.getLast()); + EXPECT_EQ(0, global_m->second->ready_buckets.getLast()); } void @@ -725,7 +545,7 @@ struct WithBuckets { class ConcurrentOperationFixture { public: - ConcurrentOperationFixture(BucketManagerTest& self) + explicit ConcurrentOperationFixture(BucketManagerTest& self) : _self(self), _state("distributor:1 storage:1") { @@ -835,21 +655,20 @@ public: { const size_t nTotal = nBucketReplies + 1; auto replies = awaitAndGetReplies(nTotal); - CPPUNIT_ASSERT_EQUAL(nTotal, replies.size()); + ASSERT_EQ(nTotal, replies.size()); for (size_t i = 0; i < nBucketReplies; ++i) { - CPPUNIT_ASSERT_EQUAL(api::MessageType::REQUESTBUCKETINFO_REPLY, - replies[i]->getType()); + ASSERT_EQ(api::MessageType::REQUESTBUCKETINFO_REPLY, replies[i]->getType()); } - CPPUNIT_ASSERT_EQUAL(msgType, replies[nBucketReplies]->getType()); + ASSERT_EQ(msgType, replies[nBucketReplies]->getType()); } void assertReplyOrdering( const std::vector<const api::MessageType*>& replyTypes) { auto replies = awaitAndGetReplies(replyTypes.size()); - CPPUNIT_ASSERT_EQUAL(replyTypes.size(), replies.size()); + ASSERT_EQ(replyTypes.size(), replies.size()); for (size_t i = 0; i < replyTypes.size(); ++i) { - CPPUNIT_ASSERT_EQUAL(*replyTypes[i], replies[i]->getType()); + ASSERT_EQ(*replyTypes[i], replies[i]->getType()); } } @@ -901,9 +720,7 @@ private: lib::ClusterState _state; }; -void -BucketManagerTest::testSplitReplyOrderedAfterBucketReply() -{ +TEST_F(BucketManagerTest, split_reply_ordered_after_bucket_reply) { ConcurrentOperationFixture fixture(*this); document::BucketId bucketA(17, 0); document::BucketId bucketB(17, 1); @@ -924,9 +741,7 @@ BucketManagerTest::testSplitReplyOrderedAfterBucketReply() 1, api::MessageType::SPLITBUCKET_REPLY); } -void -BucketManagerTest::testJoinReplyOrderedAfterBucketReply() -{ +TEST_F(BucketManagerTest, join_reply_ordered_after_bucket_reply) { ConcurrentOperationFixture fixture(*this); document::BucketId bucketA(17, 0); document::BucketId bucketB(17, 1 << 16); @@ -949,9 +764,7 @@ BucketManagerTest::testJoinReplyOrderedAfterBucketReply() // Technically, deletes being ordered after bucket info replies won't help // correctness since buckets are removed from the distributor DB upon _sending_ // the delete and not receiving it. -void -BucketManagerTest::testDeleteReplyOrderedAfterBucketReply() -{ +TEST_F(BucketManagerTest, delete_reply_ordered_after_bucket_reply) { ConcurrentOperationFixture fixture(*this); document::BucketId bucketA(17, 0); document::BucketId bucketB(17, 1); @@ -970,9 +783,7 @@ BucketManagerTest::testDeleteReplyOrderedAfterBucketReply() 1, api::MessageType::DELETEBUCKET_REPLY); } -void -BucketManagerTest::testOnlyEnqueueWhenProcessingRequest() -{ +TEST_F(BucketManagerTest, only_enqueue_when_processing_request) { ConcurrentOperationFixture fixture(*this); document::BucketId bucketA(17, 0); fixture.setUp(WithBuckets() @@ -990,9 +801,7 @@ BucketManagerTest::testOnlyEnqueueWhenProcessingRequest() // differently than full bucket info fetches and are not delegated to the // worker thread. We still require that any split/joins etc are ordered after // this reply if their reply is sent up concurrently. -void -BucketManagerTest::testOrderRepliesAfterBucketSpecificRequest() -{ +TEST_F(BucketManagerTest, order_replies_after_bucket_specific_request) { ConcurrentOperationFixture fixture(*this); document::BucketId bucketA(17, 0); fixture.setUp(WithBuckets() @@ -1025,14 +834,12 @@ BucketManagerTest::testOrderRepliesAfterBucketSpecificRequest() 1, api::MessageType::SPLITBUCKET_REPLY); } -// Test is similar to testOrderRepliesAfterBucketSpecificRequest, but has +// Test is similar to order_replies_after_bucket_specific_request, but has // two concurrent bucket info request processing instances going on; one in // the worker thread and one in the message chain itself. Since we only have // one queue, we must wait with dispatching replies until _all_ processing // has ceased. -void -BucketManagerTest::testQueuedRepliesOnlyDispatchedWhenAllProcessingDone() -{ +TEST_F(BucketManagerTest, queued_replies_only_dispatched_when_all_processing_done) { ConcurrentOperationFixture fixture(*this); document::BucketId bucketA(17, 0); fixture.setUp(WithBuckets() @@ -1085,9 +892,9 @@ struct TestParams { BUILDER_PARAM(std::vector<const api::MessageType*>, expectedOrdering); }; -TestParams::TestParams() { } +TestParams::TestParams() = default; TestParams::TestParams(const TestParams &) = default; -TestParams::~TestParams() {} +TestParams::~TestParams() = default; void BucketManagerTest::doTestMutationOrdering( @@ -1140,9 +947,7 @@ BucketManagerTest::doTestConflictingReplyIsEnqueued( doTestMutationOrdering(fixture, params); } -void -BucketManagerTest::testMutationRepliesForSplitBucketAreEnqueued() -{ +TEST_F(BucketManagerTest, mutation_replies_for_split_bucket_are_enqueued) { document::BucketId bucket(17, 0); doTestConflictingReplyIsEnqueued( bucket, @@ -1150,9 +955,7 @@ BucketManagerTest::testMutationRepliesForSplitBucketAreEnqueued() api::MessageType::SPLITBUCKET_REPLY); } -void -BucketManagerTest::testMutationRepliesForDeletedBucketAreEnqueued() -{ +TEST_F(BucketManagerTest, mutation_replies_for_deleted_bucket_are_enqueued) { document::BucketId bucket(17, 0); doTestConflictingReplyIsEnqueued( bucket, @@ -1160,9 +963,7 @@ BucketManagerTest::testMutationRepliesForDeletedBucketAreEnqueued() api::MessageType::DELETEBUCKET_REPLY); } -void -BucketManagerTest::testMutationRepliesForJoinedBucketAreEnqueued() -{ +TEST_F(BucketManagerTest, mutation_replies_for_joined_bucket_are_enqueued) { ConcurrentOperationFixture fixture(*this); document::BucketId bucketA(17, 0); document::BucketId bucketB(17, 1 << 16); @@ -1183,9 +984,7 @@ BucketManagerTest::testMutationRepliesForJoinedBucketAreEnqueued() doTestMutationOrdering(fixture, params); } -void -BucketManagerTest::testConflictingPutRepliesAreEnqueued() -{ +TEST_F(BucketManagerTest, conflicting_put_replies_are_enqueued) { ConcurrentOperationFixture fixture(*this); document::BucketId bucket(17, 0); @@ -1200,9 +999,7 @@ BucketManagerTest::testConflictingPutRepliesAreEnqueued() doTestMutationOrdering(fixture, params); } -void -BucketManagerTest::testConflictingUpdateRepliesAreEnqueued() -{ +TEST_F(BucketManagerTest, conflicting_update_replies_are_enqueued) { ConcurrentOperationFixture fixture(*this); document::BucketId bucket(17, 0); @@ -1223,9 +1020,7 @@ BucketManagerTest::testConflictingUpdateRepliesAreEnqueued() * resulting from the operation. We have to make sure remapped operations are * enqueued as well. */ -void -BucketManagerTest::testRemappedMutationIsCheckedAgainstOriginalBucket() -{ +TEST_F(BucketManagerTest, remapped_mutation_is_checked_against_original_bucket) { ConcurrentOperationFixture fixture(*this); document::BucketId bucket(17, 0); document::BucketId remappedToBucket(18, 0); @@ -1263,9 +1058,7 @@ BucketManagerTest::scheduleBucketInfoRequestWithConcurrentOps( guard.unlock(); } -void -BucketManagerTest::testBucketConflictSetIsClearedBetweenBlockingRequests() -{ +TEST_F(BucketManagerTest, bucket_conflict_set_is_cleared_between_blocking_requests) { ConcurrentOperationFixture fixture(*this); document::BucketId firstConflictBucket(17, 0); document::BucketId secondConflictBucket(18, 0); @@ -1308,9 +1101,7 @@ BucketManagerTest::sendSingleBucketInfoRequest(const document::BucketId& id) _top->sendDown(infoCmd); } -void -BucketManagerTest::testConflictSetOnlyClearedAfterAllBucketRequestsDone() -{ +TEST_F(BucketManagerTest, conflict_set_only_cleared_after_all_bucket_requests_done) { ConcurrentOperationFixture fixture(*this); document::BucketId bucketA(16, 0); document::BucketId bucketB(16, 1); @@ -1371,22 +1162,17 @@ BucketManagerTest::assertRequestWithBadHashIsRejected( _top->sendDown(infoCmd); auto replies = fixture.awaitAndGetReplies(1); auto& reply = dynamic_cast<api::RequestBucketInfoReply&>(*replies[0]); - CPPUNIT_ASSERT_EQUAL(api::ReturnCode::REJECTED, - reply.getResult().getResult()); + ASSERT_EQ(api::ReturnCode::REJECTED, reply.getResult().getResult()); } -void -BucketManagerTest::testRejectRequestWithMismatchingDistributionHash() -{ +TEST_F(BucketManagerTest, reject_request_with_mismatching_distribution_hash) { ConcurrentOperationFixture fixture(*this); document::BucketId bucket(17, 0); fixture.setUp(WithBuckets().add(bucket, api::BucketInfo(50, 100, 200))); assertRequestWithBadHashIsRejected(fixture); } -void -BucketManagerTest::testDbNotIteratedWhenAllRequestsRejected() -{ +TEST_F(BucketManagerTest, db_not_iterated_when_all_requests_rejected) { ConcurrentOperationFixture fixture(*this); document::BucketId bucket(17, 0); fixture.setUp(WithBuckets().add(bucket, api::BucketInfo(50, 100, 200))); @@ -1405,7 +1191,7 @@ BucketManagerTest::testDbNotIteratedWhenAllRequestsRejected() } // TODO remove on Vespa 8 - this is a workaround for https://github.com/vespa-engine/vespa/issues/8475 -void BucketManagerTest::fall_back_to_legacy_global_distribution_hash_on_mismatch() { +TEST_F(BucketManagerTest, fall_back_to_legacy_global_distribution_hash_on_mismatch) { ConcurrentOperationFixture f(*this); f.set_grouped_distribution_configs(); @@ -1416,7 +1202,7 @@ void BucketManagerTest::fall_back_to_legacy_global_distribution_hash_on_mismatch _top->sendDown(infoCmd); auto replies = f.awaitAndGetReplies(1); auto& reply = dynamic_cast<api::RequestBucketInfoReply&>(*replies[0]); - CPPUNIT_ASSERT_EQUAL(api::ReturnCode::OK, reply.getResult().getResult()); // _not_ REJECTED + EXPECT_EQ(api::ReturnCode::OK, reply.getResult().getResult()); // _not_ REJECTED } } // storage diff --git a/storage/src/tests/bucketdb/initializertest.cpp b/storage/src/tests/bucketdb/initializertest.cpp index 2141dbf4b53..57bb3a865d5 100644 --- a/storage/src/tests/bucketdb/initializertest.cpp +++ b/storage/src/tests/bucketdb/initializertest.cpp @@ -2,33 +2,31 @@ /** * Tests storage initialization without depending on persistence layer. */ -#include <vespa/storage/bucketdb/storagebucketdbinitializer.h> - #include <vespa/document/base/testdocman.h> - +#include <vespa/document/bucket/fixed_bucket_spaces.h> +#include <vespa/storage/bucketdb/lockablemap.hpp> +#include <vespa/storage/bucketdb/storagebucketdbinitializer.h> #include <vespa/storage/persistence/filestorage/filestormanager.h> #include <vespa/storageapi/message/bucket.h> #include <vespa/storageapi/message/persistence.h> #include <vespa/storageapi/message/state.h> #include <tests/common/teststorageapp.h> #include <tests/common/dummystoragelink.h> -#include <tests/common/testhelper.h> -#include <vespa/vdstestlib/cppunit/dirconfig.h> -#include <vespa/vdstestlib/cppunit/macros.h> -#include <vespa/storage/bucketdb/lockablemap.hpp> -#include <vespa/vdstestlib/cppunit/dirconfig.hpp> -#include <vespa/document/bucket/fixed_bucket_spaces.h> +#include <tests/common/testhelper.h> // TODO decouple from CppUnit +#include <vespa/vdstestlib/cppunit/dirconfig.hpp> // TODO decouple from CppUnit +#include <vespa/vespalib/gtest/gtest.h> #include <vespa/log/log.h> LOG_SETUP(".test.bucketdb.initializing"); using document::FixedBucketSpaces; +using namespace ::testing; namespace storage { typedef uint16_t PartitionId; -struct InitializerTest : public CppUnit::TestFixture { +struct InitializerTest : public Test { class InitParams { vdstestlib::DirConfig config; @@ -59,14 +57,8 @@ struct InitializerTest : public CppUnit::TestFixture { bucketWrongDisk(false), bucketMultipleDisks(false), failingListRequest(false), - failingInfoRequest(false) {} - - void setAllFailures() { - bucketWrongDisk = true; - bucketMultipleDisks = true; - failingListRequest = true; - failingInfoRequest = true; - } + failingInfoRequest(false) + {} vdstestlib::DirConfig& getConfig() { if (!configFinalized) { @@ -83,104 +75,44 @@ struct InitializerTest : public CppUnit::TestFixture { document::TestDocMan _docMan; - void testInitialization(InitParams& params); + void do_test_initialization(InitParams& params); +}; - /** - * Test that the status page can be shown during init without a deadlock - * or crash or anything. Don't validate much output, it might change. - */ - void testStatusPage(); +TEST_F(InitializerTest, init_with_empty_node) { + InitParams params; + params.docsPerDisk = 0; + do_test_initialization(params); +} - /** Test initializing with an empty node. */ - void testInitEmptyNode() { - InitParams params; - params.docsPerDisk = 0; - testInitialization(params); - } - /** Test initializing with some data on single disk. */ - void testInitSingleDisk() { - InitParams params; - params.diskCount = DiskCount(1); - testInitialization(params); - } - /** Test initializing with multiple disks. */ - void testInitMultiDisk() { - InitParams params; - testInitialization(params); - } - /** Test initializing with one of the disks being bad. */ - void testInitFailingMiddleDisk() { - InitParams params; - params.disksDown.insert(1); - testInitialization(params); - } - /** Test initializing with last disk being bad. */ - void testInitFailingLastDisk() { - InitParams params; - params.disksDown.insert(params.diskCount - 1); - testInitialization(params); - } - /** Test initializing with bucket on wrong disk. */ - void testInitBucketOnWrongDisk() { - InitParams params; - params.bucketWrongDisk = true; - params.bucketBitsUsed = 58; - testInitialization(params); - } - /** Test initializing with bucket on multiple disks. */ - void testInitBucketOnMultipleDisks() { - InitParams params; - params.bucketMultipleDisks = true; - params.bucketBitsUsed = 58; - testInitialization(params); - } - /** Test initializing with failing list request. */ - void testInitFailingListRequest() { - InitParams params; - params.failingListRequest = true; - testInitialization(params); - } - void testInitFailingInfoRequest() { - InitParams params; - params.failingInfoRequest = true; - testInitialization(params); - } - /** Test initializing with everything being wrong at once. */ - void testAllFailures() { - InitParams params; - params.docsPerDisk = 100; - params.diskCount = DiskCount(10); - params.disksDown.insert(0); - params.disksDown.insert(2); - params.disksDown.insert(3); - params.disksDown.insert(9); - params.setAllFailures(); - testInitialization(params); - } - void testCommandBlockingDuringInit(); - - void testBucketProgressCalculator(); - - void testBucketsInitializedByLoad(); - - CPPUNIT_TEST_SUITE(InitializerTest); - CPPUNIT_TEST(testInitEmptyNode); - CPPUNIT_TEST(testInitSingleDisk); - CPPUNIT_TEST(testInitMultiDisk); - CPPUNIT_TEST(testInitFailingMiddleDisk); - CPPUNIT_TEST(testInitFailingLastDisk); - CPPUNIT_TEST(testInitBucketOnWrongDisk); - //CPPUNIT_TEST(testInitBucketOnMultipleDisks); - //CPPUNIT_TEST(testStatusPage); - //CPPUNIT_TEST(testCommandBlockingDuringInit); - //CPPUNIT_TEST(testAllFailures); - CPPUNIT_TEST(testBucketProgressCalculator); - CPPUNIT_TEST(testBucketsInitializedByLoad); - CPPUNIT_TEST_SUITE_END(); +TEST_F(InitializerTest, init_with_data_on_single_disk) { + InitParams params; + params.diskCount = DiskCount(1); + do_test_initialization(params); +} -}; +TEST_F(InitializerTest, init_with_multiple_disks) { + InitParams params; + do_test_initialization(params); +} + +TEST_F(InitializerTest, init_with_bad_non_last_disk) { + InitParams params; + params.disksDown.insert(1); + do_test_initialization(params); +} + +TEST_F(InitializerTest, init_with_bad_last_disk) { + InitParams params; + params.disksDown.insert(params.diskCount - 1); + do_test_initialization(params); +} -CPPUNIT_TEST_SUITE_REGISTRATION(InitializerTest); +TEST_F(InitializerTest, init_with_bucket_on_wrong_disk) { + InitParams params; + params.bucketWrongDisk = true; + params.bucketBitsUsed = 58; + do_test_initialization(params); +} namespace { // Data kept on buckets we're using in test. @@ -202,7 +134,7 @@ struct BucketData { return copy; } }; -// Data reciding on one disk +// Data residing on one disk typedef std::map<document::BucketId, BucketData> DiskData; struct BucketInfoLogger { std::map<PartitionId, DiskData>& map; @@ -215,11 +147,8 @@ struct BucketInfoLogger { { document::BucketId bucket( document::BucketId::keyToBucketId(revBucket)); - CPPUNIT_ASSERT(bucket.getRawId() != 0); - CPPUNIT_ASSERT_MSG( - "Found invalid bucket in database: " + bucket.toString() - + " " + entry.getBucketInfo().toString(), - entry.getBucketInfo().valid()); + assert(bucket.getRawId() != 0); + assert(entry.getBucketInfo().valid()); DiskData& ddata(map[entry.disk]); BucketData& bdata(ddata[bucket]); bdata.info = entry.getBucketInfo(); @@ -277,10 +206,10 @@ buildBucketInfo(const document::TestDocMan& docMan, while (params.disksDown.find(partition) != params.disksDown.end()) { partition = (partition + 1) % params.diskCount;; } - LOG(info, "Putting bucket %s on wrong disk %u instead of %u", + LOG(debug, "Putting bucket %s on wrong disk %u instead of %u", bid.toString().c_str(), partition, correctPart); } - LOG(info, "Putting bucket %s on disk %u", + LOG(debug, "Putting bucket %s on disk %u", bid.toString().c_str(), partition); BucketData& data(result[partition][bid]); data.info.setDocumentCount(data.info.getDocumentCount() + 1); @@ -299,84 +228,65 @@ void verifyEqual(std::map<PartitionId, DiskData>& org, while (part1 != org.end() && part2 != existing.end()) { if (part1->first < part2->first) { if (!part1->second.empty()) { - std::ostringstream ost; - ost << "No data in partition " << part1->first << " found."; - CPPUNIT_FAIL(ost.str()); + FAIL() << "No data in partition " << part1->first << " found."; } ++part1; } else if (part1->first > part2->first) { if (!part2->second.empty()) { - std::ostringstream ost; - ost << "Found data in partition " << part2->first - << " which should not exist."; - CPPUNIT_FAIL(ost.str()); + FAIL() << "Found data in partition " << part2->first + << " which should not exist."; } ++part2; } else { - DiskData::const_iterator bucket1(part1->second.begin()); - DiskData::const_iterator bucket2(part2->second.begin()); + auto bucket1 = part1->second.begin(); + auto bucket2 = part2->second.begin(); while (bucket1 != part1->second.end() && bucket2 != part2->second.end()) { if (bucket1->first < bucket2->first) { - std::ostringstream ost; - ost << "No data in partition " << part1->first - << " for bucket " << bucket1->first << " found."; - CPPUNIT_FAIL(ost.str()); + FAIL() << "No data in partition " << part1->first + << " for bucket " << bucket1->first << " found."; } else if (bucket1->first.getId() > bucket2->first.getId()) { - std::ostringstream ost; - ost << "Found data in partition " << part2->first - << " for bucket " << bucket2->first - << " which should not exist."; - CPPUNIT_FAIL(ost.str()); + FAIL() << "Found data in partition " << part2->first + << " for bucket " << bucket2->first + << " which should not exist."; } else if (!(bucket1->second.info == bucket2->second.info)) { - std::ostringstream ost; - ost << "Bucket " << bucket1->first << " on partition " - << part1->first << " has bucket info " - << bucket2->second.info << " and not " - << bucket1->second.info << " as expected."; - CPPUNIT_FAIL(ost.str()); + FAIL() << "Bucket " << bucket1->first << " on partition " + << part1->first << " has bucket info " + << bucket2->second.info << " and not " + << bucket1->second.info << " as expected."; } ++bucket1; ++bucket2; ++equalCount; } if (bucket1 != part1->second.end()) { - std::ostringstream ost; - ost << "No data in partition " << part1->first - << " for bucket " << bucket1->first << " found."; - CPPUNIT_FAIL(ost.str()); + FAIL() << "No data in partition " << part1->first + << " for bucket " << bucket1->first << " found."; } if (bucket2 != part2->second.end()) { - std::ostringstream ost; - ost << "Found data in partition " << part2->first - << " for bucket " << bucket2->first - << " which should not exist."; - CPPUNIT_FAIL(ost.str()); + FAIL() << "Found data in partition " << part2->first + << " for bucket " << bucket2->first + << " which should not exist."; } ++part1; ++part2; } } if (part1 != org.end() && !part1->second.empty()) { - std::ostringstream ost; - ost << "No data in partition " << part1->first << " found."; - CPPUNIT_FAIL(ost.str()); + FAIL() << "No data in partition " << part1->first << " found."; } if (part2 != existing.end() && !part2->second.empty()) { - std::ostringstream ost; - ost << "Found data in partition " << part2->first - << " which should not exist."; - CPPUNIT_FAIL(ost.str()); + FAIL() << "Found data in partition " << part2->first + << " which should not exist."; } - //std::cerr << "\n " << equalCount << " buckets were matched. "; } struct MessageCallback { public: - virtual ~MessageCallback() {} + virtual ~MessageCallback() = default; virtual void onMessage(const api::StorageMessage&) = 0; }; @@ -413,7 +323,7 @@ struct FakePersistenceLayer : public StorageLink { << "it there."; fatal(ost.str()); } else { - DiskData::const_iterator it2(it->second.find(bucket)); + auto it2 = it->second.find(bucket); if (it2 == it->second.end()) { std::ostringstream ost; ost << "Have no data for " << bucket << " on disk " << partition @@ -433,10 +343,9 @@ struct FakePersistenceLayer : public StorageLink { messageCallback->onMessage(*msg); } if (msg->getType() == api::MessageType::INTERNAL) { - api::InternalCommand& cmd( - dynamic_cast<api::InternalCommand&>(*msg)); + auto& cmd = dynamic_cast<api::InternalCommand&>(*msg); if (cmd.getType() == ReadBucketList::ID) { - ReadBucketList& rbl(dynamic_cast<ReadBucketList&>(cmd)); + auto& rbl = dynamic_cast<ReadBucketList&>(cmd); ReadBucketListReply::SP reply(new ReadBucketListReply(rbl)); std::map<PartitionId, DiskData>::const_iterator it( data.find(rbl.getPartition())); @@ -448,10 +357,8 @@ struct FakePersistenceLayer : public StorageLink { fatal(ost.str()); } else { if (cmd.getBucket().getBucketSpace() == FixedBucketSpaces::default_space()) { - for (DiskData::const_iterator it2 = it->second.begin(); - it2 != it->second.end(); ++it2) - { - reply->getBuckets().push_back(it2->first); + for (const auto& bd : it->second) { + reply->getBuckets().push_back(bd.first); } } } @@ -461,7 +368,7 @@ struct FakePersistenceLayer : public StorageLink { } sendUp(reply); } else if (cmd.getType() == ReadBucketInfo::ID) { - ReadBucketInfo& rbi(dynamic_cast<ReadBucketInfo&>(cmd)); + auto& rbi = dynamic_cast<ReadBucketInfo&>(cmd); ReadBucketInfoReply::SP reply(new ReadBucketInfoReply(rbi)); StorBucketDatabase::WrappedEntry entry( bucketDatabase.get(rbi.getBucketId(), "fakelayer")); @@ -483,8 +390,7 @@ struct FakePersistenceLayer : public StorageLink { } sendUp(reply); } else if (cmd.getType() == InternalBucketJoinCommand::ID) { - InternalBucketJoinCommand& ibj( - dynamic_cast<InternalBucketJoinCommand&>(cmd)); + auto& ibj = dynamic_cast<InternalBucketJoinCommand&>(cmd); InternalBucketJoinReply::SP reply( new InternalBucketJoinReply(ibj)); StorBucketDatabase::WrappedEntry entry( @@ -521,20 +427,14 @@ struct FakePersistenceLayer : public StorageLink { } // end of anonymous namespace -#define CPPUNIT_ASSERT_METRIC_SET(x) \ - CPPUNIT_ASSERT(initializer->getMetrics().x.getValue() > 0); - void -InitializerTest::testInitialization(InitParams& params) +InitializerTest::do_test_initialization(InitParams& params) { std::map<PartitionId, DiskData> data(buildBucketInfo(_docMan, params)); spi::PartitionStateList partitions(params.diskCount); - for (std::set<uint32_t>::const_iterator it = params.disksDown.begin(); - it != params.disksDown.end(); ++it) - { - partitions[*it] = spi::PartitionState( - spi::PartitionState::DOWN, "Set down in test"); + for (const auto& p : params.disksDown) { + partitions[p] = spi::PartitionState(spi::PartitionState::DOWN, "Set down in test"); } TestServiceLayerApp node(params.diskCount, params.nodeIndex, params.getConfig().getConfigId()); @@ -549,233 +449,32 @@ InitializerTest::testInitialization(InitParams& params) top.push_back(StorageLink::UP(bottom = new FakePersistenceLayer( data, node.getStorageBucketDatabase()))); - LOG(info, "STARTING INITIALIZATION"); + LOG(debug, "STARTING INITIALIZATION"); top.open(); - /* - FileChanger updater(config, nodeIndex, params, orgBucketDatabase); - if (params.bucketWrongDisk) updater.moveBucketWrongDisk(); - if (params.bucketMultipleDisks) updater.copyBucketWrongDisk(); - if (params.failingListRequest) { - updater.removeDirPermission(6, 'r'); - updater.removeBucketsFromDBAtPath(6); - } - if (params.failingInfoRequest) { - updater.removeFilePermission(); - orgBucketDatabase.erase(updater.getBucket(8)); - } - */ - node.waitUntilInitialized(initializer); std::map<PartitionId, DiskData> initedBucketDatabase( createMapFromBucketDatabase(node.getStorageBucketDatabase())); verifyEqual(data, initedBucketDatabase); - /* - if (params.bucketWrongDisk) { - CPPUNIT_ASSERT_METRIC_SET(_wrongDisk); - } - if (params.bucketMultipleDisks) { - CPPUNIT_ASSERT_METRIC_SET(_joinedCount); - } - */ -} - -/* -namespace { - enum State { LISTING, INFO, DONE }; - void verifyStatusContent(StorageBucketDBInitializer& initializer, - State state) - { - std::ostringstream ost; - initializer.reportStatus(ost, framework::HttpUrlPath("")); - std::string status = ost.str(); - - if (state == LISTING) { - CPPUNIT_ASSERT_CONTAIN("List phase completed: false", status); - CPPUNIT_ASSERT_CONTAIN("Initialization completed: false", status); - } else if (state == INFO) { - CPPUNIT_ASSERT_CONTAIN("List phase completed: true", status); - CPPUNIT_ASSERT_CONTAIN("Initialization completed: false", status); - } else if (state == DONE) { - CPPUNIT_ASSERT_CONTAIN("List phase completed: true", status); - CPPUNIT_ASSERT_CONTAIN("Initialization completed: true", status); - } - } -} - -void -InitializerTest::testStatusPage() -{ - // Set up surrounding system to create a single bucket for us to - // do init on. - vdstestlib::DirConfig config(getStandardConfig(true)); - uint16_t nodeIndex( - config.getConfig("stor-server").getValue("node_index", 0)); - InitParams params; - params.docsPerDisk = 1; - params.diskCount = 1; - std::map<document::BucketId, api::BucketInfo> orgBucketDatabase( - buildBucketInfo(_docMan, config, nodeIndex, 1, 1, params.disksDown)); - FileChanger updater(config, nodeIndex, params, orgBucketDatabase); - - // Set up the initializer. - DummyStorageServer server(config.getConfigId()); - DummyStorageLink top; - DummyStorageLink *bottom; - StorageBucketDBInitializer* initializer; - top.push_back(StorageLink::UP(initializer = new StorageBucketDBInitializer( - config.getConfigId(), server))); - top.push_back(StorageLink::UP(bottom = new DummyStorageLink)); - - // Grab bucket database lock for bucket to init to lock the initializer - // in the init stage - StorBucketDatabase::WrappedEntry entry( - server.getStorageBucketDatabase().get( - updater.getBucket(0), "testCommandBlocking", - StorBucketDatabase::LOCK_IF_NONEXISTING_AND_NOT_CREATING)); - // Start the initializer - top.open(); - bottom->waitForMessages(1, 30); - verifyStatusContent(*initializer, LISTING); - // Attempt to send put. Should be blocked - // Attempt to send request bucket info. Should be blocked. - // Attempt to send getNodeState. Should not be blocked. - - // Unlock bucket in bucket database so listing step can complete. - // Await read info request being sent down. - entry.unlock(); - bottom->waitForMessages(1, 30); - verifyStatusContent(*initializer, INFO); - - ReadBucketInfo& cmd(dynamic_cast<ReadBucketInfo&>(*bottom->getCommand(0))); - ReadBucketInfoReply::SP reply(new ReadBucketInfoReply(cmd)); - bottom->sendUp(reply); - - node.waitUntilInitialized(initializer); - verifyStatusContent(*initializer, DONE); - } -#define ASSERT_BLOCKED(top, bottom, blocks) \ - if (blocks) { \ - top.waitForMessages(1, 30); \ - CPPUNIT_ASSERT_EQUAL(size_t(1), top.getReplies().size()); \ - CPPUNIT_ASSERT_EQUAL(size_t(0), bottom.getCommands().size()); \ - api::StorageReply& reply(dynamic_cast<api::StorageReply&>( \ - *top.getReply(0))); \ - CPPUNIT_ASSERT_EQUAL(api::ReturnCode::ABORTED, \ - reply.getResult().getResult()); \ - top.reset(); \ - } else { \ - bottom.waitForMessages(1, 30); \ - CPPUNIT_ASSERT_EQUAL(size_t(0), top.getReplies().size()); \ - CPPUNIT_ASSERT_EQUAL(size_t(1), bottom.getCommands().size()); \ - api::StorageCommand& command(dynamic_cast<api::StorageCommand&>( \ - *bottom.getCommand(0))); \ - (void) command; \ - bottom.reset(); \ - } - -namespace { - void verifyBlockingOn(DummyStorageLink& top, - DummyStorageLink& bottom, - bool blockEnabled) - { - // Attempt to send get. Should be blocked if block enabled - { - api::GetCommand::SP cmd(new api::GetCommand( - document::BucketId(16, 4), - document::DocumentId("userdoc:ns:4:test"), true)); - top.sendDown(cmd); - ASSERT_BLOCKED(top, bottom, blockEnabled); - } - // Attempt to send request bucket info. Should be blocked if enabled. - { - api::RequestBucketInfoCommand::SP cmd( - new api::RequestBucketInfoCommand( - 0, lib::ClusterState(""))); - top.sendDown(cmd); - ASSERT_BLOCKED(top, bottom, blockEnabled); - } - // Attempt to send getNodeState. Should not be blocked. - { - api::GetNodeStateCommand::SP cmd(new api::GetNodeStateCommand( - lib::NodeState::UP(0))); - top.sendDown(cmd); - ASSERT_BLOCKED(top, bottom, false); - } - } -} - -void -InitializerTest::testCommandBlockingDuringInit() -{ - // Set up surrounding system to create a single bucket for us to - // do init on. - vdstestlib::DirConfig config(getStandardConfig(true)); - uint16_t nodeIndex( - config.getConfig("stor-server").getValue("node_index", 0)); - InitParams params; - params.docsPerDisk = 1; - params.diskCount = 1; - std::map<document::BucketId, api::BucketInfo> orgBucketDatabase( - buildBucketInfo(_docMan, config, nodeIndex, 1, 1, params.disksDown)); - FileChanger updater(config, nodeIndex, params, orgBucketDatabase); - - // Set up the initializer. - DummyStorageServer server(config.getConfigId()); - DummyStorageLink top; - DummyStorageLink *bottom; - StorageBucketDBInitializer* initializer; - top.push_back(StorageLink::UP(initializer = new StorageBucketDBInitializer( - config.getConfigId(), server))); - top.push_back(StorageLink::UP(bottom = new DummyStorageLink)); - - // Grab bucket database lock for bucket to init to lock the initializer - // in the init stage - StorBucketDatabase::WrappedEntry entry( - server.getStorageBucketDatabase().get( - updater.getBucket(0), "testCommandBlocking", - StorBucketDatabase::LOCK_IF_NONEXISTING_AND_NOT_CREATING)); - // Start the initializer - top.open(); - verifyBlockingOn(top, *bottom, true); - // Attempt to send put. Should be blocked - // Attempt to send request bucket info. Should be blocked. - // Attempt to send getNodeState. Should not be blocked. - - // Unlock bucket in bucket database so listing step can complete. - // Await read info request being sent down. - entry.unlock(); - bottom->waitForMessages(1, 30); - dynamic_cast<ReadBucketInfo&>(*bottom->getCommand(0)); - CPPUNIT_ASSERT(!server.isInitialized()); - bottom->reset(); - - // Retry - Should now not block - verifyBlockingOn(top, *bottom, false); -} -*/ - -void -InitializerTest::testBucketProgressCalculator() -{ +TEST_F(InitializerTest, bucket_progress_calculator) { using document::BucketId; StorageBucketDBInitializer::BucketProgressCalculator calc; // We consider the given bucket as not being completed, so progress // will be _up to_, not _including_ the bucket. This means we can never // reach 1.0, so progress completion must be handled by other logic! - CPPUNIT_ASSERT_EQUAL(0.0, calc.calculateProgress(BucketId(1, 0))); - CPPUNIT_ASSERT_EQUAL(0.0, calc.calculateProgress(BucketId(32, 0))); + EXPECT_DOUBLE_EQ(0.0, calc.calculateProgress(BucketId(1, 0))); + EXPECT_DOUBLE_EQ(0.0, calc.calculateProgress(BucketId(32, 0))); - CPPUNIT_ASSERT_EQUAL(0.5, calc.calculateProgress(BucketId(1, 1))); + EXPECT_DOUBLE_EQ(0.5, calc.calculateProgress(BucketId(1, 1))); - CPPUNIT_ASSERT_EQUAL(0.25, calc.calculateProgress(BucketId(2, 2))); - CPPUNIT_ASSERT_EQUAL(0.5, calc.calculateProgress(BucketId(2, 1))); - CPPUNIT_ASSERT_EQUAL(0.75, calc.calculateProgress(BucketId(2, 3))); + EXPECT_DOUBLE_EQ(0.25, calc.calculateProgress(BucketId(2, 2))); + EXPECT_DOUBLE_EQ(0.5, calc.calculateProgress(BucketId(2, 1))); + EXPECT_DOUBLE_EQ(0.75, calc.calculateProgress(BucketId(2, 3))); - CPPUNIT_ASSERT_EQUAL(0.875, calc.calculateProgress(BucketId(3, 7))); + EXPECT_DOUBLE_EQ(0.875, calc.calculateProgress(BucketId(3, 7))); } struct DatabaseInsertCallback : MessageCallback @@ -809,7 +508,6 @@ struct DatabaseInsertCallback : MessageCallback _app.getStateUpdater().getReportedNodeState()); double progress(reportedState->getInitProgress().getValue()); LOG(debug, "reported progress is now %g", progress); - // CppUnit exceptions are swallowed... if (progress >= 1.0) { _errors << "progress exceeded 1.0: " << progress << "\n"; } @@ -835,8 +533,7 @@ struct DatabaseInsertCallback : MessageCallback } if (msg.getType() == api::MessageType::INTERNAL) { - const api::InternalCommand& cmd( - dynamic_cast<const api::InternalCommand&>(msg)); + auto& cmd = dynamic_cast<const api::InternalCommand&>(msg); if (cmd.getType() == ReadBucketInfo::ID) { if (cmd.getPriority() != _expectedReadBucketPriority) { _errors << "expected ReadBucketInfo priority of " @@ -871,9 +568,7 @@ struct DatabaseInsertCallback : MessageCallback } }; -void -InitializerTest::testBucketsInitializedByLoad() -{ +TEST_F(InitializerTest, buckets_initialized_by_load) { InitParams params; params.docsPerDisk = 100; params.diskCount = DiskCount(1); @@ -911,8 +606,8 @@ InitializerTest::testBucketsInitializedByLoad() // has been set. top.close(); - CPPUNIT_ASSERT(callback._invoked); - CPPUNIT_ASSERT_EQUAL(std::string(), callback._errors.str()); + ASSERT_TRUE(callback._invoked); + EXPECT_EQ(std::string(), callback._errors.str()); std::map<PartitionId, DiskData> initedBucketDatabase( createMapFromBucketDatabase(node.getStorageBucketDatabase())); @@ -922,11 +617,10 @@ InitializerTest::testBucketsInitializedByLoad() node.getStateUpdater().getReportedNodeState()); double progress(reportedState->getInitProgress().getValue()); - CPPUNIT_ASSERT(progress >= 1.0); - CPPUNIT_ASSERT(progress < 1.0001); + EXPECT_GE(progress, 1.0); + EXPECT_LT(progress, 1.0001); - CPPUNIT_ASSERT_EQUAL(params.bucketBitsUsed, - reportedState->getMinUsedBits()); + EXPECT_EQ(params.bucketBitsUsed, reportedState->getMinUsedBits()); } } // storage diff --git a/storage/src/tests/bucketdb/judyarraytest.cpp b/storage/src/tests/bucketdb/judyarraytest.cpp index 07992450fbf..94d61107fcf 100644 --- a/storage/src/tests/bucketdb/judyarraytest.cpp +++ b/storage/src/tests/bucketdb/judyarraytest.cpp @@ -1,36 +1,20 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/storage/bucketdb/judyarray.h> -#include <vespa/vdstestlib/cppunit/macros.h> -#include <boost/assign.hpp> #include <boost/random.hpp> -#include <cppunit/extensions/HelperMacros.h> +#include <vespa/vespalib/gtest/gtest.h> +#include <gmock/gmock.h> #include <map> #include <vector> -namespace storage { - -struct JudyArrayTest : public CppUnit::TestFixture { - void testIterating(); - void testDualArrayFunctions(); - void testComparing(); - void testSize(); - void testStress(); +using namespace ::testing; - CPPUNIT_TEST_SUITE(JudyArrayTest); - CPPUNIT_TEST(testIterating); - CPPUNIT_TEST(testDualArrayFunctions); - CPPUNIT_TEST(testSize); - CPPUNIT_TEST(testStress); - CPPUNIT_TEST_SUITE_END(); -}; - -CPPUNIT_TEST_SUITE_REGISTRATION(JudyArrayTest); +namespace storage { namespace { - std::vector<std::pair<JudyArray::key_type, JudyArray::data_type> > + std::vector<std::pair<JudyArray::key_type, JudyArray::data_type>> getJudyArrayContents(const JudyArray& array) { - std::vector<std::pair<JudyArray::key_type, JudyArray::data_type> > vals; + std::vector<std::pair<JudyArray::key_type, JudyArray::data_type>> vals; for (JudyArray::const_iterator it = array.begin(); it != array.end(); ++it) { @@ -40,168 +24,129 @@ namespace { } } -void -JudyArrayTest::testIterating() -{ +TEST(JudyArrayTest, iterating) { JudyArray array; - // Test that things are sane for empty document - CPPUNIT_ASSERT_EQUAL(array.begin(), array.end()); - // Add some values - using namespace boost::assign; - std::vector<std::pair<JudyArray::key_type, JudyArray::data_type> > values - = map_list_of(3,2)(5,12)(15,8)(13,10)(7,6)(9,4); + // Test that things are sane for empty document + ASSERT_EQ(array.begin(), array.end()); + // Add some values + std::vector<std::pair<JudyArray::key_type, JudyArray::data_type>> values({ + {3, 2}, {5, 12}, {15, 8}, {13, 10}, {7, 6}, {9, 4} + }); for (uint32_t i=0; i<values.size(); ++i) { array.insert(values[i].first, values[i].second); } - // Create expected result + // Create expected result std::sort(values.begin(), values.end()); - // Test that we can iterate through const iterator - std::vector<std::pair<JudyArray::key_type, JudyArray::data_type> > - foundVals = getJudyArrayContents(array); - CPPUNIT_ASSERT_EQUAL(values, foundVals); + // Test that we can iterate through const iterator + auto foundVals = getJudyArrayContents(array); + ASSERT_EQ(values, foundVals); { // Test that we can alter through non-const iterator JudyArray::iterator it = array.begin(); ++it; ++it; it.setValue(20); - CPPUNIT_ASSERT_EQUAL((JudyArray::key_type) 7, it.key()); - CPPUNIT_ASSERT_EQUAL((JudyArray::data_type) 20, array[7]); + ASSERT_EQ((JudyArray::key_type) 7, it.key()); + ASSERT_EQ((JudyArray::data_type) 20, array[7]); it.remove(); - CPPUNIT_ASSERT_EQUAL((JudyArray::size_type) 5, - getJudyArrayContents(array).size()); - CPPUNIT_ASSERT_EQUAL(array.end(), array.find(7)); + ASSERT_EQ((JudyArray::size_type) 5, getJudyArrayContents(array).size()); + ASSERT_EQ(array.end(), array.find(7)); values.erase(values.begin() + 2); - CPPUNIT_ASSERT_EQUAL(values, getJudyArrayContents(array)); - // And that we can continue iterating after removing. + ASSERT_EQ(values, getJudyArrayContents(array)); + // And that we can continue iterating after removing. ++it; - CPPUNIT_ASSERT_EQUAL((JudyArray::key_type) 9, it.key()); - CPPUNIT_ASSERT_EQUAL((JudyArray::data_type) 4, array[9]); + ASSERT_EQ((JudyArray::key_type) 9, it.key()); + ASSERT_EQ((JudyArray::data_type) 4, array[9]); } { // Test printing of iterators JudyArray::ConstIterator cit = array.begin(); - CPPUNIT_ASSERT_MATCH_REGEX( - "^ConstIterator\\(Key: 3, Valp: 0x[0-9a-f]{1,16}, Val: 2\\)$", - cit.toString()); + EXPECT_THAT(cit.toString(), MatchesRegex("^ConstIterator\\(Key: 3, Valp: 0x[0-9a-f]{1,16}, Val: 2\\)$")); JudyArray::Iterator it = array.end(); - CPPUNIT_ASSERT_MATCH_REGEX( - "^Iterator\\(Key: 0, Valp: 0\\)$", - it.toString()); + EXPECT_THAT(it.toString(), MatchesRegex("^Iterator\\(Key: 0, Valp: 0\\)$")); } } -void -JudyArrayTest::testDualArrayFunctions() -{ +TEST(JudyArrayTest, dual_array_functions) { JudyArray array1; JudyArray array2; - // Add values to array1 - using namespace boost::assign; - std::vector<std::pair<JudyArray::key_type, JudyArray::data_type> > values1 - = map_list_of(3,2)(5,12)(15,8)(13,10)(7,6)(9,4); + // Add values to array1 + std::vector<std::pair<JudyArray::key_type, JudyArray::data_type>> values1({ + {3, 2}, {5, 12}, {15, 8}, {13, 10}, {7, 6}, {9, 4} + }); for (uint32_t i=0; i<values1.size(); ++i) { array1.insert(values1[i].first, values1[i].second); } - // Add values to array2 - std::vector<std::pair<JudyArray::key_type, JudyArray::data_type> > values2 - = map_list_of(4,5)(9,40); + // Add values to array2 + std::vector<std::pair<JudyArray::key_type, JudyArray::data_type>> values2({ + {4, 5}, {9, 40} + }); for (uint32_t i=0; i<values2.size(); ++i) { array2.insert(values2[i].first, values2[i].second); } - // Create expected result + // Create expected result std::sort(values1.begin(), values1.end()); std::sort(values2.begin(), values2.end()); - CPPUNIT_ASSERT_EQUAL(values1, getJudyArrayContents(array1)); - CPPUNIT_ASSERT_EQUAL(values2, getJudyArrayContents(array2)); - CPPUNIT_ASSERT(array2 < array1); - CPPUNIT_ASSERT(array1 != array2); + EXPECT_EQ(values1, getJudyArrayContents(array1)); + EXPECT_EQ(values2, getJudyArrayContents(array2)); + EXPECT_LT(array2, array1); + EXPECT_NE(array1, array2); array1.swap(array2); - CPPUNIT_ASSERT_EQUAL(values1, getJudyArrayContents(array2)); - CPPUNIT_ASSERT_EQUAL(values2, getJudyArrayContents(array1)); - CPPUNIT_ASSERT(array1 < array2); - CPPUNIT_ASSERT(array1 != array2); + EXPECT_EQ(values1, getJudyArrayContents(array2)); + EXPECT_EQ(values2, getJudyArrayContents(array1)); + EXPECT_LT(array1, array2); + EXPECT_NE(array1, array2); - // Test some operators + // Test some operators JudyArray array3; for (uint32_t i=0; i<values1.size(); ++i) { array3.insert(values1[i].first, values1[i].second); } - CPPUNIT_ASSERT(array1 != array3); - CPPUNIT_ASSERT_EQUAL(array2, array3); - CPPUNIT_ASSERT(!(array2 < array3)); + EXPECT_NE(array1, array3); + EXPECT_EQ(array2, array3); + EXPECT_FALSE(array2 < array3); } -void -JudyArrayTest::testSize() -{ +TEST(JudyArrayTest, size) { JudyArray array; - CPPUNIT_ASSERT_EQUAL(array.begin(), array.end()); - CPPUNIT_ASSERT(array.empty()); - CPPUNIT_ASSERT_EQUAL((JudyArray::size_type) 0, array.size()); - CPPUNIT_ASSERT_EQUAL((JudyArray::size_type) 0, array.getMemoryUsage()); + EXPECT_EQ(array.begin(), array.end()); + EXPECT_TRUE(array.empty()); + EXPECT_EQ((JudyArray::size_type) 0, array.size()); + EXPECT_EQ((JudyArray::size_type) 0, array.getMemoryUsage()); - // Test each method one can insert stuff into array + // Test each method one can insert stuff into array array.insert(4, 3); - CPPUNIT_ASSERT_EQUAL(getJudyArrayContents(array).size(), array.size()); + EXPECT_EQ(getJudyArrayContents(array).size(), array.size()); array.insert(4, 7); - CPPUNIT_ASSERT_EQUAL(getJudyArrayContents(array).size(), array.size()); - if (sizeof(JudyArray::size_type) == 4) { - CPPUNIT_ASSERT_EQUAL((JudyArray::size_type) 12, array.getMemoryUsage()); - } else if (sizeof(JudyArray::size_type) == 8) { - CPPUNIT_ASSERT_EQUAL((JudyArray::size_type) 24, array.getMemoryUsage()); - } else CPPUNIT_FAIL("Unknown size of type"); + EXPECT_EQ(getJudyArrayContents(array).size(), array.size()); + EXPECT_EQ((JudyArray::size_type) 24, array.getMemoryUsage()); array[6] = 8; - CPPUNIT_ASSERT_EQUAL(getJudyArrayContents(array).size(), array.size()); + EXPECT_EQ(getJudyArrayContents(array).size(), array.size()); array[6] = 10; - CPPUNIT_ASSERT_EQUAL(getJudyArrayContents(array).size(), array.size()); - if (sizeof(JudyArray::size_type) == 4) { - CPPUNIT_ASSERT_EQUAL((JudyArray::size_type) 20, array.getMemoryUsage()); - } else if (sizeof(JudyArray::size_type) == 8) { - CPPUNIT_ASSERT_EQUAL((JudyArray::size_type) 40, array.getMemoryUsage()); - } else CPPUNIT_FAIL("Unknown size of type"); + EXPECT_EQ(getJudyArrayContents(array).size(), array.size()); + EXPECT_EQ((JudyArray::size_type) 40, array.getMemoryUsage()); bool preExisted; array.find(8, true, preExisted); - CPPUNIT_ASSERT_EQUAL(false, preExisted); - CPPUNIT_ASSERT_EQUAL(getJudyArrayContents(array).size(), array.size()); + EXPECT_EQ(false, preExisted); + EXPECT_EQ(getJudyArrayContents(array).size(), array.size()); array.find(8, true, preExisted); - CPPUNIT_ASSERT_EQUAL(true, preExisted); - CPPUNIT_ASSERT_EQUAL(getJudyArrayContents(array).size(), array.size()); - CPPUNIT_ASSERT_EQUAL((JudyArray::size_type) 3, array.size()); - if (sizeof(JudyArray::size_type) == 4) { - CPPUNIT_ASSERT_EQUAL((JudyArray::size_type) 28, array.getMemoryUsage()); - } else if (sizeof(JudyArray::size_type) == 8) { - CPPUNIT_ASSERT_EQUAL((JudyArray::size_type) 56, array.getMemoryUsage()); - } else CPPUNIT_FAIL("Unknown size of type"); + EXPECT_EQ(true, preExisted); + EXPECT_EQ(getJudyArrayContents(array).size(), array.size()); + EXPECT_EQ((JudyArray::size_type) 3, array.size()); + EXPECT_EQ((JudyArray::size_type) 56, array.getMemoryUsage()); - // Test each method one can remove stuff in array with + // Test each method one can remove stuff in array with array.erase(8); - CPPUNIT_ASSERT_EQUAL(getJudyArrayContents(array).size(), array.size()); + EXPECT_EQ(getJudyArrayContents(array).size(), array.size()); array.erase(8); - CPPUNIT_ASSERT_EQUAL(getJudyArrayContents(array).size(), array.size()); - CPPUNIT_ASSERT_EQUAL((JudyArray::size_type) 2, array.size()); - if (sizeof(JudyArray::size_type) == 4) { - CPPUNIT_ASSERT_EQUAL((JudyArray::size_type) 20, array.getMemoryUsage()); - } else if (sizeof(JudyArray::size_type) == 8) { - CPPUNIT_ASSERT_EQUAL((JudyArray::size_type) 40, array.getMemoryUsage()); - } else CPPUNIT_FAIL("Unknown size of type"); -} - -namespace { - template<typename T> - std::string toString(const T& m) { - std::cerr << "#"; - std::ostringstream ost; - ost << m; - return ost.str(); - } + EXPECT_EQ(getJudyArrayContents(array).size(), array.size()); + EXPECT_EQ((JudyArray::size_type) 2, array.size()); + EXPECT_EQ((JudyArray::size_type) 40, array.getMemoryUsage()); } -void -JudyArrayTest::testStress() -{ +TEST(JudyArrayTest, stress) { // Do a lot of random stuff to both judy array and std::map. Ensure equal // behaviour @@ -219,9 +164,6 @@ JudyArrayTest::testStress() JudyArray::key_type value(rnd()); judyArray.insert(key, value); stdMap[key] = value; - //std::pair<StdMap::iterator, bool> result - // = stdMap.insert(std::make_pair(key, value)); - //if (!result.second) result.first->second = value; } else if (optype < 50) { // operator[] JudyArray::key_type key(rnd() % 500); JudyArray::key_type value(rnd()); @@ -229,42 +171,30 @@ JudyArrayTest::testStress() stdMap[key] = value; } else if (optype < 70) { // erase() JudyArray::key_type key(rnd() % 500); - CPPUNIT_ASSERT_EQUAL_MSG( - toString(judyArray) + toString(stdMap), - stdMap.erase(key), judyArray.erase(key)); + EXPECT_EQ(stdMap.erase(key), judyArray.erase(key)); } else if (optype < 75) { // size() - CPPUNIT_ASSERT_EQUAL_MSG( - toString(judyArray) + toString(stdMap), - stdMap.size(), judyArray.size()); + EXPECT_EQ(stdMap.size(), judyArray.size()); } else if (optype < 78) { // empty() - CPPUNIT_ASSERT_EQUAL_MSG( - toString(judyArray) + toString(stdMap), - stdMap.empty(), judyArray.empty()); + EXPECT_EQ(stdMap.empty(), judyArray.empty()); } else { // find() JudyArray::key_type key(rnd() % 500); - JudyArray::iterator it = judyArray.find(key); - StdMap::iterator it2 = stdMap.find(key); - CPPUNIT_ASSERT_EQUAL_MSG( - toString(judyArray) + toString(stdMap), - it2 == stdMap.end(), it == judyArray.end()); + auto it = judyArray.find(key); + auto it2 = stdMap.find(key); + EXPECT_EQ(it2 == stdMap.end(), it == judyArray.end()); if (it != judyArray.end()) { - CPPUNIT_ASSERT_EQUAL_MSG( - toString(judyArray) + toString(stdMap), - it.key(), it2->first); - CPPUNIT_ASSERT_EQUAL_MSG( - toString(judyArray) + toString(stdMap), - it.value(), it2->second); + EXPECT_EQ(it.key(), it2->first); + EXPECT_EQ(it.value(), it2->second); } } } - // Ensure judy array contents is equal to std::map's at this point + // Ensure judy array contents is equal to std::map's at this point StdMap tmpMap; for (JudyArray::const_iterator it = judyArray.begin(); it != judyArray.end(); ++it) { tmpMap[it.key()] = it.value(); } - CPPUNIT_ASSERT_EQUAL(stdMap, tmpMap); + EXPECT_EQ(stdMap, tmpMap); } } diff --git a/storage/src/tests/bucketdb/judymultimaptest.cpp b/storage/src/tests/bucketdb/judymultimaptest.cpp index 43b83b16dec..254dbb78b18 100644 --- a/storage/src/tests/bucketdb/judymultimaptest.cpp +++ b/storage/src/tests/bucketdb/judymultimaptest.cpp @@ -2,31 +2,15 @@ #include <vespa/storage/bucketdb/judymultimap.h> #include <vespa/storage/bucketdb/judymultimap.hpp> -#include <vespa/vdstestlib/cppunit/macros.h> -#include <boost/assign.hpp> -#include <boost/random.hpp> -#include <cppunit/extensions/HelperMacros.h> +#include <vespa/vespalib/gtest/gtest.h> #include <map> #include <ostream> #include <vector> -#include <vespa/log/log.h> -LOG_SETUP(".judy_multi_map_test"); +using namespace ::testing; namespace storage { -struct JudyMultiMapTest : public CppUnit::TestFixture { - void testSimpleUsage(); - void testIterator(); - - CPPUNIT_TEST_SUITE(JudyMultiMapTest); - CPPUNIT_TEST(testSimpleUsage); - CPPUNIT_TEST(testIterator); - CPPUNIT_TEST_SUITE_END(); -}; - -CPPUNIT_TEST_SUITE_REGISTRATION(JudyMultiMapTest); - namespace { struct B; struct C; @@ -84,48 +68,44 @@ namespace { } } -void -JudyMultiMapTest::testSimpleUsage() { +TEST(JudyMultiMapTest, simple_usage) { typedef JudyMultiMap<C, B, A> MultiMap; MultiMap multiMap; - // Do some insertions + // Do some insertions bool preExisted; - CPPUNIT_ASSERT(multiMap.empty()); + EXPECT_TRUE(multiMap.empty()); multiMap.insert(16, A(1, 2, 3), preExisted); - CPPUNIT_ASSERT_EQUAL(false, preExisted); + EXPECT_EQ(false, preExisted); multiMap.insert(11, A(4, 6, 0), preExisted); - CPPUNIT_ASSERT_EQUAL(false, preExisted); + EXPECT_EQ(false, preExisted); multiMap.insert(14, A(42, 0, 0), preExisted); - CPPUNIT_ASSERT_EQUAL(false, preExisted); - CPPUNIT_ASSERT_EQUAL_MSG(multiMap.toString(), - (MultiMap::size_type) 3, multiMap.size()); + EXPECT_EQ(false, preExisted); + EXPECT_EQ((MultiMap::size_type) 3, multiMap.size()) << multiMap.toString(); multiMap.insert(11, A(4, 7, 0), preExisted); - CPPUNIT_ASSERT_EQUAL(true, preExisted); - CPPUNIT_ASSERT_EQUAL((MultiMap::size_type) 3, multiMap.size()); - CPPUNIT_ASSERT(!multiMap.empty()); - - // Access some elements - CPPUNIT_ASSERT_EQUAL(A(4, 7, 0), multiMap[11]); - CPPUNIT_ASSERT_EQUAL(A(1, 2, 3), multiMap[16]); - CPPUNIT_ASSERT_EQUAL(A(42,0, 0), multiMap[14]); - - // Do removes - CPPUNIT_ASSERT(multiMap.erase(12) == 0); - CPPUNIT_ASSERT_EQUAL((MultiMap::size_type) 3, multiMap.size()); - - CPPUNIT_ASSERT(multiMap.erase(14) == 1); - CPPUNIT_ASSERT_EQUAL((MultiMap::size_type) 2, multiMap.size()); - - CPPUNIT_ASSERT(multiMap.erase(11) == 1); - CPPUNIT_ASSERT(multiMap.erase(16) == 1); - CPPUNIT_ASSERT_EQUAL((MultiMap::size_type) 0, multiMap.size()); - CPPUNIT_ASSERT(multiMap.empty()); + EXPECT_EQ(true, preExisted); + EXPECT_EQ((MultiMap::size_type) 3, multiMap.size()); + EXPECT_FALSE(multiMap.empty()); + + // Access some elements + EXPECT_EQ(A(4, 7, 0), multiMap[11]); + EXPECT_EQ(A(1, 2, 3), multiMap[16]); + EXPECT_EQ(A(42,0, 0), multiMap[14]); + + // Do removes + EXPECT_EQ(multiMap.erase(12), 0); + EXPECT_EQ((MultiMap::size_type) 3, multiMap.size()); + + EXPECT_EQ(multiMap.erase(14), 1); + EXPECT_EQ((MultiMap::size_type) 2, multiMap.size()); + + EXPECT_EQ(multiMap.erase(11), 1); + EXPECT_EQ(multiMap.erase(16), 1); + EXPECT_EQ((MultiMap::size_type) 0, multiMap.size()); + EXPECT_TRUE(multiMap.empty()); } -void -JudyMultiMapTest::testIterator() -{ +TEST(JudyMultiMapTest, iterator) { typedef JudyMultiMap<C, B, A> MultiMap; MultiMap multiMap; bool preExisted; @@ -135,37 +115,37 @@ JudyMultiMapTest::testIterator() multiMap.insert(14, A(42, 0, 0), preExisted); MultiMap::Iterator iter = multiMap.begin(); - CPPUNIT_ASSERT_EQUAL((uint64_t)11, (uint64_t)iter.key()); - CPPUNIT_ASSERT_EQUAL(A(4, 6, 0), iter.value()); + EXPECT_EQ((uint64_t)11, (uint64_t)iter.key()); + EXPECT_EQ(A(4, 6, 0), iter.value()); ++iter; - CPPUNIT_ASSERT_EQUAL((uint64_t)14, (uint64_t)iter.key()); - CPPUNIT_ASSERT_EQUAL(A(42, 0, 0), iter.value()); + EXPECT_EQ((uint64_t)14, (uint64_t)iter.key()); + EXPECT_EQ(A(42, 0, 0), iter.value()); ++iter; - CPPUNIT_ASSERT_EQUAL((uint64_t)16, (uint64_t)iter.key()); - CPPUNIT_ASSERT_EQUAL(A(1, 2, 3), iter.value()); + EXPECT_EQ((uint64_t)16, (uint64_t)iter.key()); + EXPECT_EQ(A(1, 2, 3), iter.value()); --iter; - CPPUNIT_ASSERT_EQUAL((uint64_t)14, (uint64_t)iter.key()); - CPPUNIT_ASSERT_EQUAL(A(42, 0, 0), iter.value()); + EXPECT_EQ((uint64_t)14, (uint64_t)iter.key()); + EXPECT_EQ(A(42, 0, 0), iter.value()); ++iter; - CPPUNIT_ASSERT_EQUAL((uint64_t)16, (uint64_t)iter.key()); - CPPUNIT_ASSERT_EQUAL(A(1, 2, 3), iter.value()); + EXPECT_EQ((uint64_t)16, (uint64_t)iter.key()); + EXPECT_EQ(A(1, 2, 3), iter.value()); --iter; --iter; - CPPUNIT_ASSERT_EQUAL((uint64_t)11,(uint64_t) iter.key()); - CPPUNIT_ASSERT_EQUAL(A(4, 6, 0), iter.value()); + EXPECT_EQ((uint64_t)11,(uint64_t) iter.key()); + EXPECT_EQ(A(4, 6, 0), iter.value()); ++iter; ++iter; ++iter; - CPPUNIT_ASSERT_EQUAL(multiMap.end(), iter); + EXPECT_EQ(multiMap.end(), iter); --iter; - CPPUNIT_ASSERT_EQUAL((uint64_t)16, (uint64_t)iter.key()); - CPPUNIT_ASSERT_EQUAL(A(1, 2, 3), iter.value()); + EXPECT_EQ((uint64_t)16, (uint64_t)iter.key()); + EXPECT_EQ(A(1, 2, 3), iter.value()); --iter; - CPPUNIT_ASSERT_EQUAL((uint64_t)14, (uint64_t)iter.key()); - CPPUNIT_ASSERT_EQUAL(A(42, 0, 0), iter.value()); + EXPECT_EQ((uint64_t)14, (uint64_t)iter.key()); + EXPECT_EQ(A(42, 0, 0), iter.value()); --iter; - CPPUNIT_ASSERT_EQUAL((uint64_t)11,(uint64_t) iter.key()); - CPPUNIT_ASSERT_EQUAL(A(4, 6, 0), iter.value()); + EXPECT_EQ((uint64_t)11,(uint64_t) iter.key()); + EXPECT_EQ(A(4, 6, 0), iter.value()); } } // storage diff --git a/storage/src/tests/bucketdb/lockablemaptest.cpp b/storage/src/tests/bucketdb/lockablemaptest.cpp index 10f806f2e97..a55e258129c 100644 --- a/storage/src/tests/bucketdb/lockablemaptest.cpp +++ b/storage/src/tests/bucketdb/lockablemaptest.cpp @@ -4,84 +4,17 @@ #include <vespa/storage/bucketdb/judymultimap.h> #include <vespa/storage/bucketdb/judymultimap.hpp> #include <vespa/storage/bucketdb/lockablemap.hpp> -#include <vespa/vdstestlib/cppunit/macros.h> -#include <cppunit/extensions/HelperMacros.h> +#include <vespa/vespalib/gtest/gtest.h> #include <boost/operators.hpp> #include <vespa/log/log.h> LOG_SETUP(".lockable_map_test"); -namespace storage { +// FIXME these old tests may have the least obvious semantics and worst naming in the entire storage module + +using namespace ::testing; -struct LockableMapTest : public CppUnit::TestFixture { - void testSimpleUsage(); - void testComparison(); - void testIterating(); - void testChunkedIterationIsTransparentAcrossChunkSizes(); - void testCanAbortDuringChunkedIteration(); - void testThreadSafetyStress(); - void testFindBuckets(); - void testFindBuckets2(); - void testFindBuckets3(); - void testFindBuckets4(); - void testFindBuckets5(); - void testFindBucketsSimple(); - void testFindNoBuckets(); - void testFindAll(); - void testFindAll2(); - void testFindAllUnusedBitIsSet(); - void testFindAllInconsistentlySplit(); - void testFindAllInconsistentlySplit2(); - void testFindAllInconsistentlySplit3(); - void testFindAllInconsistentlySplit4(); - void testFindAllInconsistentlySplit5(); - void testFindAllInconsistentlySplit6(); - void testFindAllInconsistentBelow16Bits(); - void testCreate(); - void testCreate2(); - void testCreate3(); - void testCreate4(); - void testCreate5(); - void testCreate6(); - void testCreateEmpty(); - void testIsConsistent(); - - CPPUNIT_TEST_SUITE(LockableMapTest); - CPPUNIT_TEST(testSimpleUsage); - CPPUNIT_TEST(testComparison); - CPPUNIT_TEST(testIterating); - CPPUNIT_TEST(testChunkedIterationIsTransparentAcrossChunkSizes); - CPPUNIT_TEST(testCanAbortDuringChunkedIteration); - CPPUNIT_TEST(testThreadSafetyStress); - CPPUNIT_TEST(testFindBuckets); - CPPUNIT_TEST(testFindBuckets2); - CPPUNIT_TEST(testFindBuckets3); - CPPUNIT_TEST(testFindBuckets4); - CPPUNIT_TEST(testFindBuckets5); - CPPUNIT_TEST(testFindBucketsSimple); - CPPUNIT_TEST(testFindNoBuckets); - CPPUNIT_TEST(testFindAll); - CPPUNIT_TEST(testFindAll2); - CPPUNIT_TEST(testFindAllUnusedBitIsSet); - CPPUNIT_TEST(testFindAllInconsistentlySplit); - CPPUNIT_TEST(testFindAllInconsistentlySplit2); - CPPUNIT_TEST(testFindAllInconsistentlySplit3); - CPPUNIT_TEST(testFindAllInconsistentlySplit4); - CPPUNIT_TEST(testFindAllInconsistentlySplit5); - CPPUNIT_TEST(testFindAllInconsistentlySplit6); - CPPUNIT_TEST(testFindAllInconsistentBelow16Bits); - CPPUNIT_TEST(testCreate); - CPPUNIT_TEST(testCreate2); - CPPUNIT_TEST(testCreate3); - CPPUNIT_TEST(testCreate4); - CPPUNIT_TEST(testCreate5); - CPPUNIT_TEST(testCreate6); - CPPUNIT_TEST(testCreateEmpty); - CPPUNIT_TEST(testIsConsistent); - CPPUNIT_TEST_SUITE_END(); -}; - -CPPUNIT_TEST_SUITE_REGISTRATION(LockableMapTest); +namespace storage { namespace { struct A : public boost::operators<A> { @@ -112,84 +45,81 @@ namespace { typedef LockableMap<JudyMultiMap<A> > Map; } -void -LockableMapTest::testSimpleUsage() { - // Tests insert, erase, size, empty, operator[] +TEST(LockableMapTest, simple_usage) { + // Tests insert, erase, size, empty, operator[] Map map; - // Do some insertions - CPPUNIT_ASSERT(map.empty()); + // Do some insertions + EXPECT_TRUE(map.empty()); bool preExisted; map.insert(16, A(1, 2, 3), "foo", preExisted); - CPPUNIT_ASSERT_EQUAL(false, preExisted); + EXPECT_EQ(false, preExisted); map.insert(11, A(4, 6, 0), "foo", preExisted); - CPPUNIT_ASSERT_EQUAL(false, preExisted); + EXPECT_EQ(false, preExisted); map.insert(14, A(42, 0, 0), "foo", preExisted); - CPPUNIT_ASSERT_EQUAL(false, preExisted); - CPPUNIT_ASSERT_EQUAL_MSG(map.toString(), - (Map::size_type) 3, map.size()); + EXPECT_EQ(false, preExisted); + EXPECT_EQ((Map::size_type) 3, map.size()) << map.toString(); map.insert(11, A(4, 7, 0), "foo", preExisted); - CPPUNIT_ASSERT_EQUAL(true, preExisted); - CPPUNIT_ASSERT_EQUAL((Map::size_type) 3, map.size()); - CPPUNIT_ASSERT(!map.empty()); - - // Access some elements - CPPUNIT_ASSERT_EQUAL(A(4, 7, 0), *map.get(11, "foo")); - CPPUNIT_ASSERT_EQUAL(A(1, 2, 3), *map.get(16, "foo")); - CPPUNIT_ASSERT_EQUAL(A(42,0, 0), *map.get(14, "foo")); - - // Do removes - CPPUNIT_ASSERT(map.erase(12, "foo") == 0); - CPPUNIT_ASSERT_EQUAL((Map::size_type) 3, map.size()); - - CPPUNIT_ASSERT(map.erase(14, "foo") == 1); - CPPUNIT_ASSERT_EQUAL((Map::size_type) 2, map.size()); - - CPPUNIT_ASSERT(map.erase(11, "foo") == 1); - CPPUNIT_ASSERT(map.erase(16, "foo") == 1); - CPPUNIT_ASSERT_EQUAL((Map::size_type) 0, map.size()); - CPPUNIT_ASSERT(map.empty()); + EXPECT_EQ(true, preExisted); + EXPECT_EQ((Map::size_type) 3, map.size()); + EXPECT_FALSE(map.empty()); + + // Access some elements + EXPECT_EQ(A(4, 7, 0), *map.get(11, "foo")); + EXPECT_EQ(A(1, 2, 3), *map.get(16, "foo")); + EXPECT_EQ(A(42,0, 0), *map.get(14, "foo")); + + // Do removes + EXPECT_EQ(map.erase(12, "foo"), 0); + EXPECT_EQ((Map::size_type) 3, map.size()); + + EXPECT_EQ(map.erase(14, "foo"), 1); + EXPECT_EQ((Map::size_type) 2, map.size()); + + EXPECT_EQ(map.erase(11, "foo"), 1); + EXPECT_EQ(map.erase(16, "foo"), 1); + EXPECT_EQ((Map::size_type) 0, map.size()); + EXPECT_TRUE(map.empty()); } -void -LockableMapTest::testComparison() { +TEST(LockableMapTest, comparison) { Map map1; Map map2; bool preExisted; - // Check empty state is correct - CPPUNIT_ASSERT_EQUAL(map1, map2); - CPPUNIT_ASSERT(!(map1 < map2)); - CPPUNIT_ASSERT(!(map1 != map2)); + // Check empty state is correct + EXPECT_EQ(map1, map2); + EXPECT_FALSE(map1 < map2); + EXPECT_FALSE(map1 != map2); - // Check that different lengths are oki + // Check that different lengths are ok map1.insert(4, A(1, 2, 3), "foo", preExisted); - CPPUNIT_ASSERT(!(map1 == map2)); - CPPUNIT_ASSERT(!(map1 < map2)); - CPPUNIT_ASSERT(map2 < map1); - CPPUNIT_ASSERT(map1 != map2); + EXPECT_FALSE(map1 == map2); + EXPECT_FALSE(map1 < map2); + EXPECT_LT(map2, map1); + EXPECT_NE(map1, map2); - // Check that equal elements are oki + // Check that equal elements are ok map2.insert(4, A(1, 2, 3), "foo", preExisted); - CPPUNIT_ASSERT_EQUAL(map1, map2); - CPPUNIT_ASSERT(!(map1 < map2)); - CPPUNIT_ASSERT(!(map1 != map2)); + EXPECT_EQ(map1, map2); + EXPECT_FALSE(map1 < map2); + EXPECT_FALSE(map1 != map2); - // Check that non-equal values are oki + // Check that non-equal values are ok map1.insert(6, A(1, 2, 6), "foo", preExisted); map2.insert(6, A(1, 2, 3), "foo", preExisted); - CPPUNIT_ASSERT(!(map1 == map2)); - CPPUNIT_ASSERT(!(map1 < map2)); - CPPUNIT_ASSERT(map2 < map1); - CPPUNIT_ASSERT(map1 != map2); + EXPECT_FALSE(map1 == map2); + EXPECT_FALSE(map1 < map2); + EXPECT_LT(map2, map1); + EXPECT_NE(map1, map2); - // Check that non-equal keys are oki + // Check that non-equal keys are ok map1.erase(6, "foo"); map1.insert(7, A(1, 2, 3), "foo", preExisted); - CPPUNIT_ASSERT(!(map1 == map2)); - CPPUNIT_ASSERT(!(map1 < map2)); - CPPUNIT_ASSERT(map2 < map1); - CPPUNIT_ASSERT(map1 != map2); + EXPECT_FALSE(map1 == map2); + EXPECT_FALSE(map1 < map2); + EXPECT_LT(map2, map1); + EXPECT_NE(map1, map2); } namespace { @@ -225,7 +155,9 @@ namespace { std::string toString() { std::ostringstream ost; - for (uint32_t i=0; i<log.size(); ++i) ost << log[i] << "\n"; + for (uint32_t i=0; i<log.size(); ++i) { + ost << log[i] << "\n"; + } return ost.str(); } }; @@ -234,10 +166,9 @@ namespace { EntryProcessor::EntryProcessor() : count(0), log(), behaviour() {} EntryProcessor::EntryProcessor(const std::vector<Map::Decision>& decisions) : count(0), log(), behaviour(decisions) {} -EntryProcessor::~EntryProcessor() {} +EntryProcessor::~EntryProcessor() = default; -void -LockableMapTest::testIterating() { +TEST(LockableMapTest, iterating) { Map map; bool preExisted; map.insert(16, A(1, 2, 3), "foo", preExisted); @@ -247,13 +178,13 @@ LockableMapTest::testIterating() { { NonConstProcessor ncproc; map.each(ncproc, "foo"); // Locking both for each element - CPPUNIT_ASSERT_EQUAL(A(4, 7, 0), *map.get(11, "foo")); - CPPUNIT_ASSERT_EQUAL(A(42,1, 0), *map.get(14, "foo")); - CPPUNIT_ASSERT_EQUAL(A(1, 3, 3), *map.get(16, "foo")); + EXPECT_EQ(A(4, 7, 0), *map.get(11, "foo")); + EXPECT_EQ(A(42,1, 0), *map.get(14, "foo")); + EXPECT_EQ(A(1, 3, 3), *map.get(16, "foo")); map.all(ncproc, "foo"); // And for all - CPPUNIT_ASSERT_EQUAL(A(4, 8, 0), *map.get(11, "foo")); - CPPUNIT_ASSERT_EQUAL(A(42,2, 0), *map.get(14, "foo")); - CPPUNIT_ASSERT_EQUAL(A(1, 4, 3), *map.get(16, "foo")); + EXPECT_EQ(A(4, 8, 0), *map.get(11, "foo")); + EXPECT_EQ(A(42,2, 0), *map.get(14, "foo")); + EXPECT_EQ(A(1, 4, 3), *map.get(16, "foo")); } // Test that we can use const functors directly.. map.each(EntryProcessor(), "foo"); @@ -265,12 +196,12 @@ LockableMapTest::testIterating() { std::string expected("11 - A(4, 8, 0)\n" "14 - A(42, 2, 0)\n" "16 - A(1, 4, 3)\n"); - CPPUNIT_ASSERT_EQUAL(expected, proc.toString()); + EXPECT_EQ(expected, proc.toString()); EntryProcessor proc2; map.each(proc2, "foo", 12, 15); expected = "14 - A(42, 2, 0)\n"; - CPPUNIT_ASSERT_EQUAL(expected, proc2.toString()); + EXPECT_EQ(expected, proc2.toString()); } // Test that we can abort iterating { @@ -281,7 +212,7 @@ LockableMapTest::testIterating() { map.each(proc, "foo"); std::string expected("11 - A(4, 8, 0)\n" "14 - A(42, 2, 0)\n"); - CPPUNIT_ASSERT_EQUAL(expected, proc.toString()); + EXPECT_EQ(expected, proc.toString()); } // Test that we can remove during iteration { @@ -293,19 +224,16 @@ LockableMapTest::testIterating() { std::string expected("11 - A(4, 8, 0)\n" "14 - A(42, 2, 0)\n" "16 - A(1, 4, 3)\n"); - CPPUNIT_ASSERT_EQUAL(expected, proc.toString()); - CPPUNIT_ASSERT_EQUAL_MSG(map.toString(), - (Map::size_type) 2, map.size()); - CPPUNIT_ASSERT_EQUAL(A(4, 8, 0), *map.get(11, "foo")); - CPPUNIT_ASSERT_EQUAL(A(1, 4, 3), *map.get(16, "foo")); + EXPECT_EQ(expected, proc.toString()); + EXPECT_EQ((Map::size_type) 2, map.size()) << map.toString(); + EXPECT_EQ(A(4, 8, 0), *map.get(11, "foo")); + EXPECT_EQ(A(1, 4, 3), *map.get(16, "foo")); Map::WrappedEntry entry = map.get(14, "foo"); - CPPUNIT_ASSERT(!entry.exist()); + EXPECT_FALSE(entry.exist()); } } -void -LockableMapTest::testChunkedIterationIsTransparentAcrossChunkSizes() -{ +TEST(LockableMapTest, chunked_iteration_is_transparent_across_chunk_sizes) { Map map; bool preExisted; map.insert(16, A(1, 2, 3), "foo", preExisted); @@ -314,19 +242,17 @@ LockableMapTest::testChunkedIterationIsTransparentAcrossChunkSizes() NonConstProcessor ncproc; // Increments 2nd value in all entries. // chunkedAll with chunk size of 1 map.chunkedAll(ncproc, "foo", 1); - CPPUNIT_ASSERT_EQUAL(A(4, 7, 0), *map.get(11, "foo")); - CPPUNIT_ASSERT_EQUAL(A(42, 1, 0), *map.get(14, "foo")); - CPPUNIT_ASSERT_EQUAL(A(1, 3, 3), *map.get(16, "foo")); + EXPECT_EQ(A(4, 7, 0), *map.get(11, "foo")); + EXPECT_EQ(A(42, 1, 0), *map.get(14, "foo")); + EXPECT_EQ(A(1, 3, 3), *map.get(16, "foo")); // chunkedAll with chunk size larger than db size map.chunkedAll(ncproc, "foo", 100); - CPPUNIT_ASSERT_EQUAL(A(4, 8, 0), *map.get(11, "foo")); - CPPUNIT_ASSERT_EQUAL(A(42, 2, 0), *map.get(14, "foo")); - CPPUNIT_ASSERT_EQUAL(A(1, 4, 3), *map.get(16, "foo")); + EXPECT_EQ(A(4, 8, 0), *map.get(11, "foo")); + EXPECT_EQ(A(42, 2, 0), *map.get(14, "foo")); + EXPECT_EQ(A(1, 4, 3), *map.get(16, "foo")); } -void -LockableMapTest::testCanAbortDuringChunkedIteration() -{ +TEST(LockableMapTest, can_abort_during_chunked_iteration) { Map map; bool preExisted; map.insert(16, A(1, 2, 3), "foo", preExisted); @@ -340,243 +266,10 @@ LockableMapTest::testCanAbortDuringChunkedIteration() map.chunkedAll(proc, "foo", 100); std::string expected("11 - A(4, 6, 0)\n" "14 - A(42, 0, 0)\n"); - CPPUNIT_ASSERT_EQUAL(expected, proc.toString()); + EXPECT_EQ(expected, proc.toString()); } -namespace { - struct LoadGiver : public document::Runnable { - typedef std::shared_ptr<LoadGiver> SP; - Map& _map; - uint32_t _counter; - - LoadGiver(Map& map) : _map(map), _counter(0) {} - ~LoadGiver() __attribute__((noinline)); - }; - - LoadGiver::~LoadGiver() { } - - struct InsertEraseLoadGiver : public LoadGiver { - InsertEraseLoadGiver(Map& map) : LoadGiver(map) {} - - void run() override { - // Screws up order of buckets by xor'ing with 12345. - // Only operate on last 32k super buckets. - while (running()) { - uint32_t bucket = ((_counter ^ 12345) % 0x8000) + 0x8000; - if (bucket % 7 < 3) { - bool preExisted; - _map.insert(bucket, A(bucket, 0, _counter), "foo", - preExisted); - } - if (bucket % 5 < 2) { - _map.erase(bucket, "foo"); - } - ++_counter; - } - } - }; - - struct GetLoadGiver : public LoadGiver { - GetLoadGiver(Map& map) : LoadGiver(map) {} - - void run() override { - // It's legal to keep entries as long as you only request higher - // buckets. So, to test this, keep entries until you request one - // that is smaller than those stored. - std::vector<std::pair<uint32_t, Map::WrappedEntry> > stored; - while (running()) { - uint32_t bucket = (_counter ^ 52721) % 0x10000; - if (!stored.empty() && stored.back().first > bucket) { - stored.clear(); - } - stored.push_back(std::pair<uint32_t, Map::WrappedEntry>( - bucket, _map.get(bucket, "foo", _counter % 3 == 0))); - ++_counter; - } - } - }; - - struct AllLoadGiver : public LoadGiver { - AllLoadGiver(Map& map) : LoadGiver(map) {} - - void run() override { - while (running()) { - _map.all(*this, "foo"); - ++_counter; - } - } - - Map::Decision operator()(int key, A& a) { - //std::cerr << (void*) this << " - " << key << "\n"; - (void) key; - ++a._val2; - return Map::CONTINUE; - } - }; - - struct EachLoadGiver : public LoadGiver { - EachLoadGiver(Map& map) : LoadGiver(map) {} - - void run() override { - while (running()) { - _map.each(*this, "foo"); - ++_counter; - } - } - - Map::Decision operator()(int key, A& a) { - //std::cerr << (void*) this << " - " << key << "\n"; - (void) key; - ++a._val2; - return Map::CONTINUE; - } - }; - - struct RandomRangeLoadGiver : public LoadGiver { - RandomRangeLoadGiver(Map& map) : LoadGiver(map) {} - - void run() override { - while (running()) { - uint32_t min = (_counter ^ 23426) % 0x10000; - uint32_t max = (_counter ^ 40612) % 0x10000; - if (min > max) { - uint32_t tmp = min; - min = max; - max = tmp; - } - if (_counter % 7 < 5) { - _map.each(*this, "foo", min, max); - } else { - _map.all(*this, "foo", min, max); - } - ++_counter; - } - } - - Map::Decision operator()(int key, A& a) { - //std::cerr << "."; - (void) key; - ++a._val2; - return Map::CONTINUE; - } - }; - - struct GetNextLoadGiver : public LoadGiver { - GetNextLoadGiver(Map& map) : LoadGiver(map) {} - - void run() override { - while (running()) { - uint32_t bucket = (_counter ^ 60417) % 0xffff; - if (_counter % 7 < 5) { - _map.each(*this, "foo", bucket + 1, 0xffff); - } else { - _map.all(*this, "foo", bucket + 1, 0xffff); - } - ++_counter; - } - } - - Map::Decision operator()(int key, A& a) { - //std::cerr << "."; - (void) key; - ++a._val2; - return Map::ABORT; - } - }; -} - -void -LockableMapTest::testThreadSafetyStress() { - uint32_t duration = 2 * 1000; - std::cerr << "\nRunning LockableMap threadsafety test for " - << (duration / 1000) << " seconds.\n"; - // Set up multiple threads going through the bucket database at the same - // time. Ensuring all works and there are no deadlocks. - - // Initial database of 32k elements which should always be present. - // Next 32k elements may exist (loadgivers may erase and create them, "foo") - Map map; - for (uint32_t i=0; i<65536; ++i) { - bool preExisted; - map.insert(i, A(i, 0, i ^ 12345), "foo", preExisted); - } - std::vector<LoadGiver::SP> loadgivers; - for (uint32_t i=0; i<8; ++i) { - loadgivers.push_back(LoadGiver::SP(new InsertEraseLoadGiver(map))); - } - for (uint32_t i=0; i<2; ++i) { - loadgivers.push_back(LoadGiver::SP(new GetLoadGiver(map))); - } - for (uint32_t i=0; i<2; ++i) { - loadgivers.push_back(LoadGiver::SP(new AllLoadGiver(map))); - } - for (uint32_t i=0; i<2; ++i) { - loadgivers.push_back(LoadGiver::SP(new EachLoadGiver(map))); - } - for (uint32_t i=0; i<2; ++i) { - loadgivers.push_back(LoadGiver::SP(new RandomRangeLoadGiver(map))); - } - for (uint32_t i=0; i<2; ++i) { - loadgivers.push_back(LoadGiver::SP(new GetNextLoadGiver(map))); - } - - FastOS_ThreadPool pool(128 * 1024); - for (uint32_t i=0; i<loadgivers.size(); ++i) { - CPPUNIT_ASSERT(loadgivers[i]->start(pool)); - } - FastOS_Thread::Sleep(duration); - std::cerr << "Closing down test\n"; - for (uint32_t i=0; i<loadgivers.size(); ++i) { - CPPUNIT_ASSERT(loadgivers[i]->stop()); - } -// FastOS_Thread::Sleep(duration); -// std::cerr << "Didn't manage to shut down\n"; -// map._lockedKeys.print(std::cerr, true, ""); - - for (uint32_t i=0; i<loadgivers.size(); ++i) { - CPPUNIT_ASSERT(loadgivers[i]->join()); - } - std::cerr << "Loadgiver counts:"; - for (uint32_t i=0; i<loadgivers.size(); ++i) { - std::cerr << " " << loadgivers[i]->_counter; - } - std::cerr << "\nTest completed\n"; -} - -#if 0 -namespace { -struct Hex { - document::BucketId::Type val; - - Hex(document::BucketId::Type v) : val(v) {} - bool operator==(const Hex& h) const { return val == h.val; } -}; - -std::ostream& operator<<(std::ostream& out, const Hex& h) { - out << std::hex << h.val << std::dec; - return out; -} - -void -printBucket(const std::string s, const document::BucketId& b) { - std::cerr << s << "bucket=" << b << ", reversed=" << b.stripUnused().toKey() << ", hex=" << Hex(b.stripUnused().toKey()) << "\n"; -} - -void -printBuckets(const std::map<document::BucketId, Map::WrappedEntry>& results) { - for (std::map<document::BucketId, Map::WrappedEntry>::const_iterator iter = results.begin(); - iter != results.end(); - iter++) { - printBucket("Returned ", iter->first); - } -} - -} -#endif - -void -LockableMapTest::testFindBucketsSimple() { -#if __WORDSIZE == 64 +TEST(LockableMapTest, find_buckets_simple) { Map map; document::BucketId id1(17, 0x0ffff); @@ -594,17 +287,13 @@ LockableMapTest::testFindBucketsSimple() { map.insert(id3.toKey(), A(3,4,5), "foo", preExisted); document::BucketId id(22, 0xfffff); - std::map<document::BucketId, Map::WrappedEntry> results = - map.getContained(id, "foo"); + auto results = map.getContained(id, "foo"); - CPPUNIT_ASSERT_EQUAL((size_t)1, results.size()); - CPPUNIT_ASSERT_EQUAL(A(3,4,5), *results[id3]); -#endif + EXPECT_EQ(1, results.size()); + EXPECT_EQ(A(3,4,5), *results[id3]); } -void -LockableMapTest::testFindBuckets() { -#if __WORDSIZE == 64 +TEST(LockableMapTest, find_buckets) { Map map; document::BucketId id1(16, 0x0ffff); @@ -619,20 +308,16 @@ LockableMapTest::testFindBuckets() { map.insert(id4.stripUnused().toKey(), A(4,5,6), "foo", preExisted); document::BucketId id(22, 0xfffff); - std::map<document::BucketId, Map::WrappedEntry> results = - map.getContained(id, "foo"); + auto results = map.getContained(id, "foo"); - CPPUNIT_ASSERT_EQUAL((size_t)3, results.size()); + EXPECT_EQ(3, results.size()); - CPPUNIT_ASSERT_EQUAL(A(1,2,3), *results[id1.stripUnused()]); - CPPUNIT_ASSERT_EQUAL(A(4,5,6), *results[id4.stripUnused()]); - CPPUNIT_ASSERT_EQUAL(A(3,4,5), *results[id3.stripUnused()]); -#endif + EXPECT_EQ(A(1,2,3), *results[id1.stripUnused()]); + EXPECT_EQ(A(4,5,6), *results[id4.stripUnused()]); + EXPECT_EQ(A(3,4,5), *results[id3.stripUnused()]); } -void -LockableMapTest::testFindBuckets2() { // ticket 3121525 -#if __WORDSIZE == 64 +TEST(LockableMapTest, find_buckets_2) { // ticket 3121525 Map map; document::BucketId id1(16, 0x0ffff); @@ -647,20 +332,16 @@ LockableMapTest::testFindBuckets2() { // ticket 3121525 map.insert(id4.stripUnused().toKey(), A(4,5,6), "foo", preExisted); document::BucketId id(22, 0x1ffff); - std::map<document::BucketId, Map::WrappedEntry> results = - map.getContained(id, "foo"); + auto results = map.getContained(id, "foo"); - CPPUNIT_ASSERT_EQUAL((size_t)3, results.size()); + EXPECT_EQ(3, results.size()); - CPPUNIT_ASSERT_EQUAL(A(1,2,3), *results[id1.stripUnused()]); - CPPUNIT_ASSERT_EQUAL(A(4,5,6), *results[id4.stripUnused()]); - CPPUNIT_ASSERT_EQUAL(A(3,4,5), *results[id3.stripUnused()]); -#endif + EXPECT_EQ(A(1,2,3), *results[id1.stripUnused()]); + EXPECT_EQ(A(4,5,6), *results[id4.stripUnused()]); + EXPECT_EQ(A(3,4,5), *results[id3.stripUnused()]); } -void -LockableMapTest::testFindBuckets3() { // ticket 3121525 -#if __WORDSIZE == 64 +TEST(LockableMapTest, find_buckets_3) { // ticket 3121525 Map map; document::BucketId id1(16, 0x0ffff); @@ -671,18 +352,14 @@ LockableMapTest::testFindBuckets3() { // ticket 3121525 map.insert(id2.stripUnused().toKey(), A(2,3,4), "foo", preExisted); document::BucketId id(22, 0x1ffff); - std::map<document::BucketId, Map::WrappedEntry> results = - map.getContained(id, "foo"); + auto results = map.getContained(id, "foo"); - CPPUNIT_ASSERT_EQUAL((size_t)1, results.size()); + EXPECT_EQ(1, results.size()); - CPPUNIT_ASSERT_EQUAL(A(1,2,3), *results[id1.stripUnused()]); -#endif + EXPECT_EQ(A(1,2,3), *results[id1.stripUnused()]); } -void -LockableMapTest::testFindBuckets4() { // ticket 3121525 -#if __WORDSIZE == 64 +TEST(LockableMapTest, find_buckets_4) { // ticket 3121525 Map map; document::BucketId id1(16, 0x0ffff); @@ -695,18 +372,14 @@ LockableMapTest::testFindBuckets4() { // ticket 3121525 map.insert(id3.stripUnused().toKey(), A(3,4,5), "foo", preExisted); document::BucketId id(18, 0x1ffff); - std::map<document::BucketId, Map::WrappedEntry> results = - map.getContained(id, "foo"); + auto results = map.getContained(id, "foo"); - CPPUNIT_ASSERT_EQUAL((size_t)1, results.size()); + EXPECT_EQ(1, results.size()); - CPPUNIT_ASSERT_EQUAL(A(1,2,3), *results[id1.stripUnused()]); -#endif + EXPECT_EQ(A(1,2,3), *results[id1.stripUnused()]); } -void -LockableMapTest::testFindBuckets5() { // ticket 3121525 -#if __WORDSIZE == 64 +TEST(LockableMapTest, find_buckets_5) { // ticket 3121525 Map map; document::BucketId id1(16, 0x0ffff); @@ -719,31 +392,23 @@ LockableMapTest::testFindBuckets5() { // ticket 3121525 map.insert(id3.stripUnused().toKey(), A(3,4,5), "foo", preExisted); document::BucketId id(18, 0x1ffff); - std::map<document::BucketId, Map::WrappedEntry> results = - map.getContained(id, "foo"); + auto results = map.getContained(id, "foo"); - CPPUNIT_ASSERT_EQUAL((size_t)1, results.size()); + EXPECT_EQ(1, results.size()); - CPPUNIT_ASSERT_EQUAL(A(1,2,3), *results[id1.stripUnused()]); -#endif + EXPECT_EQ(A(1,2,3), *results[id1.stripUnused()]); } -void -LockableMapTest::testFindNoBuckets() { -#if __WORDSIZE == 64 +TEST(LockableMapTest, find_no_buckets) { Map map; document::BucketId id(16, 0x0ffff); - std::map<document::BucketId, Map::WrappedEntry> results = - map.getAll(id, "foo"); + auto results = map.getAll(id, "foo"); - CPPUNIT_ASSERT_EQUAL((size_t)0, results.size()); -#endif + EXPECT_EQ(0, results.size()); } -void -LockableMapTest::testFindAll() { -#if __WORDSIZE == 64 +TEST(LockableMapTest, find_all) { Map map; document::BucketId id1(16, 0x0aaaa); // contains id2-id7 @@ -766,45 +431,26 @@ LockableMapTest::testFindAll() { map.insert(id7.stripUnused().toKey(), A(7,8,9), "foo", preExisted); map.insert(id8.stripUnused().toKey(), A(8,9,10), "foo", preExisted); map.insert(id9.stripUnused().toKey(), A(9,10,11), "foo", preExisted); - //printBucket("Inserted ", id1); - //printBucket("Inserted ", id2); - //printBucket("Inserted ", id3); - //printBucket("Inserted ", id4); - //printBucket("Inserted ", id5); - //printBucket("Inserted ", id6); - //printBucket("Inserted ", id7); - //printBucket("Inserted ", id8); - //printBucket("Inserted ", id9); document::BucketId id(17, 0x1aaaa); - std::map<document::BucketId, Map::WrappedEntry> results = - map.getAll(id, "foo"); - - //std::cerr << "Done: getAll() for bucket " << id << "\n"; - //printBuckets(results); + auto results = map.getAll(id, "foo"); - CPPUNIT_ASSERT_EQUAL((size_t)4, results.size()); + EXPECT_EQ(4, results.size()); - CPPUNIT_ASSERT_EQUAL(A(1,2,3), *results[id1.stripUnused()]); // super bucket - CPPUNIT_ASSERT_EQUAL(A(5,6,7), *results[id5.stripUnused()]); // most specific match (exact match) - CPPUNIT_ASSERT_EQUAL(A(6,7,8), *results[id6.stripUnused()]); // sub bucket - CPPUNIT_ASSERT_EQUAL(A(7,8,9), *results[id7.stripUnused()]); // sub bucket + EXPECT_EQ(A(1,2,3), *results[id1.stripUnused()]); // super bucket + EXPECT_EQ(A(5,6,7), *results[id5.stripUnused()]); // most specific match (exact match) + EXPECT_EQ(A(6,7,8), *results[id6.stripUnused()]); // sub bucket + EXPECT_EQ(A(7,8,9), *results[id7.stripUnused()]); // sub bucket id = document::BucketId(16, 0xffff); results = map.getAll(id, "foo"); - //std::cerr << "Done: getAll() for bucket " << id << "\n"; - //printBuckets(results); + EXPECT_EQ(1, results.size()); - CPPUNIT_ASSERT_EQUAL((size_t)1, results.size()); - - CPPUNIT_ASSERT_EQUAL(A(9,10,11), *results[id9.stripUnused()]); // sub bucket -#endif + EXPECT_EQ(A(9,10,11), *results[id9.stripUnused()]); // sub bucket } -void -LockableMapTest::testFindAll2() { // Ticket 3121525 -#if __WORDSIZE == 64 +TEST(LockableMapTest, find_all_2) { // Ticket 3121525 Map map; document::BucketId id1(17, 0x00001); @@ -815,19 +461,15 @@ LockableMapTest::testFindAll2() { // Ticket 3121525 map.insert(id2.stripUnused().toKey(), A(2,3,4), "foo", preExisted); document::BucketId id(16, 0x00001); - std::map<document::BucketId, Map::WrappedEntry> results = - map.getAll(id, "foo"); + auto results = map.getAll(id, "foo"); - CPPUNIT_ASSERT_EQUAL((size_t)2, results.size()); + EXPECT_EQ(2, results.size()); - CPPUNIT_ASSERT_EQUAL(A(1,2,3), *results[id1.stripUnused()]); // sub bucket - CPPUNIT_ASSERT_EQUAL(A(2,3,4), *results[id2.stripUnused()]); // sub bucket -#endif + EXPECT_EQ(A(1,2,3), *results[id1.stripUnused()]); // sub bucket + EXPECT_EQ(A(2,3,4), *results[id2.stripUnused()]); // sub bucket } -void -LockableMapTest::testFindAllUnusedBitIsSet() { // ticket 2938896 -#if __WORDSIZE == 64 +TEST(LockableMapTest, find_all_unused_bit_is_set) { // ticket 2938896 Map map; document::BucketId id1(24, 0x000dc7089); @@ -843,19 +485,15 @@ LockableMapTest::testFindAllUnusedBitIsSet() { // ticket 2938896 document::BucketId id(33, 0x1053c7089); id.setUsedBits(32); // Bit 33 is set, but unused - std::map<document::BucketId, Map::WrappedEntry> results = - map.getAll(id, "foo"); + auto results = map.getAll(id, "foo"); - CPPUNIT_ASSERT_EQUAL((size_t)2, results.size()); + EXPECT_EQ(2, results.size()); - CPPUNIT_ASSERT_EQUAL(A(2,3,4), *results[id2.stripUnused()]); // sub bucket - CPPUNIT_ASSERT_EQUAL(A(3,4,5), *results[id3.stripUnused()]); // sub bucket -#endif + EXPECT_EQ(A(2,3,4), *results[id2.stripUnused()]); // sub bucket + EXPECT_EQ(A(3,4,5), *results[id3.stripUnused()]); // sub bucket } -void -LockableMapTest::testFindAllInconsistentlySplit() { // Ticket 2938896 -#if __WORDSIZE == 64 +TEST(LockableMapTest, find_all_inconsistently_split) { // Ticket 2938896 Map map; document::BucketId id1(16, 0x00001); // contains id2-id3 @@ -868,20 +506,16 @@ LockableMapTest::testFindAllInconsistentlySplit() { // Ticket 2938896 map.insert(id3.stripUnused().toKey(), A(3,4,5), "foo", preExisted); document::BucketId id(16, 0x00001); - std::map<document::BucketId, Map::WrappedEntry> results = - map.getAll(id, "foo"); + auto results = map.getAll(id, "foo"); - CPPUNIT_ASSERT_EQUAL((size_t)3, results.size()); + EXPECT_EQ(3, results.size()); - CPPUNIT_ASSERT_EQUAL(A(1,2,3), *results[id1.stripUnused()]); // most specific match (exact match) - CPPUNIT_ASSERT_EQUAL(A(2,3,4), *results[id2.stripUnused()]); // sub bucket - CPPUNIT_ASSERT_EQUAL(A(3,4,5), *results[id3.stripUnused()]); // sub bucket -#endif + EXPECT_EQ(A(1,2,3), *results[id1.stripUnused()]); // most specific match (exact match) + EXPECT_EQ(A(2,3,4), *results[id2.stripUnused()]); // sub bucket + EXPECT_EQ(A(3,4,5), *results[id3.stripUnused()]); // sub bucket } -void -LockableMapTest::testFindAllInconsistentlySplit2() { // ticket 3121525 -#if __WORDSIZE == 64 +TEST(LockableMapTest, find_all_inconsistently_split_2) { // ticket 3121525 Map map; document::BucketId id1(17, 0x10000); @@ -896,19 +530,15 @@ LockableMapTest::testFindAllInconsistentlySplit2() { // ticket 3121525 map.insert(id4.stripUnused().toKey(), A(4,5,6), "foo", preExisted); document::BucketId id(32, 0x027228034); - std::map<document::BucketId, Map::WrappedEntry> results = - map.getAll(id, "foo"); + auto results = map.getAll(id, "foo"); - CPPUNIT_ASSERT_EQUAL((size_t)2, results.size()); + EXPECT_EQ(2, results.size()); - CPPUNIT_ASSERT_EQUAL(A(2,3,4), *results[id2.stripUnused()]); // super bucket - CPPUNIT_ASSERT_EQUAL(A(3,4,5), *results[id3.stripUnused()]); // most specific match (super bucket) -#endif + EXPECT_EQ(A(2,3,4), *results[id2.stripUnused()]); // super bucket + EXPECT_EQ(A(3,4,5), *results[id3.stripUnused()]); // most specific match (super bucket) } -void -LockableMapTest::testFindAllInconsistentlySplit3() { // ticket 3121525 -#if __WORDSIZE == 64 +TEST(LockableMapTest, find_all_inconsistently_split_3) { // ticket 3121525 Map map; document::BucketId id1(16, 0x0ffff); // contains id2 @@ -919,18 +549,14 @@ LockableMapTest::testFindAllInconsistentlySplit3() { // ticket 3121525 map.insert(id2.stripUnused().toKey(), A(2,3,4), "foo", preExisted); document::BucketId id(22, 0x1ffff); - std::map<document::BucketId, Map::WrappedEntry> results = - map.getAll(id, "foo"); + auto results = map.getAll(id, "foo"); - CPPUNIT_ASSERT_EQUAL((size_t)1, results.size()); + EXPECT_EQ(1, results.size()); - CPPUNIT_ASSERT_EQUAL(A(1,2,3), *results[id1.stripUnused()]); // super bucket -#endif + EXPECT_EQ(A(1,2,3), *results[id1.stripUnused()]); // super bucket } -void -LockableMapTest::testFindAllInconsistentlySplit4() { // ticket 3121525 -#if __WORDSIZE == 64 +TEST(LockableMapTest, find_all_inconsistently_split_4) { // ticket 3121525 Map map; document::BucketId id1(16, 0x0ffff); // contains id2-id3 @@ -943,19 +569,15 @@ LockableMapTest::testFindAllInconsistentlySplit4() { // ticket 3121525 map.insert(id3.stripUnused().toKey(), A(3,4,5), "foo", preExisted); document::BucketId id(18, 0x1ffff); - std::map<document::BucketId, Map::WrappedEntry> results = - map.getAll(id, "foo"); + auto results = map.getAll(id, "foo"); - CPPUNIT_ASSERT_EQUAL((size_t)2, results.size()); + EXPECT_EQ(2, results.size()); - CPPUNIT_ASSERT_EQUAL(A(1,2,3), *results[id1.stripUnused()]); // super bucket - CPPUNIT_ASSERT_EQUAL(A(3,4,5), *results[id3.stripUnused()]); // sub bucket -#endif + EXPECT_EQ(A(1,2,3), *results[id1.stripUnused()]); // super bucket + EXPECT_EQ(A(3,4,5), *results[id3.stripUnused()]); // sub bucket } -void -LockableMapTest::testFindAllInconsistentlySplit5() { // ticket 3121525 -#if __WORDSIZE == 64 +TEST(LockableMapTest, find_all_inconsistently_split_5) { // ticket 3121525 Map map; document::BucketId id1(16, 0x0ffff); // contains id2-id3 @@ -968,18 +590,15 @@ LockableMapTest::testFindAllInconsistentlySplit5() { // ticket 3121525 map.insert(id3.stripUnused().toKey(), A(3,4,5), "foo", preExisted); document::BucketId id(18, 0x1ffff); - std::map<document::BucketId, Map::WrappedEntry> results = - map.getAll(id, "foo"); + auto results = map.getAll(id, "foo"); - CPPUNIT_ASSERT_EQUAL((size_t)2, results.size()); + EXPECT_EQ(2, results.size()); - CPPUNIT_ASSERT_EQUAL(A(1,2,3), *results[id1.stripUnused()]); // super bucket - CPPUNIT_ASSERT_EQUAL(A(3,4,5), *results[id3.stripUnused()]); // sub bucket -#endif + EXPECT_EQ(A(1,2,3), *results[id1.stripUnused()]); // super bucket + EXPECT_EQ(A(3,4,5), *results[id3.stripUnused()]); // sub bucket } -void -LockableMapTest::testFindAllInconsistentlySplit6() { +TEST(LockableMapTest, find_all_inconsistently_split_6) { Map map; document::BucketId id1(16, 0x0ffff); // contains id2-id3 @@ -992,18 +611,15 @@ LockableMapTest::testFindAllInconsistentlySplit6() { map.insert(id3.stripUnused().toKey(), A(3,4,5), "foo", preExisted); document::BucketId id(18, 0x3ffff); - std::map<document::BucketId, Map::WrappedEntry> results = - map.getAll(id, "foo"); + auto results = map.getAll(id, "foo"); - CPPUNIT_ASSERT_EQUAL((size_t)2, results.size()); + EXPECT_EQ(2, results.size()); - CPPUNIT_ASSERT_EQUAL(A(1,2,3), *results[id1.stripUnused()]); // super bucket - CPPUNIT_ASSERT_EQUAL(A(3,4,5), *results[id3.stripUnused()]); // sub bucket + EXPECT_EQ(A(1,2,3), *results[id1.stripUnused()]); // super bucket + EXPECT_EQ(A(3,4,5), *results[id3.stripUnused()]); // sub bucket } -void -LockableMapTest::testFindAllInconsistentBelow16Bits() -{ +TEST(LockableMapTest, find_all_inconsistent_below_16_bits) { Map map; document::BucketId id1(1, 0x1); // contains id2-id3 @@ -1017,50 +633,40 @@ LockableMapTest::testFindAllInconsistentBelow16Bits() document::BucketId id(3, 0x5); - std::map<document::BucketId, Map::WrappedEntry> results = - map.getAll(id, "foo"); + auto results = map.getAll(id, "foo"); - CPPUNIT_ASSERT_EQUAL(size_t(2), results.size()); + EXPECT_EQ(2, results.size()); - CPPUNIT_ASSERT_EQUAL(A(1,2,3), *results[id1.stripUnused()]); // super bucket - CPPUNIT_ASSERT_EQUAL(A(3,4,5), *results[id3.stripUnused()]); // sub bucket + EXPECT_EQ(A(1,2,3), *results[id1.stripUnused()]); // super bucket + EXPECT_EQ(A(3,4,5), *results[id3.stripUnused()]); // sub bucket } -void -LockableMapTest::testCreate() { -#if __WORDSIZE == 64 +TEST(LockableMapTest, create) { Map map; { document::BucketId id1(58, 0x43d6c878000004d2ull); - std::map<document::BucketId, Map::WrappedEntry> entries( - map.getContained(id1, "foo")); + auto entries = map.getContained(id1, "foo"); - CPPUNIT_ASSERT_EQUAL((size_t)0, entries.size()); + EXPECT_EQ(0, entries.size()); Map::WrappedEntry entry = map.createAppropriateBucket(36, "", id1); - CPPUNIT_ASSERT_EQUAL(document::BucketId(36,0x8000004d2ull), - entry.getBucketId()); + EXPECT_EQ(document::BucketId(36,0x8000004d2ull), entry.getBucketId()); } { document::BucketId id1(58, 0x423bf1e0000004d2ull); - std::map<document::BucketId, Map::WrappedEntry> entries( - map.getContained(id1, "foo")); - CPPUNIT_ASSERT_EQUAL((size_t)0, entries.size()); + auto entries = map.getContained(id1, "foo"); + EXPECT_EQ(0, entries.size()); Map::WrappedEntry entry = map.createAppropriateBucket(36, "", id1); - CPPUNIT_ASSERT_EQUAL(document::BucketId(36,0x0000004d2ull), - entry.getBucketId()); + EXPECT_EQ(document::BucketId(36,0x0000004d2ull), entry.getBucketId()); } - CPPUNIT_ASSERT_EQUAL((size_t)2, map.size()); -#endif + EXPECT_EQ(2, map.size()); } -void -LockableMapTest::testCreate2() { -#if __WORDSIZE == 64 +TEST(LockableMapTest, create_2) { Map map; { document::BucketId id1(58, 0xeaf77782000004d2); @@ -1069,24 +675,19 @@ LockableMapTest::testCreate2() { } { document::BucketId id1(58, 0x00000000000004d2); - std::map<document::BucketId, Map::WrappedEntry> entries( - map.getContained(id1, "foo")); + auto entries = map.getContained(id1, "foo"); - CPPUNIT_ASSERT_EQUAL((size_t)0, entries.size()); + EXPECT_EQ(0, entries.size()); Map::WrappedEntry entry = map.createAppropriateBucket(16, "", id1); - CPPUNIT_ASSERT_EQUAL(document::BucketId(34, 0x0000004d2ull), - entry.getBucketId()); + EXPECT_EQ(document::BucketId(34, 0x0000004d2ull), entry.getBucketId()); } - CPPUNIT_ASSERT_EQUAL((size_t)2, map.size()); -#endif + EXPECT_EQ(2, map.size()); } -void -LockableMapTest::testCreate3() { -#if __WORDSIZE == 64 +TEST(LockableMapTest, create_3) { Map map; { document::BucketId id1(58, 0xeaf77780000004d2); @@ -1100,21 +701,16 @@ LockableMapTest::testCreate3() { } { document::BucketId id1(58, 0x00000000000004d2); - std::map<document::BucketId, Map::WrappedEntry> entries( - map.getContained(id1, "foo")); + auto entries = map.getContained(id1, "foo"); - CPPUNIT_ASSERT_EQUAL((size_t)0, entries.size()); + EXPECT_EQ(0, entries.size()); Map::WrappedEntry entry = map.createAppropriateBucket(16, "", id1); - CPPUNIT_ASSERT_EQUAL(document::BucketId(40, 0x0000004d2ull), - entry.getBucketId()); + EXPECT_EQ(document::BucketId(40, 0x0000004d2ull), entry.getBucketId()); } -#endif } -void -LockableMapTest::testCreate4() { -#if __WORDSIZE == 64 +TEST(LockableMapTest, create_4) { Map map; { document::BucketId id1(16, 0x00000000000004d1); @@ -1130,15 +726,11 @@ LockableMapTest::testCreate4() { document::BucketId id1(58, 0x00000000010004d2); Map::WrappedEntry entry = map.createAppropriateBucket(16, "", id1); - CPPUNIT_ASSERT_EQUAL(document::BucketId(25, 0x0010004d2ull), - entry.getBucketId()); + EXPECT_EQ(document::BucketId(25, 0x0010004d2ull), entry.getBucketId()); } -#endif } -void -LockableMapTest::testCreate6() { -#if __WORDSIZE == 64 +TEST(LockableMapTest, create_5) { Map map; { document::BucketId id1(0x8c000000000004d2); @@ -1165,16 +757,11 @@ LockableMapTest::testCreate6() { { document::BucketId id1(0xe9944a44000004d2); Map::WrappedEntry entry = map.createAppropriateBucket(16, "", id1); - CPPUNIT_ASSERT_EQUAL(document::BucketId(0x90000004000004d2), - entry.getBucketId()); + EXPECT_EQ(document::BucketId(0x90000004000004d2), entry.getBucketId()); } -#endif } - -void -LockableMapTest::testCreate5() { -#if __WORDSIZE == 64 +TEST(LockableMapTest, create_6) { Map map; { document::BucketId id1(58, 0xeaf77780000004d2); @@ -1190,28 +777,20 @@ LockableMapTest::testCreate5() { { document::BucketId id1(58, 0x00000000010004d2); Map::WrappedEntry entry = map.createAppropriateBucket(16, "", id1); - CPPUNIT_ASSERT_EQUAL(document::BucketId(25, 0x0010004d2ull), - entry.getBucketId()); + EXPECT_EQ(document::BucketId(25, 0x0010004d2ull), entry.getBucketId()); } -#endif } -void -LockableMapTest::testCreateEmpty() { -#if __WORDSIZE == 64 +TEST(LockableMapTest, create_empty) { Map map; { document::BucketId id1(58, 0x00000000010004d2); Map::WrappedEntry entry = map.createAppropriateBucket(16, "", id1); - CPPUNIT_ASSERT_EQUAL(document::BucketId(16, 0x0000004d2ull), - entry.getBucketId()); + EXPECT_EQ(document::BucketId(16, 0x0000004d2ull), entry.getBucketId()); } -#endif } -void -LockableMapTest::testIsConsistent() -{ +TEST(LockableMapTest, is_consistent) { Map map; document::BucketId id1(16, 0x00001); // contains id2-id3 document::BucketId id2(17, 0x00001); @@ -1221,13 +800,13 @@ LockableMapTest::testIsConsistent() { Map::WrappedEntry entry( map.get(id1.stripUnused().toKey(), "foo", true)); - CPPUNIT_ASSERT(map.isConsistent(entry)); + EXPECT_TRUE(map.isConsistent(entry)); } map.insert(id2.stripUnused().toKey(), A(1,2,3), "foo", preExisted); { Map::WrappedEntry entry( map.get(id1.stripUnused().toKey(), "foo", true)); - CPPUNIT_ASSERT(!map.isConsistent(entry)); + EXPECT_FALSE(map.isConsistent(entry)); } } diff --git a/storage/src/tests/distributor/messagesenderstub.h b/storage/src/tests/distributor/messagesenderstub.h index b86863890a1..1b526813ef7 100644 --- a/storage/src/tests/distributor/messagesenderstub.h +++ b/storage/src/tests/distributor/messagesenderstub.h @@ -4,6 +4,7 @@ #include <vespa/storage/distributor/distributormessagesender.h> #include <cassert> #include <vector> +#include <string> namespace storage { diff --git a/storage/src/vespa/storage/bucketdb/lockablemap.hpp b/storage/src/vespa/storage/bucketdb/lockablemap.hpp index 440a76b92fd..3cef17c9025 100644 --- a/storage/src/vespa/storage/bucketdb/lockablemap.hpp +++ b/storage/src/vespa/storage/bucketdb/lockablemap.hpp @@ -8,6 +8,7 @@ #include <vespa/vespalib/stllike/hash_set.hpp> #include <thread> #include <chrono> +#include <ostream> namespace storage { diff --git a/storage/src/vespa/storage/bucketdb/stdmapwrapper.h b/storage/src/vespa/storage/bucketdb/stdmapwrapper.h index 8cd2d6a7578..889227f1747 100644 --- a/storage/src/vespa/storage/bucketdb/stdmapwrapper.h +++ b/storage/src/vespa/storage/bucketdb/stdmapwrapper.h @@ -12,6 +12,7 @@ #include <map> #include <vespa/vespalib/util/printable.h> +#include <ostream> namespace storage { diff --git a/storage/src/vespa/storage/common/storagelinkqueued.cpp b/storage/src/vespa/storage/common/storagelinkqueued.cpp index 5b3aacd11de..036a1269958 100644 --- a/storage/src/vespa/storage/common/storagelinkqueued.cpp +++ b/storage/src/vespa/storage/common/storagelinkqueued.cpp @@ -65,7 +65,7 @@ void StorageLinkQueued::logError(const char* err) { }; void StorageLinkQueued::logDebug(const char* err) { - LOG(info, "%s", err); + LOG(debug, "%s", err); }; template class StorageLinkQueued::Dispatcher<storage::api::StorageMessage>; diff --git a/storage/src/vespa/storage/distributor/maintenance/simplemaintenancescanner.cpp b/storage/src/vespa/storage/distributor/maintenance/simplemaintenancescanner.cpp index 799b3641c64..e143f4d8570 100644 --- a/storage/src/vespa/storage/distributor/maintenance/simplemaintenancescanner.cpp +++ b/storage/src/vespa/storage/distributor/maintenance/simplemaintenancescanner.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "simplemaintenancescanner.h" #include <vespa/storage/distributor/distributor_bucket_space.h> +#include <ostream> namespace storage::distributor { diff --git a/storage/src/vespa/storage/distributor/messagetracker.h b/storage/src/vespa/storage/distributor/messagetracker.h index 626335e1ba6..75ae287d98f 100644 --- a/storage/src/vespa/storage/distributor/messagetracker.h +++ b/storage/src/vespa/storage/distributor/messagetracker.h @@ -4,6 +4,7 @@ #include <vespa/storage/common/messagesender.h> #include <vector> #include <map> +#include <string> namespace storage::api { class BucketCommand; diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.cpp b/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.cpp index 784ea7253b6..66ce4fc0485 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.cpp @@ -3,6 +3,7 @@ #include <vespa/storage/distributor/idealstatemanager.h> #include <vespa/storage/distributor/distributor_bucket_space.h> #include <vespa/storage/distributor/pendingmessagetracker.h> +#include <array> #include <vespa/log/bufferedlogger.h> LOG_SETUP(".distributor.operation.idealstate.merge"); diff --git a/storage/src/vespa/storage/persistence/diskthread.h b/storage/src/vespa/storage/persistence/diskthread.h index 0489ad3144a..926b766aca0 100644 --- a/storage/src/vespa/storage/persistence/diskthread.h +++ b/storage/src/vespa/storage/persistence/diskthread.h @@ -16,6 +16,7 @@ #include <vespa/vespalib/util/document_runnable.h> #include <vespa/config-stor-filestor.h> #include <vespa/storageframework/generic/thread/runnable.h> +#include <ostream> namespace storage { diff --git a/storage/src/vespa/storage/persistence/filestorage/mergestatus.cpp b/storage/src/vespa/storage/persistence/filestorage/mergestatus.cpp index 86c70a4a287..24f4c9cd731 100644 --- a/storage/src/vespa/storage/persistence/filestorage/mergestatus.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/mergestatus.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "mergestatus.h" +#include <ostream> #include <vespa/log/log.h> LOG_SETUP(".mergestatus"); diff --git a/storage/src/vespa/storage/persistence/messages.cpp b/storage/src/vespa/storage/persistence/messages.cpp index f38d3d0fac3..61a10f71868 100644 --- a/storage/src/vespa/storage/persistence/messages.cpp +++ b/storage/src/vespa/storage/persistence/messages.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "messages.h" +#include <ostream> using document::BucketSpace; diff --git a/storage/src/vespa/storage/tools/generatedistributionbits.cpp b/storage/src/vespa/storage/tools/generatedistributionbits.cpp index 73f11b39e67..98c5e56b90c 100644 --- a/storage/src/vespa/storage/tools/generatedistributionbits.cpp +++ b/storage/src/vespa/storage/tools/generatedistributionbits.cpp @@ -9,6 +9,7 @@ #include <iomanip> #include <iostream> #include <algorithm> +#include <sstream> #include <vespa/config-stor-distribution.h> namespace storage { diff --git a/storage/src/vespa/storage/visiting/commandqueue.h b/storage/src/vespa/storage/visiting/commandqueue.h index 8cc565884ec..d129506eb64 100644 --- a/storage/src/vespa/storage/visiting/commandqueue.h +++ b/storage/src/vespa/storage/visiting/commandqueue.h @@ -19,6 +19,7 @@ #include <vespa/fastos/timestamp.h> #include <vespa/storageframework/generic/clock/clock.h> #include <list> +#include <ostream> namespace storage { diff --git a/storage/src/vespa/storage/visiting/visitor.h b/storage/src/vespa/storage/visiting/visitor.h index c8d34139364..f53ca5a60a0 100644 --- a/storage/src/vespa/storage/visiting/visitor.h +++ b/storage/src/vespa/storage/visiting/visitor.h @@ -24,6 +24,7 @@ #include <vespa/persistence/spi/read_consistency.h> #include <list> #include <deque> +#include <ostream> namespace document { class Document; diff --git a/storageapi/src/vespa/storageapi/message/datagram.cpp b/storageapi/src/vespa/storageapi/message/datagram.cpp index 66b753185a4..3376761ee41 100644 --- a/storageapi/src/vespa/storageapi/message/datagram.cpp +++ b/storageapi/src/vespa/storageapi/message/datagram.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "datagram.h" +#include <ostream> using document::BucketSpace; diff --git a/storageapi/src/vespa/storageapi/message/documentsummary.cpp b/storageapi/src/vespa/storageapi/message/documentsummary.cpp index e1bf84a8e7c..1d81c6a4c16 100644 --- a/storageapi/src/vespa/storageapi/message/documentsummary.cpp +++ b/storageapi/src/vespa/storageapi/message/documentsummary.cpp @@ -1,5 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "documentsummary.h" +#include <ostream> namespace storage { namespace api { diff --git a/storageapi/src/vespa/storageapi/message/queryresult.cpp b/storageapi/src/vespa/storageapi/message/queryresult.cpp index 67b39d19f6d..bf81083b954 100644 --- a/storageapi/src/vespa/storageapi/message/queryresult.cpp +++ b/storageapi/src/vespa/storageapi/message/queryresult.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "queryresult.h" +#include <ostream> namespace storage { namespace api { diff --git a/storageapi/src/vespa/storageapi/message/searchresult.cpp b/storageapi/src/vespa/storageapi/message/searchresult.cpp index caa698fa67b..4299109e6b5 100644 --- a/storageapi/src/vespa/storageapi/message/searchresult.cpp +++ b/storageapi/src/vespa/storageapi/message/searchresult.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "searchresult.h" +#include <ostream> using vdslib::SearchResult; diff --git a/storageapi/src/vespa/storageapi/message/visitor.cpp b/storageapi/src/vespa/storageapi/message/visitor.cpp index f5850de98ad..8adec61a34e 100644 --- a/storageapi/src/vespa/storageapi/message/visitor.cpp +++ b/storageapi/src/vespa/storageapi/message/visitor.cpp @@ -3,6 +3,7 @@ #include "visitor.h" #include <vespa/vespalib/util/array.hpp> #include <climits> +#include <ostream> namespace storage::api { diff --git a/storageframework/src/vespa/storageframework/generic/clock/time.cpp b/storageframework/src/vespa/storageframework/generic/clock/time.cpp index 0e8178b03bb..7ead5e50077 100644 --- a/storageframework/src/vespa/storageframework/generic/clock/time.cpp +++ b/storageframework/src/vespa/storageframework/generic/clock/time.cpp @@ -5,6 +5,7 @@ #include <iomanip> #include <vector> #include <cassert> +#include <sstream> namespace storage { namespace framework { diff --git a/streamingvisitors/src/vespa/searchvisitor/queryenvironment.h b/streamingvisitors/src/vespa/searchvisitor/queryenvironment.h index b9391ac838c..db0f95cb6bb 100644 --- a/streamingvisitors/src/vespa/searchvisitor/queryenvironment.h +++ b/streamingvisitors/src/vespa/searchvisitor/queryenvironment.h @@ -54,6 +54,8 @@ public: // inherit documentation virtual const search::attribute::IAttributeContext & getAttributeContext() const override { return *_attrCtx; } + double get_average_field_length(const vespalib::string &) const override { return 1.0; } + // inherit documentation virtual const search::fef::IIndexEnvironment & getIndexEnvironment() const override { return _indexEnv; } diff --git a/tenant-auth/OWNERS b/tenant-auth/OWNERS new file mode 100644 index 00000000000..d0a102ecbf4 --- /dev/null +++ b/tenant-auth/OWNERS @@ -0,0 +1 @@ +jonmv diff --git a/tenant-auth/README.md b/tenant-auth/README.md new file mode 100644 index 00000000000..0514b68400e --- /dev/null +++ b/tenant-auth/README.md @@ -0,0 +1 @@ +# Utilities that authenticate users to the hosted Vespa API, or to hosted Vespa applications. diff --git a/tenant-auth/pom.xml b/tenant-auth/pom.xml new file mode 100644 index 00000000000..be8b42dd6c2 --- /dev/null +++ b/tenant-auth/pom.xml @@ -0,0 +1,40 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<project xmlns="http://maven.apache.org/POM/4.0.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> + + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>com.yahoo.vespa</groupId> + <artifactId>parent</artifactId> + <version>7-SNAPSHOT</version> + <relativePath>../parent/pom.xml</relativePath> + </parent> + <artifactId>tenant-auth</artifactId> + <description>Provides resources for authenticating with the hosted Vespa API or application containers</description> + + <dependencies> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>hosted-api</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>config-provisioning</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>security-utils</artifactId> + <version>${project.version}</version> + </dependency> + + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + </dependencies> +</project> diff --git a/tenant-auth/src/main/java/ai/vespa/hosted/auth/Authenticator.java b/tenant-auth/src/main/java/ai/vespa/hosted/auth/Authenticator.java new file mode 100644 index 00000000000..6ecf1100630 --- /dev/null +++ b/tenant-auth/src/main/java/ai/vespa/hosted/auth/Authenticator.java @@ -0,0 +1,73 @@ +package ai.vespa.hosted.auth; + +import ai.vespa.hosted.api.ControllerHttpClient; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.security.KeyUtils; +import com.yahoo.security.SslContextBuilder; +import com.yahoo.security.X509CertificateUtils; + +import javax.net.ssl.SSLContext; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.net.URI; +import java.net.http.HttpRequest; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.security.PrivateKey; +import java.security.cert.X509Certificate; +import java.time.Instant; +import java.util.Optional; + +/** + * Authenticates {@link HttpRequest}s against a hosted Vespa application based on mutual TLS. + * + * @author jonmv + */ +public class Authenticator { + + /** Returns an SSLContext from "key" and "cert" files found under {@code System.getProperty("vespa.test.credentials.root")}. */ + public SSLContext sslContext() { + try { + Path credentialsRoot = Path.of(System.getProperty("vespa.test.credentials.root")); + Path certificateFile = credentialsRoot.resolve("cert"); + Path privateKeyFile = credentialsRoot.resolve("key"); + + X509Certificate certificate = X509CertificateUtils.fromPem(new String(Files.readAllBytes(certificateFile))); + if (Instant.now().isBefore(certificate.getNotBefore().toInstant()) + || Instant.now().isAfter(certificate.getNotAfter().toInstant())) + throw new IllegalStateException("Certificate at '" + certificateFile + "' is valid between " + + certificate.getNotBefore() + " and " + certificate.getNotAfter() + " — not now."); + + PrivateKey privateKey = KeyUtils.fromPemEncodedPrivateKey(new String(Files.readAllBytes(privateKeyFile))); + return new SslContextBuilder().withKeyStore(privateKey, certificate).build(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + public HttpRequest.Builder authenticated(HttpRequest.Builder request) { + return request; + } + + ApplicationId id = ApplicationId.from(requireNonBlankProperty("tenant"), + requireNonBlankProperty("application"), + getNonBlankProperty("instance").orElse("default")); + + URI endpoint = URI.create(requireNonBlankProperty("endpoint")); + Path privateKeyFile = Paths.get(requireNonBlankProperty("privateKeyFile")); + Optional<Path> certificateFile = getNonBlankProperty("certificateFile").map(Paths::get); + + ControllerHttpClient controller = certificateFile.isPresent() + ? ControllerHttpClient.withKeyAndCertificate(endpoint, privateKeyFile, certificateFile.get()) + : ControllerHttpClient.withSignatureKey(endpoint, privateKeyFile, id); + + static Optional<String> getNonBlankProperty(String name) { + return Optional.ofNullable(System.getProperty(name)).filter(value -> ! value.isBlank()); + } + + static String requireNonBlankProperty(String name) { + return getNonBlankProperty(name).orElseThrow(() -> new IllegalStateException("Missing required property '" + name + "'")); + } + +} diff --git a/tenant-auth/src/test/java/ai/vespa/hosted/auth/AuthenticatorTest.java b/tenant-auth/src/test/java/ai/vespa/hosted/auth/AuthenticatorTest.java new file mode 100644 index 00000000000..ff4bebce3ff --- /dev/null +++ b/tenant-auth/src/test/java/ai/vespa/hosted/auth/AuthenticatorTest.java @@ -0,0 +1,5 @@ +package ai.vespa.hosted.auth; + +public class AuthenticatorTest { + +} diff --git a/tenant-base/pom.xml b/tenant-base/pom.xml index 8d5fb626789..9c3a28964ed 100644 --- a/tenant-base/pom.xml +++ b/tenant-base/pom.xml @@ -41,7 +41,6 @@ <dependencyManagement> <dependencies> - <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>container-dependency-versions</artifactId> @@ -49,7 +48,6 @@ <type>pom</type> <scope>import</scope> </dependency> - </dependencies> </dependencyManagement> @@ -224,21 +222,9 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-surefire-plugin</artifactId> - <version>${surefire_version}</version> - <configuration> - <groups>com.yahoo.vespa.tenant.cd.SystemTest</groups> - <excludedGroups /> - <reportsDirectory>${env.TEST_DIR}</reportsDirectory> - <redirectTestOutputToFile>false</redirectTestOutputToFile> - <trimStackTrace>false</trimStackTrace> - </configuration> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-surefire-report-plugin</artifactId> - <version>${surefire_version}</version> <configuration> - <reportsDirectory>${env.TEST_DIR}</reportsDirectory> + <groups>ai.vespa.hosted.cd.SystemTest</groups> + <excludedGroups>ai.vespa.hosted.cd.EmptyGroup</excludedGroups> </configuration> </plugin> </plugins> @@ -252,21 +238,9 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-surefire-plugin</artifactId> - <version>${surefire_version}</version> <configuration> - <groups>com.yahoo.vespa.tenant.cd.StagingTest</groups> - <excludedGroups /> - <reportsDirectory>${env.TEST_DIR}</reportsDirectory> - <redirectTestOutputToFile>false</redirectTestOutputToFile> - <trimStackTrace>false</trimStackTrace> - </configuration> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-surefire-report-plugin</artifactId> - <version>${surefire_version}</version> - <configuration> - <reportsDirectory>${env.TEST_DIR}</reportsDirectory> + <groups>ai.vespa.hosted.cd.StagingTest</groups> + <excludedGroups>ai.vespa.hosted.cd.EmptyGroup</excludedGroups> </configuration> </plugin> </plugins> @@ -280,21 +254,9 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-surefire-plugin</artifactId> - <version>${surefire_version}</version> <configuration> - <groups>com.yahoo.vespa.tenant.cd.ProductionTest</groups> - <excludedGroups /> - <reportsDirectory>${env.TEST_DIR}</reportsDirectory> - <redirectTestOutputToFile>false</redirectTestOutputToFile> - <trimStackTrace>false</trimStackTrace> - </configuration> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-surefire-report-plugin</artifactId> - <version>${surefire_version}</version> - <configuration> - <reportsDirectory>${env.TEST_DIR}</reportsDirectory> + <groups>ai.vespa.hosted.cd.ProductionTest</groups> + <excludedGroups>ai.vespa.hosted.cd.EmptyGroup</excludedGroups> </configuration> </plugin> </plugins> @@ -304,8 +266,38 @@ <build> <finalName>${project.artifactId}</finalName> - <plugins> + <pluginManagement> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + <version>${surefire_version}</version> + <configuration> + <reportsDirectory>${env.TEST_DIR}</reportsDirectory> + <redirectTestOutputToFile>false</redirectTestOutputToFile> + <trimStackTrace>false</trimStackTrace> + <systemPropertyVariables> + <application>${application}</application> + <tenant>${tenant}</tenant> + <instance>${instance}</instance> + <endpoint>${endpoint}</endpoint> + <privateKeyFile>${privateKeyFile}</privateKeyFile> + <certificateFile>${certificateFile}</certificateFile> + </systemPropertyVariables> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-report-plugin</artifactId> + <version>${surefire_version}</version> + <configuration> + <reportsDirectory>${env.TEST_DIR}</reportsDirectory> + </configuration> + </plugin> + </plugins> + </pluginManagement> + <plugins> <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-enforcer-plugin</artifactId> @@ -369,25 +361,14 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-surefire-plugin</artifactId> - <version>${surefire_version}</version> <configuration> - <reportsDirectory>${env.TEST_DIR}</reportsDirectory> <excludedGroups> - com.yahoo.vespa.tenant.cd.SystemTest, - com.yahoo.vespa.tenant.cd.StagingTest, - com.yahoo.vespa.tenant.cd.ProductionTest + ai.vespa.hosted.cd.SystemTest, + ai.vespa.hosted.cd.StagingTest, + ai.vespa.hosted.cd.ProductionTest </excludedGroups> </configuration> </plugin> - - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-surefire-report-plugin</artifactId> - <version>${surefire_version}</version> - <configuration> - <reportsDirectory>${env.TEST_DIR}</reportsDirectory> - </configuration> - </plugin> </plugins> </build> </project> diff --git a/tenant-cd/pom.xml b/tenant-cd/pom.xml index 8907e56762c..7cc2c9a2d5b 100644 --- a/tenant-cd/pom.xml +++ b/tenant-cd/pom.xml @@ -5,6 +5,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> <modelVersion>4.0.0</modelVersion> + <groupId>ai.vespa.hosted</groupId> <artifactId>tenant-cd</artifactId> <name>Hosted Vespa tenant CD</name> <description>Test library for hosted Vespa applications.</description> @@ -20,6 +21,36 @@ <dependencies> <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>security-utils</artifactId> + <version>${project.version}</version> + </dependency> + + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespajlib</artifactId> + <version>${project.version}</version> + </dependency> + + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>config-provisioning</artifactId> + <version>${project.version}</version> + </dependency> + + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>tenant-auth</artifactId> + <version>${project.version}</version> + </dependency> + + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>hosted-api</artifactId> + <version>${project.version}</version> + </dependency> + + <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> <version>4.12</version> diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/Deployment.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Deployment.java new file mode 100644 index 00000000000..277632b74c7 --- /dev/null +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Deployment.java @@ -0,0 +1,19 @@ +package ai.vespa.hosted.cd; + +/** + * A deployment of a Vespa application, which contains endpoints for document and metrics retrieval. + * + * @author jonmv + */ +public interface Deployment { + + /** Returns an Endpoint in the cluster with the "default" id. */ + Endpoint endpoint(); + + /** Returns an Endpoint in the cluster with the given id. */ + Endpoint endpoint(String id); + + /** Returns a {@link TestDeployment} representation of this, or throws if this is a production deployment. */ + TestDeployment asTestDeployment(); + +} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/Digest.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Digest.java new file mode 100644 index 00000000000..dee13fdca13 --- /dev/null +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Digest.java @@ -0,0 +1,28 @@ +package ai.vespa.hosted.cd; + +import java.util.Set; + +/** + * An immutable report of the outcome of a {@link Feed} sent to a {@link TestEndpoint}. + * + * @author jonmv + */ +public class Digest { + + public Set<DocumentId> created() { + return null; + } + + public Set<DocumentId> updated() { + return null; + } + + public Set<DocumentId> deleted() { + return null; + } + + public Set<DocumentId> failed() { + return null; + } + +} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/Document.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Document.java new file mode 100644 index 00000000000..91adeded65c --- /dev/null +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Document.java @@ -0,0 +1,16 @@ +package ai.vespa.hosted.cd; + +/** + * A schema-less representation of a generic Vespa document. + * + * @author jonmv + */ +public class Document { + + + /** Returns a copy of this document, updated with the data in the given document. */ + public Document updatedBy(Document update) { + return null; + } + +} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/DocumentId.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/DocumentId.java new file mode 100644 index 00000000000..9aa8e80c977 --- /dev/null +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/DocumentId.java @@ -0,0 +1,71 @@ +package ai.vespa.hosted.cd; + +import java.util.Arrays; +import java.util.List; + +/** + * Unique, immutable ID of a Vespa document, which contains information pertinent to its storage. + * + * @author jonmv + */ +public class DocumentId { + + private final String namespace; + private final String documentType; + private final String group; + private final Long number; + private final String userDefined; + + private DocumentId(String namespace, String documentType, String group, Long number, String userDefined) { + this.namespace = namespace; + this.documentType = documentType; + this.group = group; + this.number = number; + this.userDefined = userDefined; + } + + public static DocumentId of(String namespace, String documentType, String id) { + return new DocumentId(requireNonEmpty(namespace), requireNonEmpty(documentType), null, null, requireNonEmpty(id)); + } + + public static DocumentId of(String namespace, String documentType, String group, String id) { + return new DocumentId(requireNonEmpty(namespace), requireNonEmpty(documentType), requireNonEmpty(group), null, requireNonEmpty(id)); + } + + public static DocumentId of(String namespace, String documentType, long number, String id) { + return new DocumentId(requireNonEmpty(namespace), requireNonEmpty(documentType), null, number, requireNonEmpty(id)); + } + + public static DocumentId ofValue(String value) { + List<String> parts = Arrays.asList(value.split(":")); + String id = String.join(":", parts.subList(4, parts.size())); + if ( parts.size() < 5 + || ! parts.get(0).equals("id") + || id.isEmpty() + || ! parts.get(3).matches("((n=\\d+)|(g=\\w+))?")) + throw new IllegalArgumentException("Document id must be on the form" + + " 'id:<namespace>:<document type>:n=<integer>|g=<name>|<empty>:<user defined id>'," + + " but was '" + value + "'."); + + if (parts.get(3).matches("n=\\d+")) + return of(parts.get(1), parts.get(2), Long.parseLong(parts.get(3).substring(2)), id); + if (parts.get(3).matches("g=\\w+")) + return of(parts.get(1), parts.get(2), parts.get(3).substring(2), id); + return of(parts.get(1), parts.get(2), id); + } + + public String asValue() { + return "id:" + namespace + ":" + documentType + ":" + grouper() + ":" + userDefined; + } + + private String grouper() { + return group != null ? group : number != null ? number.toString() : ""; + } + + private static String requireNonEmpty(String string) { + if (string.isEmpty()) + throw new IllegalArgumentException("The empty string is not allowed."); + return string; + } + +} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/EmptyGroup.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/EmptyGroup.java new file mode 100644 index 00000000000..8deca3cfb11 --- /dev/null +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/EmptyGroup.java @@ -0,0 +1,9 @@ +package ai.vespa.hosted.cd; + +/** + * The Surefire configuration element <excludedGroups> requires a non-empty argument to reset another. + * This class serves that purpose. Without it, no tests run in the various integration test profiles. + * + * @author jonmv + */ +public interface EmptyGroup { } diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/Endpoint.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Endpoint.java new file mode 100644 index 00000000000..348fc329682 --- /dev/null +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Endpoint.java @@ -0,0 +1,21 @@ +package ai.vespa.hosted.cd; + +import ai.vespa.hosted.cd.metric.Metrics; + +/** + * An endpoint in a Vespa application {@link Deployment}, which allows document and metrics retrieval. + * + * The endpoint translates {@link Query}s to {@link Search}s, and {@link Selection}s to {@link Visit}s. + * It also supplies {@link Metrics}. + * + * @author jonmv + */ +public interface Endpoint { + + Search search(Query query); + + Visit visit(Selection selection); + + Metrics metrics(); + +} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/Feed.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Feed.java new file mode 100644 index 00000000000..e9a0a0aeff0 --- /dev/null +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Feed.java @@ -0,0 +1,25 @@ +package ai.vespa.hosted.cd; + +import java.util.Map; +import java.util.Set; + +/** + * An immutable set of document feed / update / delete operations, which can be sent to a Vespa {@link TestEndpoint}. + * + * @author jonmv + */ +public class Feed { + + Map<DocumentId, Document> creations() { + return null; + } + + Map<DocumentId, Document> updates() { + return null; + } + + Set<DocumentId> deletions() { + return null; + } + +} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/FunctionalTest.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/FunctionalTest.java new file mode 100644 index 00000000000..e6beb313d28 --- /dev/null +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/FunctionalTest.java @@ -0,0 +1,31 @@ +package ai.vespa.hosted.cd; + +/** + * Tests that compare the behaviour of a Vespa application deployment against a fixed specification. + * + * These tests are run whenever a change is pushed to a Vespa application, and whenever the Vespa platform + * is upgraded, and before any deployments to production zones. When these tests fails, the tested change to + * the Vespa application is not rolled out. + * + * A typical functional test is to feed some documents, optionally verifying that the documents have been processed + * as expected, and then to see that queries give the expected results. Another common use is to verify integration + * with external services. + * + * @author jonmv + */ +public interface FunctionalTest { + + // Want to feed some documents. + // Want to verify document processing and routing is as expected. + // Want to check recall on those documents. + // Want to verify queries give expected documents. + // Want to verify searchers. + // Want to verify updates. + // Want to verify deletion. + // May want to verify reprocessing. + // Must likely delete documents between tests. + // Must be able to feed documents, setting route. + // Must be able to search. + // Must be able to visit. + +} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/ProductionTest.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/ProductionTest.java index a756b665c1a..6cf5fb07f58 100644 --- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/ProductionTest.java +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/ProductionTest.java @@ -1,6 +1,23 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.hosted.cd; -public class ProductionTest { +/** + * Tests that verify the health of production deployments of Vespa applications. + * + * These tests are typically run some time after deployment to a production zone, to ensure + * the deployment is still healthy and working as expected. When these tests fail, deployment + * of the tested change is halted until it succeeds, or is superseded by a remedying change. + * + * A typical production test is to verify that a set of metrics, measured by the Vespa + * deployment itself, are within specified parameters, or that some higher-level measure + * of quality, such as engagement among end users of the application, is as expected. + * + * @author jonmv + */ +public interface ProductionTest { + + // Want to verify metrics (Vespa). + // Want to verify external metrics (YAMAS, other). + // May want to verify search gives expected results. } diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/Query.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Query.java new file mode 100644 index 00000000000..d421dc14322 --- /dev/null +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Query.java @@ -0,0 +1,60 @@ +package ai.vespa.hosted.cd; + +import java.util.Map; +import java.util.stream.Stream; + +import static java.util.Map.copyOf; +import static java.util.stream.Collectors.joining; +import static java.util.stream.Collectors.toUnmodifiableMap; + +/** + * An immutable query to send to a Vespa {@link Endpoint}, to receive a {@link Search}. + * + * @author jonmv + */ +public class Query { + + private final String rawQuery; + private final Map<String, String> parameters; + + private Query(String rawQuery, Map<String, String> parameters) { + this.rawQuery = rawQuery; + this.parameters = parameters; + } + + /** Creates a query with the given raw query part. */ + public static Query ofRaw(String rawQuery) { + if (rawQuery.isBlank()) + throw new IllegalArgumentException("Query can not be blank."); + + return new Query(rawQuery, + Stream.of(rawQuery.split("&")) + .map(pair -> pair.split("=")) + .collect(toUnmodifiableMap(pair -> pair[0], pair -> pair[1]))); + } + + /** Creates a query with the given name-value pairs. */ + public static Query ofParameters(Map<String, String> parameters) { + if (parameters.isEmpty()) + throw new IllegalArgumentException("Parameters can not be empty."); + + return new Query(parameters.entrySet().stream() + .map(entry -> entry.getKey() + "=" + entry.getValue()) + .collect(joining("&")), + copyOf(parameters)); + } + + /** Returns a copy of this with the given name-value pair added, potentially overriding any current value. */ + public Query withParameter(String name, String value) { + return ofParameters(Stream.concat(parameters.entrySet().stream().filter(entry -> ! entry.getKey().equals(name)), + Stream.of(Map.entry(name, value))) + .collect(toUnmodifiableMap(Map.Entry::getKey, Map.Entry::getValue))); + } + + /** Returns the raw string representation of this query. */ + public String rawQuery() { return rawQuery; } + + /** Returns the parameters of this query. */ + public Map<String, String> parameters() { return parameters; } + +} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/Search.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Search.java new file mode 100644 index 00000000000..a6c1d188591 --- /dev/null +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Search.java @@ -0,0 +1,24 @@ +package ai.vespa.hosted.cd; + +import java.util.Map; + +/** + * The immutable result of sending a {@link Query} to a Vespa {@link Endpoint}. + * + * @author jonmv + */ +public class Search { + + // hits + // coverage + // searched + // full? + // results? + // resultsFull? + + /** Returns the documents that were returned as the result, with iteration order as returned. */ + Map<DocumentId, Document> documents() { + return null; + } + +} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/Selection.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Selection.java new file mode 100644 index 00000000000..158ae279cb6 --- /dev/null +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Selection.java @@ -0,0 +1,58 @@ +package ai.vespa.hosted.cd; + +/** + * A document selection expression, type and cluster, which can be used to visit an {@link Endpoint}. + * + * @author jonmv + */ +public class Selection { + + private final String selection; + private final String namespace; + private final String type; + private final String group; + private final String cluster; + private final int concurrency; + + private Selection(String selection, String namespace, String type, String group, String cluster, int concurrency) { + this.selection = selection; + this.namespace = namespace; + this.type = type; + this.group = group; + this.cluster = cluster; + this.concurrency = concurrency; + } + + /** Returns a new selection which will visit documents in the given cluster. */ + public static Selection in(String cluster) { + if (cluster.isBlank()) throw new IllegalArgumentException("Cluster name can not be blank."); + return new Selection(null, null, null, cluster, null, 1); + } + + /** Returns a new selection which will visit documents in the given namespace and of the given type. */ + public static Selection of(String namespace, String type) { + if (namespace.isBlank()) throw new IllegalArgumentException("Namespace can not be blank."); + if (type.isBlank()) throw new IllegalArgumentException("Document type can not be blank."); + return new Selection(null, namespace, type, null, null, 1); + } + + /** Returns a copy of this with the given selection criterion set. */ + public Selection matching(String selection) { + if (selection.isBlank()) throw new IllegalArgumentException("Selection can not be blank."); + return new Selection(selection, namespace, type, cluster, group, concurrency); + } + + /** Returns a copy of this selection, with the group set to the specified value. Requires namespace and type to be set. */ + public Selection limitedTo(String group) { + if (namespace == null || type == null) throw new IllegalArgumentException("Namespace and type must be specified to set group."); + if (group.isBlank()) throw new IllegalArgumentException("Group name can not be blank."); + return new Selection(selection, namespace, type, cluster, group, concurrency); + } + + /** Returns a copy of this, with concurrency set to the given positive value. */ + public Selection concurrently(int concurrency) { + if (concurrency < 1) throw new IllegalArgumentException("Concurrency must be a positive integer."); + return new Selection(selection, namespace, type, cluster, group, concurrency); + } + +} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/StagingTest.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/StagingTest.java index 789b9deadb0..ee2ee0add4c 100644 --- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/StagingTest.java +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/StagingTest.java @@ -1,6 +1,10 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.hosted.cd; +/** + * @deprecated Use {@link UpgradeTest}. + */ +@Deprecated public class StagingTest { } diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/SystemTest.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/SystemTest.java index 889acb8b9c4..6a8d1b4cbe4 100644 --- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/SystemTest.java +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/SystemTest.java @@ -1,6 +1,10 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.hosted.cd; +/** + * @deprecated use {@link FunctionalTest}. + */ +@Deprecated public class SystemTest { } diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/TestConfig.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/TestConfig.java new file mode 100644 index 00000000000..36c14a38b37 --- /dev/null +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/TestConfig.java @@ -0,0 +1,101 @@ +package ai.vespa.hosted.cd; + +import ai.vespa.hosted.api.ControllerHttpClient; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.SystemName; +import com.yahoo.slime.ArrayTraverser; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.JsonDecoder; +import com.yahoo.slime.ObjectTraverser; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.slime.Slime; + +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** + * The place to obtain environment-dependent configuration for the current test run. + * + * If the system property 'vespa.test.config' is set, this class attempts to parse config + * from a JSON file at that location -- otherwise, attempts to access the config will return null. + * + * @author jvenstad + */ +public class TestConfig { + + private static TestConfig theConfig; + + private final ApplicationId application; + private final ZoneId zone; + private final SystemName system; + private final Map<ZoneId, Deployment> deployments; + + private TestConfig(ApplicationId application, ZoneId zone, SystemName system, Map<ZoneId, Deployment> deployments) { + this.application = application; + this.zone = zone; + this.system = system; + this.deployments = Map.copyOf(deployments); + } + + /** Returns the config for this test, or null if it has not been provided. */ + public static synchronized TestConfig get() { + if (theConfig == null) { + String configPath = System.getProperty("vespa.test.config"); + theConfig = configPath != null ? fromFile(configPath) : fromController(); + } + return theConfig; + } + + /** Returns the full id of the application to be tested. */ + public ApplicationId application() { return application; } + + /** Returns the zone of the deployment to test. */ + public ZoneId zone() { return zone; } + + /** Returns an immutable view of all configured endpoints for each zone of the application to test. */ + public Map<ZoneId, Deployment> allDeployments() { return deployments; } + + /** Returns the deployment to test in this test runtime. */ + public Deployment deploymentToTest() { return deployments.get(zone); } + + /** Returns the system this is run against. */ + public SystemName system() { return system; } + + static TestConfig fromFile(String path) { + if (path == null) + return null; + + try { + return fromJson(Files.readAllBytes(Paths.get(path))); + } + catch (Exception e) { + throw new IllegalArgumentException("Failed reading config from '" + path + "'!", e); + } + } + + static TestConfig fromController() { + return null; + } + + static TestConfig fromJson(byte[] jsonBytes) { + Inspector config = new JsonDecoder().decode(new Slime(), jsonBytes).get(); + ApplicationId application = ApplicationId.fromSerializedForm(config.field("application").asString()); + ZoneId zone = ZoneId.from(config.field("zone").asString()); + SystemName system = SystemName.from(config.field("system").asString()); + Map<ZoneId, Deployment> endpoints = new HashMap<>(); + config.field("endpoints").traverse((ObjectTraverser) (zoneId, endpointArray) -> { + List<URI> uris = new ArrayList<>(); + endpointArray.traverse((ArrayTraverser) (__, uri) -> uris.add(URI.create(uri.asString()))); + endpoints.put(ZoneId.from(zoneId), null); // TODO jvenstad + }); + return new TestConfig(application, zone, system, endpoints); + } + +} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/TestDeployment.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/TestDeployment.java new file mode 100644 index 00000000000..3360c12e374 --- /dev/null +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/TestDeployment.java @@ -0,0 +1,14 @@ +package ai.vespa.hosted.cd; + +/** + * A deployment of a Vespa application, which also contains endpoints for document manipulation. + * + * @author jonmv + */ +public interface TestDeployment extends Deployment { + + TestEndpoint endpoint(); + + TestEndpoint endpoint(String id); + +} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/TestEndpoint.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/TestEndpoint.java new file mode 100644 index 00000000000..f6f8a722f19 --- /dev/null +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/TestEndpoint.java @@ -0,0 +1,13 @@ +package ai.vespa.hosted.cd; + +/** + * An endpoint in a Vespa application {@link TestDeployment}, which also translates {@link Feed}s to {@link Digest}s. + * + * @author jonmv + */ +public interface TestEndpoint extends Endpoint { + + /** Sends the given Feed to this TestEndpoint, blocking until it is digested, and returns a feed report. */ + Digest digest(Feed feed); + +} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/UpgradeTest.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/UpgradeTest.java new file mode 100644 index 00000000000..32083fbd5f6 --- /dev/null +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/UpgradeTest.java @@ -0,0 +1,23 @@ +package ai.vespa.hosted.cd; + +/** + * Tests that assert continuity of behaviour for Vespa application deployments, through upgrades. + * + * These tests are run whenever a change is pushed to a Vespa application, and whenever the Vespa platform + * is upgraded, and before any deployments to production zones. When these tests fails, the tested change to + * the Vespa application is not rolled out. + * + * A typical upgrade test is to do some operations against a test deployment prior to upgrade, like feed and + * search for some documents, perhaps recording some metrics from the deployment, and then to upgrade it, + * repeat the exercise, and compare the results from pre and post upgrade. + * + * TODO Split in platform upgrades and application upgrades? + * + * @author jonmv + */ +public interface UpgradeTest { + + // Want to verify documents are not damaged by upgrade. + // May want to verify metrics during upgrade. + +} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/Visit.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Visit.java new file mode 100644 index 00000000000..3bb2f59de97 --- /dev/null +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Visit.java @@ -0,0 +1,17 @@ +package ai.vespa.hosted.cd; + +import java.util.Map; + +/** + * A stateful visit operation against a {@link Endpoint}. + * + * @author jonmv + */ +public class Visit { + + // Delegate to a blocking iterator, which can be used for iteration as visit is ongoing. + public Map<DocumentId, Document> documents() { + return null; + } + +} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/VisitEndpoint.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/VisitEndpoint.java new file mode 100644 index 00000000000..618a004a571 --- /dev/null +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/VisitEndpoint.java @@ -0,0 +1,10 @@ +package ai.vespa.hosted.cd; + +/** + * A remote endpoint in a Vespa application {@link Deployment}, which translates {@link Selection}s to {@link Visit}s. + * + * @author jonmv + */ +public interface VisitEndpoint { + +} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/http/HttpEndpoint.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/http/HttpEndpoint.java new file mode 100644 index 00000000000..e0d3787a21c --- /dev/null +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/http/HttpEndpoint.java @@ -0,0 +1,85 @@ +package ai.vespa.hosted.cd.http; + +import ai.vespa.hosted.auth.Authenticator; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.JsonDecoder; +import com.yahoo.slime.Slime; +import ai.vespa.hosted.cd.Digest; +import ai.vespa.hosted.cd.Feed; +import ai.vespa.hosted.cd.Query; +import ai.vespa.hosted.cd.Search; +import ai.vespa.hosted.cd.Selection; +import ai.vespa.hosted.cd.TestEndpoint; +import ai.vespa.hosted.cd.Visit; +import ai.vespa.hosted.cd.metric.Metrics; + +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.time.Duration; + +import static java.util.Objects.requireNonNull; + +public class HttpEndpoint implements TestEndpoint { + + static final String metricsPath = "/state/v1/metrics"; + static final String documentApiPath = "/document/v1"; + static final String searchApiPath = "/search"; + + private final URI endpoint; + private final HttpClient client; + private final Authenticator authenticator; + + public HttpEndpoint(URI endpoint) { + this.endpoint = requireNonNull(endpoint); + this.authenticator = new Authenticator(); + this.client = HttpClient.newBuilder() + .sslContext(authenticator.sslContext()) + .connectTimeout(Duration.ofSeconds(5)) + .version(HttpClient.Version.HTTP_1_1) + .build(); + } + + @Override + public Digest digest(Feed feed) { + return null; + } + + @Override + public Search search(Query query) { + try { + URI target = endpoint.resolve(searchApiPath).resolve("?" + query.rawQuery()); + HttpRequest request = HttpRequest.newBuilder() + .timeout(Duration.ofSeconds(5)) + .uri(target) + .build(); + HttpResponse<byte[]> response = client.send(request, HttpResponse.BodyHandlers.ofByteArray()); + if (response.statusCode() / 100 != 2) // TODO consider allowing 504 if specified. + throw new RuntimeException("Non-OK status code " + response.statusCode() + " at " + target + + ", with response \n" + new String(response.body())); + + return toSearch(response.body()); + } + catch (Exception e) { + throw new RuntimeException(e); + } + } + + static Search toSearch(byte[] body) { + Inspector rootObject = new JsonDecoder().decode(new Slime(), body).get(); + // TODO jvenstad + return new Search(); + } + + @Override + public Visit visit(Selection selection) { + return null; + } + + @Override + public Metrics metrics() { + return null; + } + +} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Metric.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Metric.java new file mode 100644 index 00000000000..cb3c8e77a9a --- /dev/null +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Metric.java @@ -0,0 +1,87 @@ +package ai.vespa.hosted.cd.metric; + +import java.util.HashMap; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.Set; +import java.util.StringJoiner; + +import static java.util.Map.copyOf; +import static java.util.stream.Collectors.toUnmodifiableMap; + +/** + * A set of statistics for a metric, for points over a space with named dimensions of arbitrary type. + * + * @author jonmv + */ +public class Metric { + + private final Map<Map<String, ?>, Statistic> statistics; + + private Metric(Map<Map<String, ?>, Statistic> statistics) { + this.statistics = statistics; + } + + /** Creates a new Metric with a copy of the given data. */ + public static Metric of(Map<Map<String, ?>, Statistic> data) { + if (data.isEmpty()) + throw new IllegalArgumentException("No data given."); + + Map<Map<String, ?>, Statistic> copies = new HashMap<>(); + Set<String> dimensions = data.keySet().iterator().next().keySet(); + data.forEach((point, statistic) -> { + if ( ! point.keySet().equals(dimensions)) + throw new IllegalArgumentException("Given data has inconsistent dimensions: '" + dimensions + "' vs '" + point.keySet() + "'."); + + copies.put(copyOf(point), statistic); + }); + + return new Metric(copyOf(copies)); + } + + /** Returns a Metric view of the subset of points in the given hyperplane; its dimensions must be a subset of those of this Metric. */ + public Metric at(Map<String, ?> hyperplane) { + return new Metric(statistics.keySet().stream() + .filter(point -> point.entrySet().containsAll(hyperplane.entrySet())) + .collect(toUnmodifiableMap(point -> point, statistics::get))); + } + + /** Returns a version of this where statistics along the given hyperspace are aggregated. This does not preserve last, 95 and 99 percentile values. */ + public Metric collapse(Set<String> hyperspace) { + return new Metric(statistics.keySet().stream() + .collect(toUnmodifiableMap(point -> point.keySet().stream() + .filter(dimension -> ! hyperspace.contains(dimension)) + .collect(toUnmodifiableMap(dimension -> dimension, point::get)), + statistics::get, + Statistic::mergedWith))); + } + + /** Returns a collapsed version of this, with all statistics aggregated. This does not preserve last, 95 and 99 percentile values. */ + public Metric collapse() { + return collapse(statistics.keySet().iterator().next().keySet()); + } + + /** If this Metric contains a single point, returns the Statistic of that point; otherwise, throws an exception. */ + public Statistic statistic() { + if (statistics.size() == 1) + return statistics.values().iterator().next(); + + if (statistics.isEmpty()) + throw new NoSuchElementException("This Metric has no data."); + + throw new IllegalStateException("This Metric has more than one point of data."); + } + + /** Returns the underlying, unmodifiable Map. */ + public Map<Map<String, ?>, Statistic> asMap() { + return statistics; + } + + @Override + public String toString() { + return new StringJoiner(", ", Metric.class.getSimpleName() + "[", "]") + .add("statistics=" + statistics) + .toString(); + } + +} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Metrics.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Metrics.java new file mode 100644 index 00000000000..3aa5a126745 --- /dev/null +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Metrics.java @@ -0,0 +1,73 @@ +package ai.vespa.hosted.cd.metric; + +import ai.vespa.hosted.cd.Endpoint; + +import java.time.Instant; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.StringJoiner; + +import static java.util.Map.copyOf; + +/** + * Metrics from a Vespa application {@link Endpoint}, indexed by their names, and optionally by a set of custom dimensions. + * + * Metrics are collected from the <a href="https://docs.vespa.ai/documentation/reference/metrics-health-format.html">metrics</a> + * API of a Vespa endpoint, and contain the current health status of the endpoint, values for all configured metrics in + * that endpoint, and the time interval from which these metrics were sampled. + * + * Each metric is indexed by a name, and, optionally, along a custom set of dimensions, given by a {@code Map<String, String>}. + * + * @author jonmv + */ +public class Metrics { + + private final Instant start, end; + private final Map<String, Metric> metrics; + + private Metrics(Instant start, Instant end, Map<String, Metric> metrics) { + this.start = start; + this.end = end; + this.metrics = metrics; + } + + public static Metrics of(Instant start, Instant end, Map<String, Metric> metrics) { + if ( ! start.isBefore(end)) + throw new IllegalArgumentException("Given time interval must be positive: '" + start + "' to '" + end + "'."); + + return new Metrics(start, end, copyOf(metrics)); + } + + /** Returns the start of the time window from which these metrics were sampled, or throws if the status is {@code Status.down}. */ + public Instant start() { + return start; + } + + /** Returns the end of the time window from which these metrics were sampled, or throws if the status is {@code Status.down}. */ + public Instant end() { + return end; + } + + /** Returns the metric with the given name, or throws a NoSuchElementException if no such Metric is known. */ + public Metric get(String name) { + if ( ! metrics.containsKey(name)) + throw new NoSuchElementException("No metric with name '" + name + "'."); + + return metrics.get(name); + } + + /** Returns the underlying, unmodifiable Map. */ + public Map<String, Metric> asMap() { + return metrics; + } + + @Override + public String toString() { + return new StringJoiner(", ", Metrics.class.getSimpleName() + "[", "]") + .add("start=" + start) + .add("end=" + end) + .add("metrics=" + metrics) + .toString(); + } + +} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Space.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Space.java new file mode 100644 index 00000000000..ea771ca5dd9 --- /dev/null +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Space.java @@ -0,0 +1,44 @@ +package ai.vespa.hosted.cd.metric; + +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.IntStream; + +import static java.util.stream.Collectors.toUnmodifiableMap; + +/** + * Used to easily generate points for a pre-defined space. + * + * @author jonmv + */ +public class Space { + + private final List<String> dimensions; + + private Space(List<String> dimensions) { + this.dimensions = dimensions; + } + + /** Creates a new space with the given named dimensions, in order. */ + public static Space of(List<String> dimensions) { + if (Set.copyOf(dimensions).size() != dimensions.size()) + throw new IllegalArgumentException("Duplicated dimension names in '" + dimensions + "'."); + + return new Space(List.copyOf(dimensions)); + } + + /** Returns a point in this space, with the given values along each dimensions, in order. */ + public Map<String, ?> at(List<?> values) { + if (dimensions.size() != values.size()) + throw new IllegalArgumentException("This space has " + dimensions.size() + " dimensions, but " + values.size() + " were given."); + + return IntStream.range(0, dimensions.size()).boxed().collect(toUnmodifiableMap(dimensions::get, values::get)); + } + + /** Returns a point in this space, with the given values along each dimensions, in order. */ + public Map<String, ?> at(Object... values) { + return at(List.of(values)); + } + +} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Statistic.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Statistic.java new file mode 100644 index 00000000000..fc52900bdac --- /dev/null +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Statistic.java @@ -0,0 +1,68 @@ +package ai.vespa.hosted.cd.metric; + +import java.util.HashMap; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.StringJoiner; + +import static java.util.Map.copyOf; + +/** + * Known statistic about a metric, at a certain point. + * + * @author jonmv + */ +public class Statistic { + + private final Map<Type, Double> data; + + /** Creates a new Statistic with a copy of the given data. */ + private Statistic(Map<Type, Double> data) { + this.data = data; + } + + public static Statistic of(Map<Type, Double> data) { + return new Statistic(copyOf(data)); + } + + /** Returns the value of the given type, or throws a NoSuchElementException if this isn't known. */ + public double get(Type key) { + if ( ! data.containsKey(key)) + throw new NoSuchElementException("No value with key '" + key + "' is known."); + + return data.get(key); + } + + /** Returns the underlying, unmodifiable Map. */ + public Map<Type, Double> asMap() { + return data; + } + + Statistic mergedWith(Statistic other) { + if (data.keySet().equals(other.data.keySet())) + throw new IllegalArgumentException("Incompatible key sets '" + data.keySet() + "' and '" + other.data.keySet() + "'."); + + Map<Type, Double> merged = new HashMap<>(); + double n1 = get(Type.count), n2 = other.get(Type.count); + for (Type type : data.keySet()) switch (type) { + case count: merged.put(type, n1 + n2); break; + case rate: merged.put(type, get(Type.rate) + other.get(Type.rate)); break; + case max: merged.put(type, Math.max(get(Type.max), other.get(Type.max))); break; + case min: merged.put(type, Math.min(get(Type.min), other.get(Type.min))); break; + case average: merged.put(type, (n1 * get(Type.average) + n2 * other.get(Type.average)) / (n1 + n2)); break; + case last: + case percentile95: + case percentile99: break; + default: throw new IllegalArgumentException("Unexpected type '" + type + "'."); + } + return of(merged); + } + + @Override + public String toString() { + return new StringJoiner(", ", Statistic.class.getSimpleName() + "[", "]") + .add("data=" + data) + .toString(); + } + +} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Type.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Type.java new file mode 100644 index 00000000000..d48b4566f6d --- /dev/null +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Type.java @@ -0,0 +1,32 @@ +package ai.vespa.hosted.cd.metric; + +/** + * Known statistic types. + */ +public enum Type { + + /** 95th percentile measurement. */ + percentile95, + + /** 99th percentile measurement. */ + percentile99, + + /** Average over all measurements. */ + average, + + /** Number of measurements. */ + count, + + /** Last measurement. */ + last, + + /** Maximum measurement. */ + max, + + /** Minimum measurement. */ + min, + + /** Number of measurements per second. */ + rate; + +} diff --git a/vbench/src/vbench/http/benchmark_headers.h b/vbench/src/vbench/http/benchmark_headers.h index 92c4a05271b..04b29813760 100644 --- a/vbench/src/vbench/http/benchmark_headers.h +++ b/vbench/src/vbench/http/benchmark_headers.h @@ -5,6 +5,7 @@ #include <vbench/core/string.h> #include <vespa/vespalib/locale/c.h> +#include <cerrno> namespace vbench { diff --git a/vdslib/src/vespa/vdslib/container/parameters.cpp b/vdslib/src/vespa/vdslib/container/parameters.cpp index 0803c346b0a..c830f89d5a9 100644 --- a/vdslib/src/vespa/vdslib/container/parameters.cpp +++ b/vdslib/src/vespa/vdslib/container/parameters.cpp @@ -5,6 +5,7 @@ #include <vespa/vespalib/objects/hexdump.h> #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/vespalib/util/xmlstream.h> +#include <ostream> using namespace vdslib; diff --git a/vespajlib/abi-spec.json b/vespajlib/abi-spec.json index 04e68e60178..b2b895040bc 100644 --- a/vespajlib/abi-spec.json +++ b/vespajlib/abi-spec.json @@ -1352,7 +1352,8 @@ "public static com.yahoo.tensor.TensorType$Value[] values()", "public static com.yahoo.tensor.TensorType$Value valueOf(java.lang.String)", "public static com.yahoo.tensor.TensorType$Value largestOf(java.util.List)", - "public static com.yahoo.tensor.TensorType$Value largestOf(com.yahoo.tensor.TensorType$Value, com.yahoo.tensor.TensorType$Value)" + "public static com.yahoo.tensor.TensorType$Value largestOf(com.yahoo.tensor.TensorType$Value, com.yahoo.tensor.TensorType$Value)", + "public java.lang.String toString()" ], "fields": [ "public static final enum com.yahoo.tensor.TensorType$Value DOUBLE", diff --git a/vespajlib/src/main/java/com/yahoo/tensor/IndexedDoubleTensor.java b/vespajlib/src/main/java/com/yahoo/tensor/IndexedDoubleTensor.java index 7f1351cc42b..219a3fa2278 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/IndexedDoubleTensor.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/IndexedDoubleTensor.java @@ -108,7 +108,13 @@ class IndexedDoubleTensor extends IndexedTensor { @Override public void cellByDirectIndex(long index, double value) { - values[(int)index] = value; + try { + values[(int) index] = value; + } + catch (IndexOutOfBoundsException e) { + throw new IllegalArgumentException("Can not set the cell at position " + index + " in a tensor " + + "of type " + type + ": Index is too large"); + } } } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java b/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java index aeb3da8ac40..aca2bfc1b0f 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java @@ -16,7 +16,7 @@ import java.util.Set; import java.util.function.DoubleBinaryOperator; /** - * An indexed (dense) tensor backed by a double array. + * An indexed (dense) tensor backed by an array. * * @author bratseth */ @@ -143,9 +143,8 @@ public abstract class IndexedTensor implements Tensor { long valueIndex = 0; for (int i = 0; i < indexes.length; i++) { - if (indexes[i] >= sizes.size(i)) { - throw new IllegalArgumentException(indexes + " are not within bounds"); - } + if (indexes[i] >= sizes.size(i)) + throw new IllegalArgumentException(Arrays.toString(indexes) + " are not within bounds"); valueIndex += productOfDimensionsAfter(i, sizes) * indexes[i]; } return valueIndex; diff --git a/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java b/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java index 22ff793e6fa..c2aa155d6bb 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java @@ -333,7 +333,7 @@ public interface Tensor { } else { x = Math.nextAfter(x, y); } - return x==y; + return x == y; } // ----------------- Factories @@ -367,9 +367,7 @@ public interface Tensor { return TensorParser.tensorFrom(tensorString, Optional.empty()); } - /** - * Returns a double as a tensor: A dimensionless tensor containing the value as its cell - */ + /** Returns a double as a tensor: A dimensionless tensor containing the value as its cell */ static Tensor from(double value) { return Tensor.Builder.of(TensorType.empty).cell(value).build(); } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/TensorParser.java b/vespajlib/src/main/java/com/yahoo/tensor/TensorParser.java index 45a9992c9ad..4d8b34b7dcf 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/TensorParser.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/TensorParser.java @@ -8,44 +8,59 @@ import java.util.Optional; */ class TensorParser { - static Tensor tensorFrom(String tensorString, Optional<TensorType> type) { + static Tensor tensorFrom(String tensorString, Optional<TensorType> explicitType) { + Optional<TensorType> type; + String valueString; + tensorString = tensorString.trim(); - try { - if (tensorString.startsWith("tensor")) { - int colonIndex = tensorString.indexOf(':'); - String typeString = tensorString.substring(0, colonIndex); - String valueString = tensorString.substring(colonIndex + 1); - TensorType typeFromString = TensorTypeParser.fromSpec(typeString); - if (type.isPresent() && ! type.get().equals(typeFromString)) - throw new IllegalArgumentException("Got tensor with type string '" + typeString + "', but was " + - "passed type " + type.get()); - return tensorFromValueString(valueString, typeFromString); - } - else if (tensorString.startsWith("{")) { - return tensorFromValueString(tensorString, type.orElse(typeFromValueString(tensorString))); - } - else { - if (type.isPresent() && ! type.get().equals(TensorType.empty)) - throw new IllegalArgumentException("Got zero-dimensional tensor '" + tensorString + - "' where type " + type.get() + " is required"); + if (tensorString.startsWith("tensor")) { + int colonIndex = tensorString.indexOf(':'); + String typeString = tensorString.substring(0, colonIndex); + TensorType typeFromString = TensorTypeParser.fromSpec(typeString); + if (explicitType.isPresent() && ! explicitType.get().equals(typeFromString)) + throw new IllegalArgumentException("Got tensor with type string '" + typeString + "', but was " + + "passed type " + explicitType.get()); + type = Optional.of(typeFromString); + valueString = tensorString.substring(colonIndex + 1); + } + else { + type = explicitType; + valueString = tensorString; + } + + valueString = valueString.trim(); + if (valueString.startsWith("{")) { + return tensorFromSparseValueString(valueString, type); + } + else if (valueString.startsWith("[")) { + return tensorFromDenseValueString(valueString, type); + } + else { + if (explicitType.isPresent() && ! explicitType.get().equals(TensorType.empty)) + throw new IllegalArgumentException("Got a zero-dimensional tensor value ('" + tensorString + + "') where type " + explicitType.get() + " is required"); + try { return Tensor.Builder.of(TensorType.empty).cell(Double.parseDouble(tensorString)).build(); } - } - catch (NumberFormatException e) { - throw new IllegalArgumentException("Excepted a number or a string starting by { or tensor(, got '" + - tensorString + "'"); + catch (NumberFormatException e) { + throw new IllegalArgumentException("Excepted a number or a string starting by {, [ or tensor(...):, got '" + + tensorString + "'"); + } } } - /** Derive the tensor type from the first address string in the given tensor string */ - private static TensorType typeFromValueString(String s) { - s = s.substring(1).trim(); // remove tensor start + /** Derives the tensor type from the first address string in the given tensor string */ + private static TensorType typeFromSparseValueString(String valueString) { + String s = valueString.substring(1).trim(); // remove tensor start int firstKeyOrTensorEnd = s.indexOf('}'); + if (firstKeyOrTensorEnd < 0) + throw new IllegalArgumentException("Excepted a number or a string starting by {, [ or tensor(...):, got '" + + valueString + "'"); String addressBody = s.substring(0, firstKeyOrTensorEnd).trim(); if (addressBody.isEmpty()) return TensorType.empty; // Empty tensor if ( ! addressBody.startsWith("{")) return TensorType.empty; // Single value tensor - addressBody = addressBody.substring(1); // remove key start + addressBody = addressBody.substring(1, addressBody.length()); // remove key start if (addressBody.isEmpty()) return TensorType.empty; // Empty key TensorType.Builder builder = new TensorType.Builder(TensorType.Value.DOUBLE); @@ -60,21 +75,76 @@ class TensorParser { return builder.build(); } - private static Tensor tensorFromValueString(String tensorValueString, TensorType type) { - Tensor.Builder builder = Tensor.Builder.of(type); - tensorValueString = tensorValueString.trim(); + private static Tensor tensorFromSparseValueString(String valueString, Optional<TensorType> type) { try { - if (tensorValueString.startsWith("{")) - return fromCellString(builder, tensorValueString); - else - return builder.cell(Double.parseDouble(tensorValueString)).build(); + valueString = valueString.trim(); + Tensor.Builder builder = Tensor.Builder.of(type.orElse(typeFromSparseValueString(valueString))); + return fromCellString(builder, valueString); } catch (NumberFormatException e) { throw new IllegalArgumentException("Excepted a number or a string starting by { or tensor(, got '" + - tensorValueString + "'"); + valueString + "'"); } } + private static Tensor tensorFromDenseValueString(String valueString, Optional<TensorType> type) { + if (type.isEmpty()) + throw new IllegalArgumentException("The dense tensor form requires an explicit tensor type " + + "on the form 'tensor(dimensions):..."); + if (type.get().dimensions().stream().anyMatch(d -> ( d.size().isEmpty()))) + throw new IllegalArgumentException("The dense tensor form requires a tensor type containing " + + "only dense dimensions with a given size"); + + IndexedTensor.BoundBuilder builder = (IndexedTensor.BoundBuilder)IndexedTensor.Builder.of(type.get()); + long index = 0; + int currentChar; + int nextNumberEnd = 0; + // Since we know the dimensions the brackets are just syntactic sugar: + while ((currentChar = nextStartCharIndex(nextNumberEnd + 1, valueString)) < valueString.length()) { + nextNumberEnd = nextStopCharIndex(currentChar, valueString); + if (currentChar == nextNumberEnd) return builder.build(); + + TensorType.Value cellValueType = builder.type().valueType(); + String cellValueString = valueString.substring(currentChar, nextNumberEnd); + try { + if (cellValueType == TensorType.Value.DOUBLE) + builder.cellByDirectIndex(index, Double.parseDouble(cellValueString)); + else if (cellValueType == TensorType.Value.FLOAT) + builder.cellByDirectIndex(index, Float.parseFloat(cellValueString)); + else + throw new IllegalArgumentException(cellValueType + " is not supported"); + } + catch (NumberFormatException e) { + throw new IllegalArgumentException("At index " + index + ": '" + + cellValueString + "' is not a valid " + cellValueType); + } + index++; + } + return builder.build(); + } + + /** Returns the position of the next character that should contain a number, or if none the string length */ + private static int nextStartCharIndex(int charIndex, String valueString) { + for (; charIndex < valueString.length(); charIndex++) { + if (valueString.charAt(charIndex) == ']') continue; + if (valueString.charAt(charIndex) == '[') continue; + if (valueString.charAt(charIndex) == ',') continue; + if (valueString.charAt(charIndex) == ' ') continue; + return charIndex; + } + return valueString.length(); + } + + private static int nextStopCharIndex(int charIndex, String valueString) { + while (charIndex < valueString.length()) { + if (valueString.charAt(charIndex) == ',') return charIndex; + if (valueString.charAt(charIndex) == ']') return charIndex; + charIndex++; + } + throw new IllegalArgumentException("Malformed tensor value '" + valueString + + "': Expected a ',' or ']' after position " + charIndex); + } + private static Tensor fromCellString(Tensor.Builder builder, String s) { int index = 1; index = skipSpace(index, s); @@ -97,8 +167,21 @@ class TensorParser { } TensorAddress address = addressBuilder.build(); - Double value = asDouble(address, s.substring(index, valueEnd).trim()); - builder.cell(address, value); + TensorType.Value cellValueType = builder.type().valueType(); + String cellValueString = s.substring(index, valueEnd).trim(); + try { + if (cellValueType == TensorType.Value.DOUBLE) + builder.cell(address, Double.parseDouble(cellValueString)); + else if (cellValueType == TensorType.Value.FLOAT) + builder.cell(address, Float.parseFloat(cellValueString)); + else + throw new IllegalArgumentException(cellValueType + " is not supported"); + } + catch (NumberFormatException e) { + throw new IllegalArgumentException("At " + address.toString(builder.type()) + ": '" + + cellValueString + "' is not a valid " + cellValueType); + } + index = valueEnd+1; index = skipSpace(index, s); } @@ -130,13 +213,4 @@ class TensorParser { } } - private static Double asDouble(TensorAddress address, String s) { - try { - return Double.valueOf(s); - } - catch (NumberFormatException e) { - throw new IllegalArgumentException("At " + address + ": Expected a floating point number, got '" + s + "'"); - } - } - } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/TensorType.java b/vespajlib/src/main/java/com/yahoo/tensor/TensorType.java index b1c7a2341c0..8e566fac0b6 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/TensorType.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/TensorType.java @@ -48,6 +48,9 @@ public class TensorType { return FLOAT; } + @Override + public String toString() { return name().toLowerCase(); } + }; /** The empty tensor type - which is the same as a double */ diff --git a/vespajlib/src/main/java/com/yahoo/tensor/TensorTypeParser.java b/vespajlib/src/main/java/com/yahoo/tensor/TensorTypeParser.java index d5f77be0dd0..1f426942c5f 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/TensorTypeParser.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/TensorTypeParser.java @@ -24,6 +24,7 @@ public class TensorTypeParser { private static final Pattern mappedPattern = Pattern.compile("(\\w+)\\{\\}"); public static TensorType fromSpec(String specString) { + specString = specString.trim(); if ( ! specString.startsWith(START_STRING) || ! specString.endsWith(END_STRING)) throw formatException(specString); String specBody = specString.substring(START_STRING.length(), specString.length() - END_STRING.length()); @@ -112,9 +113,9 @@ public class TensorTypeParser { private static IllegalArgumentException formatException(String spec, Optional<String> errorDetail) { throw new IllegalArgumentException("A tensor type spec must be on the form " + - "tensor[<valuetype>]?(dimensionidentifier[{}|[length?]*), but was '" + spec + "'. " + + "tensor[<valuetype>]?(dimensionidentifier[{}|[length]*), but was '" + spec + "'. " + errorDetail.map(s -> s + ". ").orElse("") + - "Examples: tensor(x[]), tensor<float>(name{}, x[10])"); + "Examples: tensor(x[3]), tensor<float>(name{}, x[10])"); } } diff --git a/vespajlib/src/test/java/com/yahoo/tensor/TensorParserTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/TensorParserTestCase.java index 04ea118280c..63fe40565bd 100644 --- a/vespajlib/src/test/java/com/yahoo/tensor/TensorParserTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/tensor/TensorParserTestCase.java @@ -9,13 +9,58 @@ import static org.junit.Assert.fail; public class TensorParserTestCase { @Test - public void testParsing() { + public void testSparseParsing() { assertEquals(Tensor.Builder.of(TensorType.fromSpec("tensor()")).build(), Tensor.from("{}")); assertEquals(Tensor.Builder.of(TensorType.fromSpec("tensor(x{})")).cell(1.0, 0).build(), Tensor.from("{{x:0}:1.0}")); assertEquals(Tensor.Builder.of(TensorType.fromSpec("tensor(x{})")).cell().label("x", "l0").value(1.0).build(), Tensor.from("{{x:l0}:1.0}")); + assertEquals("If the type is specified, a dense tensor can be created from the sparse text form", + Tensor.Builder.of(TensorType.fromSpec("tensor(x[1])")).cell(1.0, 0).build(), + Tensor.from("tensor(x[1]):{{x:0}:1.0}")); + } + + @Test + public void testDenseParsing() { + assertEquals(Tensor.Builder.of(TensorType.fromSpec("tensor()")).build(), + Tensor.from("tensor():[]")); + assertEquals(Tensor.Builder.of(TensorType.fromSpec("tensor(x[1])")).cell(1.0, 0).build(), + Tensor.from("tensor(x[1]):[1.0]")); + assertEquals(Tensor.Builder.of(TensorType.fromSpec("tensor(x[2])")).cell(1.0, 0).cell(2.0, 1).build(), + Tensor.from("tensor(x[2]):[1.0, 2.0]")); + assertEquals(Tensor.Builder.of(TensorType.fromSpec("tensor(x[2],y[3])")) + .cell(1.0, 0, 0) + .cell(2.0, 0, 1) + .cell(3.0, 0, 2) + .cell(4.0, 1, 0) + .cell(5.0, 1, 1) + .cell(6.0, 1, 2).build(), + Tensor.from("tensor(x[2],y[3]):[[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]]")); + assertEquals(Tensor.Builder.of(TensorType.fromSpec("tensor(x[1],y[2],z[3])")) + .cell(1.0, 0, 0, 0) + .cell(2.0, 0, 0, 1) + .cell(3.0, 0, 0, 2) + .cell(4.0, 0, 1, 0) + .cell(5.0, 0, 1, 1) + .cell(6.0, 0, 1, 2).build(), + Tensor.from("tensor(x[1],y[2],z[3]):[[[1.0], [2.0]], [[3.0], [4.0]], [[5.0], [6.0]]]")); + assertEquals(Tensor.Builder.of(TensorType.fromSpec("tensor(x[3],y[2],z[1])")) + .cell(1.0, 0, 0, 0) + .cell(2.0, 0, 1, 0) + .cell(3.0, 1, 0, 0) + .cell(4.0, 1, 1, 0) + .cell(5.0, 2, 0, 0) + .cell(6.0, 2, 1, 0).build(), + Tensor.from("tensor(x[3],y[2],z[1]):[[[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]]")); + assertEquals(Tensor.Builder.of(TensorType.fromSpec("tensor(x[3],y[2],z[1])")) + .cell( 1.0, 0, 0, 0) + .cell( 2.0, 0, 1, 0) + .cell( 3.0, 1, 0, 0) + .cell( 4.0, 1, 1, 0) + .cell( 5.0, 2, 0, 0) + .cell(-6.0, 2, 1, 0).build(), + Tensor.from("tensor( x[3],y[2],z[1]) : [ [ [1.0, 2.0, 3.0] , [4.0, 5,-6.0] ] ]")); } @Test @@ -26,6 +71,10 @@ public class TensorParserTestCase { "{{'x':\"l0\"}:1.0}"); assertIllegal("dimension must be an identifier or integer, not '\"x\"'", "{{\"x\":\"l0\", \"y\":\"l0\"}:1.0, {\"x\":\"l0\", \"y\":\"l1\"}:2.0}"); + assertIllegal("At {x:0}: '1-.0' is not a valid double", + "{{x:0}:1-.0}"); + assertIllegal("At index 0: '1-.0' is not a valid double", + "tensor(x[1]):[1-.0]"); } private void assertIllegal(String message, String tensor) { diff --git a/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java index b01d171792c..c53db160806 100644 --- a/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java @@ -54,7 +54,7 @@ public class TensorTestCase { fail("Expected parse error"); } catch (IllegalArgumentException expected) { - assertEquals("Excepted a number or a string starting by { or tensor(, got '--'", expected.getMessage()); + assertEquals("Excepted a number or a string starting by {, [ or tensor(...):, got '--'", expected.getMessage()); } } diff --git a/vespalib/src/vespa/vespalib/btree/btree.hpp b/vespalib/src/vespa/vespalib/btree/btree.hpp index 928d8d6cfcd..7bba2e936f3 100644 --- a/vespalib/src/vespa/vespalib/btree/btree.hpp +++ b/vespalib/src/vespa/vespalib/btree/btree.hpp @@ -4,8 +4,7 @@ #include "btree.h" -namespace search { -namespace btree { +namespace search::btree { template <typename KeyT, typename DataT, typename AggrT, typename CompareT, typename TraitsT, class AggrCalcT> @@ -24,7 +23,4 @@ BTree<KeyT, DataT, AggrT, CompareT, TraitsT, AggrCalcT>::~BTree() _alloc.clearHoldLists(); } - -} // namespace search::btree -} // namespace search - +} diff --git a/vespalib/src/vespa/vespalib/btree/btreeinserter.cpp b/vespalib/src/vespa/vespalib/btree/btreeinserter.cpp index f307c474f90..c00d2342eed 100644 --- a/vespalib/src/vespa/vespalib/btree/btreeinserter.cpp +++ b/vespalib/src/vespa/vespalib/btree/btreeinserter.cpp @@ -6,16 +6,11 @@ #include "btreeinserter.hpp" #include "btreenode.hpp" -#include <vespa/log/log.h> -LOG_SETUP(".searchlib.btree.btreeinserter"); - namespace search::btree { template class BTreeInserter<uint32_t, uint32_t, NoAggregated>; template class BTreeInserter<uint32_t, BTreeNoLeafData, NoAggregated>; template class BTreeInserter<uint32_t, int32_t, MinMaxAggregated, - std::less<uint32_t>, - BTreeDefaultTraits, - MinMaxAggrCalc>; + std::less<uint32_t>, BTreeDefaultTraits, MinMaxAggrCalc>; } diff --git a/vespalib/src/vespa/vespalib/btree/btreeinserter.h b/vespalib/src/vespa/vespalib/btree/btreeinserter.h index a3fa2916a88..a66a7bc3f92 100644 --- a/vespalib/src/vespa/vespalib/btree/btreeinserter.h +++ b/vespalib/src/vespa/vespalib/btree/btreeinserter.h @@ -10,11 +10,7 @@ #include "minmaxaggrcalc.h" #include "btreeiterator.h" -namespace search -{ - -namespace btree -{ +namespace search::btree { template <typename KeyT, typename DataT, @@ -32,12 +28,9 @@ public: TraitsT::INTERNAL_SLOTS, TraitsT::LEAF_SLOTS, AggrCalcT> Aggregator; - typedef BTreeIterator<KeyT, DataT, AggrT, - CompareT, TraitsT> Iterator; - typedef BTreeInternalNode<KeyT, AggrT, TraitsT::INTERNAL_SLOTS> - InternalNodeType; - typedef BTreeLeafNode<KeyT, DataT, AggrT, TraitsT::LEAF_SLOTS> - LeafNodeType; + typedef BTreeIterator<KeyT, DataT, AggrT, CompareT, TraitsT> Iterator; + typedef BTreeInternalNode<KeyT, AggrT, TraitsT::INTERNAL_SLOTS> InternalNodeType; + typedef BTreeLeafNode<KeyT, DataT, AggrT, TraitsT::LEAF_SLOTS> LeafNodeType; typedef KeyT KeyType; typedef DataT DataType; typedef typename InternalNodeType::RefPair InternalNodeTypeRefPair; @@ -49,19 +42,12 @@ private: public: static void - insert(BTreeNode::Ref &root, - Iterator &itr, - const KeyType &key, const DataType &data, - const AggrCalcT &aggrCalc); + insert(BTreeNode::Ref &root, Iterator &itr, const KeyType &key, const DataType &data, const AggrCalcT &aggrCalc); }; extern template class BTreeInserter<uint32_t, uint32_t, NoAggregated>; extern template class BTreeInserter<uint32_t, BTreeNoLeafData, NoAggregated>; extern template class BTreeInserter<uint32_t, int32_t, MinMaxAggregated, - std::less<uint32_t>, - BTreeDefaultTraits, - MinMaxAggrCalc>; - -} // namespace search::btree -} // namespace search + std::less<uint32_t>, BTreeDefaultTraits, MinMaxAggrCalc>; +} diff --git a/vespalib/src/vespa/vespalib/btree/btreeinserter.hpp b/vespalib/src/vespa/vespalib/btree/btreeinserter.hpp index d1da94c1b17..b24874088a2 100644 --- a/vespalib/src/vespa/vespalib/btree/btreeinserter.hpp +++ b/vespalib/src/vespa/vespalib/btree/btreeinserter.hpp @@ -7,8 +7,7 @@ #include "btreeiterator.hpp" #include <vespa/vespalib/stllike/asciistream.h> -namespace search { -namespace btree { +namespace search::btree { namespace { @@ -178,7 +177,4 @@ insert(BTreeNode::Ref &root, } } - -} // namespace search::btree -} // namespace search - +} diff --git a/vespalib/src/vespa/vespalib/btree/btreeiterator.hpp b/vespalib/src/vespa/vespalib/btree/btreeiterator.hpp index b26f249c51b..13d41ef61f3 100644 --- a/vespalib/src/vespa/vespalib/btree/btreeiterator.hpp +++ b/vespalib/src/vespa/vespalib/btree/btreeiterator.hpp @@ -9,8 +9,6 @@ namespace search::btree { -#define STRICT_BTREE_ITERATOR_SEEK - template <typename KeyT, typename DataT, typename AggrT, uint32_t INTERNAL_SLOTS, uint32_t LEAF_SLOTS, uint32_t PATH_SIZE> BTreeIteratorBase<KeyT, DataT, AggrT, INTERNAL_SLOTS, LEAF_SLOTS, PATH_SIZE>:: @@ -81,10 +79,7 @@ operator=(const BTreeIteratorBase &other) template <typename KeyT, typename DataT, typename AggrT, uint32_t INTERNAL_SLOTS, uint32_t LEAF_SLOTS, uint32_t PATH_SIZE> -BTreeIteratorBase<KeyT, DataT, AggrT, INTERNAL_SLOTS, LEAF_SLOTS, PATH_SIZE>:: -~BTreeIteratorBase() -{ -} +BTreeIteratorBase<KeyT, DataT, AggrT, INTERNAL_SLOTS, LEAF_SLOTS, PATH_SIZE>::~BTreeIteratorBase() = default; template <typename KeyT, typename DataT, typename AggrT, uint32_t INTERNAL_SLOTS, uint32_t LEAF_SLOTS, uint32_t PATH_SIZE> @@ -674,11 +669,7 @@ BTreeConstIterator<KeyT, DataT, AggrT, CompareT, TraitsT>:: binarySeek(const KeyType & key, CompareT comp) { const LeafNodeType *lnode = _leaf.getNode(); - uint32_t lidx = _leaf.getIdx(); -#ifdef STRICT_BTREE_ITERATOR_SEEK - assert(_leaf.valid() && comp(lnode->getKey(lidx), key)); -#endif - ++lidx; + uint32_t lidx = _leaf.getIdx() + 1; if (lidx < lnode->validSlots()) { if (!comp(lnode->getKey(lidx), key)) { _leaf.setIdx(lidx); @@ -723,11 +714,7 @@ BTreeConstIterator<KeyT, DataT, AggrT, CompareT, TraitsT>:: linearSeek(const KeyType & key, CompareT comp) { const LeafNodeType *lnode = _leaf.getNode(); - uint32_t lidx = _leaf.getIdx(); -#ifdef STRICT_BTREE_ITERATOR_SEEK - assert(_leaf.valid() && comp(lnode->getKey(lidx), key)); -#endif - ++lidx; + uint32_t lidx = _leaf.getIdx() + 1; if (lidx < lnode->validSlots()) { if (!comp(lnode->getKey(lidx), key)) { _leaf.setIdx(lidx); @@ -792,11 +779,7 @@ BTreeConstIterator<KeyT, DataT, AggrT, CompareT, TraitsT>:: binarySeekPast(const KeyType & key, CompareT comp) { const LeafNodeType *lnode = _leaf.getNode(); - uint32_t lidx = _leaf.getIdx(); -#ifdef STRICT_BTREE_ITERATOR_SEEK - assert(_leaf.valid() && !comp(key, lnode->getKey(lidx))); -#endif - ++lidx; + uint32_t lidx = _leaf.getIdx() + 1; if (lidx < lnode->validSlots()) { if (comp(key, lnode->getKey(lidx))) { _leaf.setIdx(lidx); @@ -841,11 +824,8 @@ BTreeConstIterator<KeyT, DataT, AggrT, CompareT, TraitsT>:: linearSeekPast(const KeyType & key, CompareT comp) { const LeafNodeType *lnode = _leaf.getNode(); - uint32_t lidx = _leaf.getIdx(); -#ifdef STRICT_BTREE_ITERATOR_SEEK - assert(_leaf.valid() && !comp(key, lnode->getKey(lidx))); -#endif - ++lidx; + uint32_t lidx = _leaf.getIdx() + 1; + if (lidx < lnode->validSlots()) { if (comp(key, lnode->getKey(lidx))) { _leaf.setIdx(lidx); diff --git a/vespalib/src/vespa/vespalib/btree/btreeremover.cpp b/vespalib/src/vespa/vespalib/btree/btreeremover.cpp index 2322eebf784..f5ada77fed6 100644 --- a/vespalib/src/vespa/vespalib/btree/btreeremover.cpp +++ b/vespalib/src/vespa/vespalib/btree/btreeremover.cpp @@ -11,8 +11,6 @@ namespace search::btree { template class BTreeRemover<uint32_t, uint32_t, NoAggregated>; template class BTreeRemover<uint32_t, BTreeNoLeafData, NoAggregated>; template class BTreeRemover<uint32_t, int32_t, MinMaxAggregated, - std::less<uint32_t>, - BTreeDefaultTraits, - MinMaxAggrCalc>; + std::less<uint32_t>, BTreeDefaultTraits, MinMaxAggrCalc>; } diff --git a/vespalib/src/vespa/vespalib/btree/btreeremover.h b/vespalib/src/vespa/vespalib/btree/btreeremover.h index 87355aa4ce7..bbb825d6299 100644 --- a/vespalib/src/vespa/vespalib/btree/btreeremover.h +++ b/vespalib/src/vespa/vespalib/btree/btreeremover.h @@ -10,11 +10,7 @@ #include "minmaxaggrcalc.h" #include "btreeiterator.h" -namespace search -{ - -namespace btree -{ +namespace search::btree { template <typename KeyT, typename DataT, @@ -82,23 +78,15 @@ public: typedef DataT DataType; typedef typename InternalNodeType::RefPair InternalNodeTypeRefPair; typedef typename LeafNodeType::RefPair LeafNodeTypeRefPair; - typedef BTreeIterator<KeyT, DataT, AggrT, - CompareT, TraitsT> Iterator; + typedef BTreeIterator<KeyT, DataT, AggrT, CompareT, TraitsT> Iterator; static void - remove(BTreeNode::Ref &root, - Iterator &itr, - const AggrCalcT &aggrCalc); + remove(BTreeNode::Ref &root, Iterator &itr, const AggrCalcT &aggrCalc); }; extern template class BTreeRemover<uint32_t, uint32_t, NoAggregated>; extern template class BTreeRemover<uint32_t, BTreeNoLeafData, NoAggregated>; -extern template class BTreeRemover<uint32_t, int32_t, - MinMaxAggregated, - std::less<uint32_t>, - BTreeDefaultTraits, - MinMaxAggrCalc>; - -} // namespace search::btree -} // namespace search +extern template class BTreeRemover<uint32_t, int32_t, MinMaxAggregated, + std::less<uint32_t>, BTreeDefaultTraits, MinMaxAggrCalc>; +} diff --git a/vespalib/src/vespa/vespalib/btree/btreeremover.hpp b/vespalib/src/vespa/vespalib/btree/btreeremover.hpp index c304ea13016..2281fd99f6d 100644 --- a/vespalib/src/vespa/vespalib/btree/btreeremover.hpp +++ b/vespalib/src/vespa/vespalib/btree/btreeremover.hpp @@ -6,11 +6,7 @@ #include "btreerootbase.hpp" #include <vespa/vespalib/stllike/asciistream.h> -namespace search -{ - -namespace btree -{ +namespace search::btree { template <typename KeyT, typename DataT, typename AggrT, size_t INTERNAL_SLOTS, size_t LEAF_SLOTS, class AggrCalcT> @@ -179,7 +175,4 @@ remove(BTreeNode::Ref &root, ++itr; } - -} // namespace search::btree -} // namespace search - +} diff --git a/vespalib/src/vespa/vespalib/data/databuffer.cpp b/vespalib/src/vespa/vespalib/data/databuffer.cpp index 9b04724b601..758922aec6d 100644 --- a/vespalib/src/vespa/vespalib/data/databuffer.cpp +++ b/vespalib/src/vespa/vespalib/data/databuffer.cpp @@ -1,5 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "databuffer.h" +#include <algorithm> namespace vespalib { diff --git a/vespalib/src/vespa/vespalib/gtest/gtest.h b/vespalib/src/vespa/vespalib/gtest/gtest.h index 67d3d438a52..e5bfcf2ae55 100644 --- a/vespalib/src/vespa/vespalib/gtest/gtest.h +++ b/vespalib/src/vespa/vespalib/gtest/gtest.h @@ -1,5 +1,7 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + #include <gtest/gtest.h> /** diff --git a/vespalib/src/vespa/vespalib/util/benchmark_timer.h b/vespalib/src/vespa/vespalib/util/benchmark_timer.h index 8d4147907fc..8e0821d6127 100644 --- a/vespalib/src/vespa/vespalib/util/benchmark_timer.h +++ b/vespalib/src/vespa/vespalib/util/benchmark_timer.h @@ -3,6 +3,7 @@ #include <chrono> #include <functional> +#include <algorithm> namespace vespalib { diff --git a/vespalog/src/test/threads/testthreads.cpp b/vespalog/src/test/threads/testthreads.cpp index 1723f35e432..802514285b6 100644 --- a/vespalog/src/test/threads/testthreads.cpp +++ b/vespalog/src/test/threads/testthreads.cpp @@ -3,6 +3,7 @@ #include <vespa/fastos/time.h> #include <vespa/fastos/thread.h> #include <vespa/log/bufferedlogger.h> +#include <array> #include <iostream> #include <thread> #include <chrono> diff --git a/vespalog/src/vespa/log/log_message.h b/vespalog/src/vespa/log/log_message.h index 832b5f6d47d..ac00e4237ac 100644 --- a/vespalog/src/vespa/log/log_message.h +++ b/vespalog/src/vespa/log/log_message.h @@ -4,6 +4,7 @@ #include "log.h" #include <string_view> +#include <string> namespace ns_log { |