diff options
45 files changed, 256 insertions, 162 deletions
diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java index bc0e16abbe3..b5d091cca8d 100644 --- a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java @@ -154,8 +154,8 @@ public class ConvertParsedFields { var dataType = field.getDataType(); var otherType = summaryField.getType(); if (otherType != null && summaryField.getHasExplicitType()) { - schema.getDeployLogger().log(Level.FINE, () -> "For " + schema.getName() + - ", field '" + field.getName() + + schema.getDeployLogger().log(Level.FINE, () -> "For schema '" + schema.getName() + + "', field '" + field.getName() + "', summary '" + summaryField.name() + "': Specifying the type is deprecated, ignored and will be an error in Vespa 9." + " Remove the type specification to silence this warning."); diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedSchemas.java b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedSchemas.java index 4fd4ab7c69b..ee15b95b198 100644 --- a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedSchemas.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedSchemas.java @@ -233,8 +233,8 @@ public class ConvertParsedSchemas { var parsedType = parsedField.getType(); if (parsedType != null) { var log = schema.getDeployLogger(); - log.log(Level.FINE, () -> "For " + schema.getName() + - ", document-summary '" + parsed.name() + + log.log(Level.FINE, () -> "For schema '" + schema.getName() + + "', document-summary '" + parsed.name() + "', summary field '" + parsedField.name() + "': Specifying the type is deprecated, ignored and will be an error in Vespa 9." + " Remove the type specification to silence this warning."); diff --git a/config-model/src/test/java/com/yahoo/schema/SummaryTestCase.java b/config-model/src/test/java/com/yahoo/schema/SummaryTestCase.java index 4f008e661c8..6442edd547d 100644 --- a/config-model/src/test/java/com/yahoo/schema/SummaryTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/SummaryTestCase.java @@ -360,7 +360,7 @@ public class SummaryTestCase { if (explicit) { assertEquals(1, logger.entries.size()); assertEquals(Level.FINE, logger.entries.get(0).level); - assertEquals("For test, field 'foo', summary 'bar':" + + assertEquals("For schema 'test', field 'foo', summary 'bar':" + " Specifying the type is deprecated, ignored and will be an error in Vespa 9." + " Remove the type specification to silence this warning.", logger.entries.get(0).message); } else { @@ -393,7 +393,7 @@ public class SummaryTestCase { if (explicit) { assertEquals(1, logger.entries.size()); assertEquals(Level.FINE, logger.entries.get(0).level); - assertEquals("For test, document-summary 'bar', summary field 'foo':" + + assertEquals("For schema 'test', document-summary 'bar', summary field 'foo':" + " Specifying the type is deprecated, ignored and will be an error in Vespa 9." + " Remove the type specification to silence this warning.", logger.entries.get(0).message); } else { diff --git a/container-core/src/main/java/com/yahoo/container/Container.java b/container-core/src/main/java/com/yahoo/container/Container.java index c86f53d86bb..7f46c6d8a02 100755 --- a/container-core/src/main/java/com/yahoo/container/Container.java +++ b/container-core/src/main/java/com/yahoo/container/Container.java @@ -95,18 +95,11 @@ public class Container { } // Only intended for use by the Server instance. - public void setupFileAcquirer(QrConfig.Filedistributor filedistributorConfig) { + public void setupFileAcquirer() { if (usingCustomFileAcquirer) return; - if (filedistributorConfig.configid().isEmpty()) { - if (fileAcquirer != null) - logger.warning("Disabling file distribution"); - fileAcquirer = null; - } else { - fileAcquirer = FileAcquirerFactory.create(filedistributorConfig.configid()); - } - + fileAcquirer = FileAcquirerFactory.create(); setPathAcquirer(fileAcquirer); } diff --git a/container-core/src/main/resources/configdefinitions/container.qr.def b/container-core/src/main/resources/configdefinitions/container.qr.def index ba2207541e8..e49e334a299 100644 --- a/container-core/src/main/resources/configdefinitions/container.qr.def +++ b/container-core/src/main/resources/configdefinitions/container.qr.def @@ -5,7 +5,8 @@ namespace=container ### connection to the config system ## filedistributor rpc configuration -filedistributor.configid reference default="" restart +## TODO: Unused, remove in Vespa 9 +filedistributor.configid reference default="" ## Is RPC server enabled? rpc.enabled bool default=false restart diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java index 623d11cc473..6edfcecdda9 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java @@ -168,7 +168,7 @@ public final class ConfiguredApplication implements Application { public void start() { qrConfig = getConfig(QrConfig.class); reconfigure(qrConfig.shutdown()); - hackToInitializeServer(qrConfig); + hackToInitializeServer(); ContainerBuilder builder = createBuilderWithGuiceBindings(); configurer = createConfigurer(builder.guiceModules().activate()); @@ -242,9 +242,9 @@ public final class ConfiguredApplication implements Application { } } - private static void hackToInitializeServer(QrConfig config) { + private static void hackToInitializeServer() { try { - Container.get().setupFileAcquirer(config.filedistributor()); + Container.get().setupFileAcquirer(); Container.get().setupUrlDownloader(); } catch (Exception e) { log.log(Level.SEVERE, "Caught exception when initializing server. Exiting.", e); diff --git a/fileacquirer/abi-spec.json b/fileacquirer/abi-spec.json index 63e3ba71f2d..caec5f8629c 100644 --- a/fileacquirer/abi-spec.json +++ b/fileacquirer/abi-spec.json @@ -21,7 +21,7 @@ ], "methods" : [ "public void <init>()", - "public static com.yahoo.filedistribution.fileacquirer.FileAcquirer create(java.lang.String)" + "public static com.yahoo.filedistribution.fileacquirer.FileAcquirer create()" ], "fields" : [ ] }, diff --git a/fileacquirer/src/main/java/com/yahoo/filedistribution/fileacquirer/FileAcquirerFactory.java b/fileacquirer/src/main/java/com/yahoo/filedistribution/fileacquirer/FileAcquirerFactory.java index 24744728f78..8de5f87d8b0 100644 --- a/fileacquirer/src/main/java/com/yahoo/filedistribution/fileacquirer/FileAcquirerFactory.java +++ b/fileacquirer/src/main/java/com/yahoo/filedistribution/fileacquirer/FileAcquirerFactory.java @@ -9,8 +9,8 @@ package com.yahoo.filedistribution.fileacquirer; */ public class FileAcquirerFactory { - public static FileAcquirer create(String configId) { - return new FileAcquirerImpl(configId); + public static FileAcquirer create() { + return new FileAcquirerImpl(); } } diff --git a/fileacquirer/src/main/java/com/yahoo/filedistribution/fileacquirer/FileAcquirerImpl.java b/fileacquirer/src/main/java/com/yahoo/filedistribution/fileacquirer/FileAcquirerImpl.java index b3c4382c397..ddddcfc2170 100644 --- a/fileacquirer/src/main/java/com/yahoo/filedistribution/fileacquirer/FileAcquirerImpl.java +++ b/fileacquirer/src/main/java/com/yahoo/filedistribution/fileacquirer/FileAcquirerImpl.java @@ -1,9 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.filedistribution.fileacquirer; -import com.yahoo.cloud.config.filedistribution.FiledistributorrpcConfig; import com.yahoo.config.FileReference; -import com.yahoo.config.subscription.ConfigSubscriber; import com.yahoo.jrt.ErrorCode; import com.yahoo.jrt.Request; import com.yahoo.jrt.Spec; @@ -12,6 +10,7 @@ import com.yahoo.jrt.Supervisor; import com.yahoo.jrt.Target; import com.yahoo.jrt.Transport; import com.yahoo.vespa.config.FileReferenceDoesNotExistException; + import java.io.File; import java.time.Duration; import java.util.concurrent.TimeUnit; @@ -20,6 +19,8 @@ import java.util.concurrent.locks.ReentrantLock; import java.util.logging.Level; import java.util.logging.Logger; +import static com.yahoo.net.HostName.getLocalhost; + /** * Retrieves the path to a file or directory on the local file system * that has been transferred with the vespa file distribution @@ -43,15 +44,15 @@ class FileAcquirerImpl implements FileAcquirer { private final Supervisor supervisor = new Supervisor(new Transport("fileaquirer")); - private final ConfigSubscriber configSubscriber; + private class Connection { - private class Connection implements ConfigSubscriber.SingleSubscriber<FiledistributorrpcConfig> { - private final Lock targetLock = new ReentrantLock(); - private Target target; + private static final int configProxyRpcPort = 19090; - private volatile Spec spec; + private final Lock targetLock = new ReentrantLock(); + private final Spec spec = new Spec(getLocalhost(), configProxyRpcPort); private long pauseTime = 0; //milliseconds + private Target target; private long nextLogTime = 0; private long logCount = 0; @@ -85,7 +86,7 @@ class FileAcquirerImpl implements FileAcquirer { private void logWarning() { if (logCount == 0 || System.currentTimeMillis() > nextLogTime ) { - log.warning("Could not connect to the config proxy '" + spec.toString() + "'" + " - " + this + "@" + System.identityHashCode(this)); + log.warning("Could not connect to the config proxy '" + spec + "'" + " - " + this + "@" + System.identityHashCode(this)); nextLogTime = System.currentTimeMillis() + Math.min(TimeUnit.DAYS.toMillis(1), @@ -94,11 +95,6 @@ class FileAcquirerImpl implements FileAcquirer { } } - @Override - public void configure(FiledistributorrpcConfig filedistributorrpcConfig) { - spec = new Spec(filedistributorrpcConfig.connectionspec()); - } - public Target getTarget(Timer timer) throws InterruptedException { TimeUnit unit = TimeUnit.MILLISECONDS; @@ -122,17 +118,10 @@ class FileAcquirerImpl implements FileAcquirer { }; } - public FileAcquirerImpl(String configId) { - configSubscriber = new ConfigSubscriber(); - configSubscriber.subscribe(connection, FiledistributorrpcConfig.class, configId); - } - public void shutdown() { - configSubscriber.close(); supervisor.transport().shutdown().join(); } - /** * Returns the path to a file or directory corresponding to the * given file reference. File references are produced by the diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index d701a35c2ef..06530fdc962 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -14,12 +14,10 @@ import java.util.TreeMap; import java.util.function.Predicate; import static com.yahoo.vespa.flags.FetchVector.Dimension.APPLICATION; -import static com.yahoo.vespa.flags.FetchVector.Dimension.INSTANCE_ID; import static com.yahoo.vespa.flags.FetchVector.Dimension.CLOUD_ACCOUNT; -import static com.yahoo.vespa.flags.FetchVector.Dimension.CLUSTER_ID; -import static com.yahoo.vespa.flags.FetchVector.Dimension.CLUSTER_TYPE; import static com.yahoo.vespa.flags.FetchVector.Dimension.CONSOLE_USER_EMAIL; import static com.yahoo.vespa.flags.FetchVector.Dimension.HOSTNAME; +import static com.yahoo.vespa.flags.FetchVector.Dimension.INSTANCE_ID; import static com.yahoo.vespa.flags.FetchVector.Dimension.NODE_TYPE; import static com.yahoo.vespa.flags.FetchVector.Dimension.TENANT_ID; import static com.yahoo.vespa.flags.FetchVector.Dimension.VESPA_VERSION; @@ -374,13 +372,6 @@ public class Flags { "Takes effect on next host provisioning / run of host-admin", HOSTNAME, CLOUD_ACCOUNT); - public static final UnboundIntFlag MIN_EXCLUSIVE_ADVERTISED_MEMORY_GB = defineIntFlag( - "min-exclusive-advertised-memory-gb", 8, - List.of("freva"), "2023-09-08", "2024-01-15", - "Minimum amount of advertised memory for exclusive nodes", - "Takes effect immediately", - INSTANCE_ID, CLUSTER_ID, CLUSTER_TYPE); - public static final UnboundIntFlag CONTENT_LAYER_METADATA_FEATURE_LEVEL = defineIntFlag( "content-layer-metadata-feature-level", 0, List.of("vekterli"), "2022-09-12", "2024-02-01", diff --git a/hosted-tenant-base/pom.xml b/hosted-tenant-base/pom.xml index 9851659bef0..7dd66003581 100644 --- a/hosted-tenant-base/pom.xml +++ b/hosted-tenant-base/pom.xml @@ -262,6 +262,22 @@ </configuration> </execution> <execution> + <id>enforce-no-compile-scope</id> + <goals> + <goal>enforce</goal> + </goals> + <configuration> + <rules> + <bannedDependencies> + <!-- Fail validation for provided container deps which will fail when embedded in application bundle --> + <excludes> + <exclude>org.slf4j:slf4j-api:*:jar:compile</exclude> + </excludes> + </bannedDependencies> + </rules> + </configuration> + </execution> + <execution> <id>enforce-just-our-jupiter</id> <goals> <goal>enforce</goal> diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java index dfbe41e31d7..a4a4c42d9c1 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java @@ -7,6 +7,8 @@ import com.yahoo.concurrent.maintenance.JobControl; import com.yahoo.config.provision.ApplicationTransaction; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.DockerImage; +import com.yahoo.config.provision.EndpointsChecker.HealthChecker; +import com.yahoo.config.provision.EndpointsChecker.HealthCheckerProvider; import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.Zone; import com.yahoo.config.provisioning.NodeRepositoryConfig; @@ -19,6 +21,8 @@ import com.yahoo.vespa.hosted.provision.Node.State; import com.yahoo.vespa.hosted.provision.applications.Applications; import com.yahoo.vespa.hosted.provision.archive.ArchiveUriManager; import com.yahoo.vespa.hosted.provision.autoscale.MetricsDb; +import com.yahoo.vespa.hosted.provision.lb.LoadBalancer; +import com.yahoo.vespa.hosted.provision.lb.LoadBalancerInstance; import com.yahoo.vespa.hosted.provision.lb.LoadBalancers; import com.yahoo.vespa.hosted.provision.maintenance.InfrastructureVersions; import com.yahoo.vespa.hosted.provision.node.Agent; @@ -34,6 +38,7 @@ import com.yahoo.vespa.hosted.provision.provisioning.FirmwareChecks; import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator; import com.yahoo.vespa.hosted.provision.provisioning.NodeResourceLimits; import com.yahoo.vespa.hosted.provision.provisioning.ProvisionServiceProvider; +import com.yahoo.vespa.hosted.provision.provisioning.ProvisionServiceProvider.ProtoHealthChecker; import com.yahoo.vespa.orchestrator.Orchestrator; import java.time.Clock; @@ -45,7 +50,7 @@ import java.util.Optional; * * @author bratseth */ -public class NodeRepository extends AbstractComponent { +public class NodeRepository extends AbstractComponent implements HealthCheckerProvider { private final CuratorDb db; private final Clock clock; @@ -67,6 +72,7 @@ public class NodeRepository extends AbstractComponent { private final MetricsDb metricsDb; private final Orchestrator orchestrator; private final int spareCount; + private final ProtoHealthChecker healthChecker; private final JacksonFlag<SharedHost> sharedHosts; /** @@ -142,6 +148,7 @@ public class NodeRepository extends AbstractComponent { this.orchestrator = orchestrator; this.spareCount = spareCount; this.sharedHosts = PermanentFlags.SHARED_HOST.bindTo(flagSource()); + this.healthChecker = provisionServiceProvider.getHealthChecker(); nodes.rewrite(); } @@ -250,4 +257,14 @@ public class NodeRepository extends AbstractComponent { return Optional.of(image).filter(s -> !s.isEmpty()).map(DockerImage::fromString); } + @Override + public HealthChecker getHealthChecker() { + return endpoint -> healthChecker.healthy(endpoint, + loadBalancers.list(endpoint.applicationId()) + .cluster(endpoint.clusterName()) + .first() + .flatMap(LoadBalancer::instance) + .flatMap(LoadBalancerInstance::idSeed)); + } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerInstance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerInstance.java index c0931ecbc70..04ea9d20edf 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerInstance.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerInstance.java @@ -11,6 +11,7 @@ import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.UUID; /** * Represents a load balancer instance. This contains the fields that are owned by a {@link LoadBalancerService} and is @@ -20,6 +21,7 @@ import java.util.Set; */ public class LoadBalancerInstance { + private final Optional<UUID> idSeed; private final Optional<DomainName> hostname; private final Optional<String> ip4Address; private final Optional<String> ip6Address; @@ -31,9 +33,10 @@ public class LoadBalancerInstance { private final List<PrivateServiceId> serviceIds; private final CloudAccount cloudAccount; - public LoadBalancerInstance(Optional<DomainName> hostname, Optional<String> ip4Address, Optional<String> ip6Address, + public LoadBalancerInstance(Optional<UUID> idSeed, Optional<DomainName> hostname, Optional<String> ip4Address, Optional<String> ip6Address, Optional<DnsZone> dnsZone, Set<Integer> ports, Set<String> networks, Set<Real> reals, ZoneEndpoint settings, List<PrivateServiceId> serviceIds, CloudAccount cloudAccount) { + this.idSeed = Objects.requireNonNull(idSeed, "idSeed must be non-null"); this.hostname = Objects.requireNonNull(hostname, "hostname must be non-null"); this.ip4Address = Objects.requireNonNull(ip4Address, "ip4Address must be non-null"); this.ip6Address = Objects.requireNonNull(ip6Address, "ip6Address must be non-null"); @@ -51,6 +54,11 @@ public class LoadBalancerInstance { } } + /** A unique seed to use when generating cloud-specific resource IDs for this load balancer instance. */ + public Optional<UUID> idSeed() { + return idSeed; + } + /** Fully-qualified domain name of this load balancer. This hostname can be used for query and feed */ public Optional<DomainName> hostname() { return hostname; @@ -121,7 +129,7 @@ public class LoadBalancerInstance { public LoadBalancerInstance with(Set<Real> reals, ZoneEndpoint settings, Optional<PrivateServiceId> serviceId) { List<PrivateServiceId> ids = new ArrayList<>(serviceIds); serviceId.filter(id -> ! ids.contains(id)).ifPresent(ids::add); - return new LoadBalancerInstance(hostname, ip4Address, ip6Address, dnsZone, ports, networks, + return new LoadBalancerInstance(idSeed, hostname, ip4Address, ip6Address, dnsZone, ports, networks, reals, settings, ids, cloudAccount); } @@ -130,7 +138,7 @@ public class LoadBalancerInstance { public LoadBalancerInstance withServiceIds(List<PrivateServiceId> serviceIds) { List<PrivateServiceId> ids = new ArrayList<>(serviceIds); for (PrivateServiceId id : this.serviceIds) if ( ! ids.contains(id)) ids.add(id); - return new LoadBalancerInstance(hostname, ip4Address, ip6Address, dnsZone, ports, networks, reals, settings, ids, cloudAccount); + return new LoadBalancerInstance(idSeed, hostname, ip4Address, ip6Address, dnsZone, ports, networks, reals, settings, ids, cloudAccount); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerService.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerService.java index 6d41f664fad..40323a4f2d3 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerService.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerService.java @@ -2,15 +2,20 @@ package com.yahoo.vespa.hosted.provision.lb; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.EndpointsChecker.Availability; +import com.yahoo.config.provision.EndpointsChecker.Endpoint; import com.yahoo.config.provision.EndpointsChecker.HealthChecker; import com.yahoo.config.provision.NodeType; +import java.util.Optional; +import java.util.UUID; + /** * A managed load balance service. * * @author mpolden */ -public interface LoadBalancerService extends HealthChecker { +public interface LoadBalancerService { /** * Provisions load balancers from the given specification. Implementations are expected to be idempotent @@ -40,6 +45,9 @@ public interface LoadBalancerService extends HealthChecker { /** Returns whether load balancers created by this service can forward traffic to given node and cluster type */ boolean supports(NodeType nodeType, ClusterSpec.Type clusterType); + /** See {@link HealthChecker#healthy(Endpoint)}. */ + Availability healthy(Endpoint endpoint, Optional<UUID> idSeed); + /** Load balancer protocols */ enum Protocol { ipv4, diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java index df4b83d1543..3b444312b14 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java @@ -14,6 +14,8 @@ import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicBoolean; /** * @author mpolden @@ -23,6 +25,7 @@ public class LoadBalancerServiceMock implements LoadBalancerService { private final Map<LoadBalancerId, LoadBalancerInstance> instances = new HashMap<>(); private boolean throwOnCreate = false; private boolean supportsProvisioning = true; + private final AtomicBoolean uuid = new AtomicBoolean(true); public Map<LoadBalancerId, LoadBalancerInstance> instances() { return Collections.unmodifiableMap(instances); @@ -53,8 +56,10 @@ public class LoadBalancerServiceMock implements LoadBalancerService { @Override public LoadBalancerInstance provision(LoadBalancerSpec spec) { if (throwOnCreate) throw new IllegalStateException("Did not expect a new load balancer to be created"); + Optional<UUID> idSeed = uuid.getAndSet(false) ? Optional.of(UUID.fromString("c11272ab-d20e-4c86-b808-ffedaa00c480")) : Optional.empty(); var id = new LoadBalancerId(spec.application(), spec.cluster()); var instance = new LoadBalancerInstance( + idSeed, Optional.of(DomainName.of("lb-" + spec.application().toShortString() + "-" + spec.cluster().value())), Optional.empty(), Optional.empty(), @@ -89,7 +94,7 @@ public class LoadBalancerServiceMock implements LoadBalancerService { } @Override - public Availability healthy(Endpoint endpoint) { + public Availability healthy(Endpoint endpoint, Optional<UUID> idSeed) { return Availability.ready; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java index e463f5aabe6..eb99e5647df 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java @@ -11,6 +11,7 @@ import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.UUID; /** * This implementation of {@link LoadBalancerService} returns the load balancer(s) that exist by default in the shared @@ -42,7 +43,8 @@ public class SharedLoadBalancerService implements LoadBalancerService { private LoadBalancerInstance create(LoadBalancerSpec spec) { if ( ! spec.settings().isPublicEndpoint()) throw new IllegalArgumentException("non-public endpoints is not supported with " + getClass()); - return new LoadBalancerInstance(Optional.of(DomainName.of(vipHostname)), + return new LoadBalancerInstance(Optional.empty(), + Optional.of(DomainName.of(vipHostname)), Optional.empty(), Optional.empty(), Optional.empty(), @@ -72,7 +74,7 @@ public class SharedLoadBalancerService implements LoadBalancerService { } @Override - public Availability healthy(Endpoint endpoint) { + public Availability healthy(Endpoint endpoint, Optional<UUID> idSeed) { return Availability.ready; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java index 7262daf758b..9e81cd3d3b7 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java @@ -26,7 +26,9 @@ import java.util.LinkedHashSet; import java.util.List; import java.util.Optional; import java.util.Set; +import java.util.UUID; import java.util.function.Function; +import java.util.function.Predicate; /** * Serializer for load balancers. @@ -43,6 +45,7 @@ public class LoadBalancerSerializer { // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. private static final String idField = "id"; + private static final String idSeedField = "idSeed"; private static final String hostnameField = "hostname"; private static final String lbIpAddressField = "ipAddress"; private static final String lbIp6AddressField = "ip6Address"; @@ -69,6 +72,7 @@ public class LoadBalancerSerializer { Cursor root = slime.setObject(); root.setString(idField, loadBalancer.id().serializedForm()); + loadBalancer.instance().flatMap(LoadBalancerInstance::idSeed).ifPresent(idSeed -> root.setString(idSeedField, idSeed.toString())); loadBalancer.instance().flatMap(LoadBalancerInstance::hostname).ifPresent(hostname -> root.setString(hostnameField, hostname.value())); loadBalancer.instance().flatMap(LoadBalancerInstance::ip4Address).ifPresent(ip -> root.setString(lbIpAddressField, ip)); loadBalancer.instance().flatMap(LoadBalancerInstance::ip6Address).ifPresent(ip -> root.setString(lbIp6AddressField, ip)); @@ -124,9 +128,10 @@ public class LoadBalancerSerializer { Set<String> networks = new LinkedHashSet<>(); object.field(networksField).traverse((ArrayTraverser) (i, network) -> networks.add(network.asString())); - Optional<DomainName> hostname = optionalString(object.field(hostnameField), Function.identity()).filter(s -> !s.isEmpty()).map(DomainName::of); - Optional<String> ip4Address = optionalString(object.field(lbIpAddressField), Function.identity()).filter(s -> !s.isEmpty()); - Optional<String> ip6Address = optionalString(object.field(lbIp6AddressField), Function.identity()).filter(s -> !s.isEmpty()); + Optional<UUID> idSeed = SlimeUtils.optionalString(object.field(idSeedField)).map(UUID::fromString); + Optional<DomainName> hostname = SlimeUtils.optionalString(object.field(hostnameField)).map(DomainName::of); + Optional<String> ip4Address = SlimeUtils.optionalString(object.field(lbIpAddressField)); + Optional<String> ip6Address = SlimeUtils.optionalString(object.field(lbIp6AddressField)); Optional<DnsZone> dnsZone = optionalString(object.field(dnsZoneField), DnsZone::new); ZoneEndpoint settings = zoneEndpoint(object.field(settingsField)); Optional<PrivateServiceId> serviceId = optionalString(object.field(serviceIdField), PrivateServiceId::of); @@ -136,7 +141,7 @@ public class LoadBalancerSerializer { CloudAccount cloudAccount = optionalString(object.field(cloudAccountField), CloudAccount::from).orElse(CloudAccount.empty); Optional<LoadBalancerInstance> instance = hostname.isEmpty() && ip4Address.isEmpty() && ip6Address.isEmpty() ? Optional.empty() - : Optional.of(new LoadBalancerInstance(hostname, ip4Address, ip6Address, dnsZone, ports, networks, reals, settings, serviceIds, cloudAccount)); + : Optional.of(new LoadBalancerInstance(idSeed, hostname, ip4Address, ip6Address, dnsZone, ports, networks, reals, settings, serviceIds, cloudAccount)); return new LoadBalancer(LoadBalancerId.fromSerializedForm(object.field(idField).asString()), instance, diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java index cd331cbd5fa..e2f07163e9d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java @@ -7,9 +7,6 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.Zone; -import com.yahoo.vespa.flags.FetchVector; -import com.yahoo.vespa.flags.Flags; -import com.yahoo.vespa.flags.IntFlag; import com.yahoo.vespa.hosted.provision.NodeRepository; import java.util.Locale; @@ -23,11 +20,9 @@ import java.util.Locale; public class NodeResourceLimits { private final NodeRepository nodeRepository; - private final IntFlag minExclusiveAdvertisedMemoryGbFlag; public NodeResourceLimits(NodeRepository nodeRepository) { this.nodeRepository = nodeRepository; - this.minExclusiveAdvertisedMemoryGbFlag = Flags.MIN_EXCLUSIVE_ADVERTISED_MEMORY_GB.bindTo(nodeRepository.flagSource()); } /** Validates the resources applications ask for (which are in "advertised" resource space) */ @@ -87,11 +82,7 @@ public class NodeResourceLimits { private double minAdvertisedMemoryGb(ApplicationId applicationId, ClusterSpec cluster, boolean exclusive) { if (cluster.type() == ClusterSpec.Type.admin) return 1; if (!exclusive) return 4; - return minExclusiveAdvertisedMemoryGbFlag - .with(FetchVector.Dimension.INSTANCE_ID, applicationId.serializedForm()) - .with(FetchVector.Dimension.CLUSTER_ID, cluster.id().value()) - .with(FetchVector.Dimension.CLUSTER_TYPE, cluster.type().name()) - .value(); + return 8; } private double minAdvertisedDiskGb(NodeResources requested, boolean exclusive) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionServiceProvider.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionServiceProvider.java index 601a4008110..7b7971318cd 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionServiceProvider.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionServiceProvider.java @@ -1,17 +1,20 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; +import com.yahoo.config.provision.EndpointsChecker.Availability; +import com.yahoo.config.provision.EndpointsChecker.Endpoint; import com.yahoo.config.provision.EndpointsChecker.HealthCheckerProvider; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerService; import java.util.Optional; +import java.util.UUID; /** * Injectable component that provides provision service for load-balancers and hosts * * @author freva */ -public interface ProvisionServiceProvider extends HealthCheckerProvider { +public interface ProvisionServiceProvider { Optional<LoadBalancerService> getLoadBalancerService(); @@ -19,4 +22,12 @@ public interface ProvisionServiceProvider extends HealthCheckerProvider { HostResourcesCalculator getHostResourcesCalculator(); + default ProtoHealthChecker getHealthChecker() { + return (endpoint, idSeed) -> Availability.ready; + } + + interface ProtoHealthChecker { + Availability healthy(Endpoint endpoint, Optional<UUID> idSeed); + } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersResponse.java index 175ea8d294e..e40e1ba5951 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersResponse.java @@ -90,6 +90,7 @@ public class LoadBalancersResponse extends SlimeJsonResponse { } instance.serviceId().ifPresent(serviceId -> lbObject.setString("serviceId", serviceId.value())); lbObject.setBool("public", instance.settings().isPublicEndpoint()); + instance.idSeed().ifPresent(idSeed -> lbObject.setString("idSeed", idSeed.toString())); }); lb.instance() .map(LoadBalancerInstance::cloudAccount) diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializerTest.java index 17bb7502484..146b9df1722 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializerTest.java @@ -20,6 +20,7 @@ import java.time.Instant; import java.util.List; import java.util.Optional; import java.util.Set; +import java.util.UUID; import static java.time.temporal.ChronoUnit.MILLIS; import static org.junit.Assert.assertEquals; @@ -38,6 +39,7 @@ public class LoadBalancerSerializerTest { { var loadBalancer = new LoadBalancer(loadBalancerId, Optional.of(new LoadBalancerInstance( + Optional.of(UUID.randomUUID()), Optional.of(DomainName.of("lb-host")), Optional.empty(), Optional.empty(), @@ -58,6 +60,7 @@ public class LoadBalancerSerializerTest { var serialized = LoadBalancerSerializer.fromJson(LoadBalancerSerializer.toJson(loadBalancer)); assertEquals(loadBalancer.id(), serialized.id()); + assertEquals(loadBalancer.instance().get().idSeed(), serialized.instance().get().idSeed()); assertEquals(loadBalancer.instance().get().hostname(), serialized.instance().get().hostname()); assertEquals(loadBalancer.instance().get().dnsZone(), serialized.instance().get().dnsZone()); assertEquals(loadBalancer.instance().get().ports(), serialized.instance().get().ports()); @@ -73,6 +76,7 @@ public class LoadBalancerSerializerTest { var loadBalancer = new LoadBalancer(loadBalancerId, Optional.of(new LoadBalancerInstance( Optional.empty(), + Optional.empty(), Optional.of("1.2.3.4"), Optional.of("fd00::1"), Optional.of(new DnsZone("zone-id-1")), @@ -87,6 +91,7 @@ public class LoadBalancerSerializerTest { var serialized = LoadBalancerSerializer.fromJson(LoadBalancerSerializer.toJson(loadBalancer)); assertEquals(loadBalancer.id(), serialized.id()); + assertEquals(loadBalancer.instance().get().idSeed(), serialized.instance().get().idSeed()); assertEquals(loadBalancer.instance().get().hostname(), serialized.instance().get().hostname()); assertEquals(loadBalancer.instance().get().ip4Address(), serialized.instance().get().ip4Address()); assertEquals(loadBalancer.instance().get().ip6Address(), serialized.instance().get().ip6Address()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/load-balancers.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/load-balancers.json index 96213dea0a7..e0c370fc376 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/load-balancers.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/load-balancers.json @@ -1,15 +1,13 @@ { "loadBalancers": [ { - "id": "tenant1:application1:instance1:id1", - "state": "active", - "changedAt": 123, "application": "application1", - "tenant": "tenant1", - "instance": "instance1", + "changedAt": 123, "cluster": "id1", - "hostname": "lb-tenant1.application1.instance1-id1", "dnsZone": "zone-id-1", + "hostname": "lb-tenant1.application1.instance1-id1", + "id": "tenant1:application1:instance1:id1", + "instance": "instance1", "networks": [ "10.2.3.0/24", "10.4.5.0/24" @@ -17,6 +15,7 @@ "ports": [ 4443 ], + "public": false, "reals": [ { "hostname": "host1.yahoo.com", @@ -29,6 +28,7 @@ "port": 4443 } ], + "serviceId": "service", "settings": { "allowedUrns": [ { @@ -37,19 +37,18 @@ } ] }, - "serviceId": "service", - "public": false + "state": "active", + "tenant": "tenant1" }, { - "id": "hosted-vespa:zone-config-servers:default:zone-config-servers", - "state": "active", - "changedAt": 123, "application": "zone-config-servers", - "tenant": "hosted-vespa", - "instance": "default", + "changedAt": 123, "cluster": "zone-config-servers", - "hostname": "lb-hosted-vespa.zone-config-servers-zone-config-servers", "dnsZone": "zone-id-1", + "hostname": "lb-hosted-vespa.zone-config-servers-zone-config-servers", + "id": "hosted-vespa:zone-config-servers:default:zone-config-servers", + "idSeed": "c11272ab-d20e-4c86-b808-ffedaa00c480", + "instance": "default", "networks": [ "10.2.3.0/24", "10.4.5.0/24" @@ -57,6 +56,7 @@ "ports": [ 4443 ], + "public": true, "reals": [ { "hostname": "cfg1.yahoo.com", @@ -69,18 +69,17 @@ "port": 4443 } ], - "public": true + "state": "active", + "tenant": "hosted-vespa" }, { - "id": "tenant4:application4:instance4:id4", - "state": "active", - "changedAt": 123, "application": "application4", - "tenant": "tenant4", - "instance": "instance4", + "changedAt": 123, "cluster": "id4", - "hostname": "lb-tenant4.application4.instance4-id4", "dnsZone": "zone-id-1", + "hostname": "lb-tenant4.application4.instance4-id4", + "id": "tenant4:application4:instance4:id4", + "instance": "instance4", "networks": [ "10.2.3.0/24", "10.4.5.0/24" @@ -88,6 +87,7 @@ "ports": [ 4443 ], + "public": true, "reals": [ { "hostname": "host13.yahoo.com", @@ -100,7 +100,8 @@ "port": 4443 } ], - "public": true + "state": "active", + "tenant": "tenant4" } ] -} +}
\ No newline at end of file diff --git a/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp b/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp index f5c9e8baffa..c9951f0b60e 100644 --- a/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp +++ b/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp @@ -23,7 +23,6 @@ #include <vespa/vespalib/test/insertion_operators.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/hw_info.h> -#include <vespa/vespalib/util/size_literals.h> #include <vespa/vespalib/util/threadstackexecutor.h> #include <filesystem> #include <thread> @@ -188,7 +187,7 @@ assertWhiteList(const SimpleResult &exp, Blueprint::UP whiteListBlueprint, bool { MatchDataLayout mdl; MatchData::UP md = mdl.createMatchData(); - whiteListBlueprint->fetchPostings(search::queryeval::ExecuteInfo::create(strict)); + whiteListBlueprint->fetchPostings(search::queryeval::ExecuteInfo::createForTest(strict)); whiteListBlueprint->setDocIdLimit(docIdLimit); SearchIterator::UP sb = whiteListBlueprint->createSearch(*md, strict); diff --git a/searchcore/src/tests/proton/matching/query_test.cpp b/searchcore/src/tests/proton/matching/query_test.cpp index 928e13a9fae..0b87734cec2 100644 --- a/searchcore/src/tests/proton/matching/query_test.cpp +++ b/searchcore/src/tests/proton/matching/query_test.cpp @@ -711,7 +711,7 @@ void Test::requireThatQueryGluesEverythingTogether() { EXPECT_EQUAL(1u, md->getNumTermFields()); query.optimize(); - query.fetchPostings(search::queryeval::ExecuteInfo::create(true, &requestContext.getDoom())); + query.fetchPostings(ExecuteInfo::create(true, 1.0F, &requestContext.getDoom())); SearchIterator::UP search = query.createSearch(*md); ASSERT_TRUE(search.get()); } @@ -744,7 +744,7 @@ void checkQueryAddsLocation(const string &loc_in, const string &loc_out) { MatchData::UP md = mdl.createMatchData(); EXPECT_EQUAL(2u, md->getNumTermFields()); - query.fetchPostings(search::queryeval::ExecuteInfo::create(true, &requestContext.getDoom())); + query.fetchPostings(ExecuteInfo::create(true, 1.0F, &requestContext.getDoom())); SearchIterator::UP search = query.createSearch(*md); ASSERT_TRUE(search.get()); if (!EXPECT_NOT_EQUAL(string::npos, search->asString().find(loc_out))) { @@ -966,7 +966,7 @@ Test::requireThatWhiteListBlueprintCanBeUsed() MatchData::UP md = mdl.createMatchData(); query.optimize(); - query.fetchPostings(search::queryeval::ExecuteInfo::create(true, &requestContext.getDoom())); + query.fetchPostings(ExecuteInfo::create(true, 1.0F, &requestContext.getDoom())); SearchIterator::UP search = query.createSearch(*md); SimpleResult exp = SimpleResult().addHit(1).addHit(5).addHit(7).addHit(11); SimpleResult act; @@ -1129,7 +1129,7 @@ public: { set_want_global_filter(want_global_filter); } - ~GlobalFilterBlueprint(); + ~GlobalFilterBlueprint() override; void set_global_filter(const GlobalFilter& filter_, double estimated_hit_ratio_) override { filter = filter_.shared_from_this(); estimated_hit_ratio = estimated_hit_ratio_; diff --git a/searchcore/src/vespa/searchcore/proton/matching/query.cpp b/searchcore/src/vespa/searchcore/proton/matching/query.cpp index 955cd30714f..00a3f6bab7e 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/query.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/query.cpp @@ -247,7 +247,7 @@ Query::optimize() } void -Query::fetchPostings(const search::queryeval::ExecuteInfo & executeInfo) +Query::fetchPostings(const ExecuteInfo & executeInfo) { _blueprint->fetchPostings(executeInfo); } @@ -265,7 +265,7 @@ Query::handle_global_filter(const vespalib::Doom & doom, uint32_t docid_limit, _blueprint = Blueprint::optimize(std::move(_blueprint)); LOG(debug, "blueprint after handle_global_filter:\n%s\n", _blueprint->asString().c_str()); // strictness may change if optimized order changed: - fetchPostings(search::queryeval::ExecuteInfo::create(true, &doom)); + fetchPostings(ExecuteInfo::create(true, 1.0F, &doom)); } bool diff --git a/searchcore/src/vespa/searchcore/proton/matching/query.h b/searchcore/src/vespa/searchcore/proton/matching/query.h index c2d2d389d2c..1b5d6dbca60 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/query.h +++ b/searchcore/src/vespa/searchcore/proton/matching/query.h @@ -23,6 +23,7 @@ class Query private: using Blueprint = search::queryeval::Blueprint; using GlobalFilter = search::queryeval::GlobalFilter; + using ExecuteInfo = search::queryeval::ExecuteInfo; search::query::Node::UP _query_tree; Blueprint::UP _blueprint; Blueprint::UP _whiteListBlueprint; @@ -98,7 +99,7 @@ public: * test to verify the original query without optimization. **/ void optimize(); - void fetchPostings(const search::queryeval::ExecuteInfo & executeInfo) ; + void fetchPostings(const ExecuteInfo & executeInfo) ; void handle_global_filter(const vespalib::Doom & doom, uint32_t docid_limit, double global_filter_lower_limit, double global_filter_upper_limit, diff --git a/searchlib/src/tests/attribute/enum_comparator/enum_comparator_test.cpp b/searchlib/src/tests/attribute/enum_comparator/enum_comparator_test.cpp index f99a4c4b3f9..c578f823dfd 100644 --- a/searchlib/src/tests/attribute/enum_comparator/enum_comparator_test.cpp +++ b/searchlib/src/tests/attribute/enum_comparator/enum_comparator_test.cpp @@ -267,14 +267,14 @@ TEST(EnumComparatorTest, require_that_cased_less_is_working) EXPECT_EQ((EnumIndexVector{e1, e4, e3, e2}), vec); } -TEST(DfaStringComparatorTest, require_that_less_is_working) +TEST(DfaStringComparatorTest, require_that_folded_less_is_working) { StringEnumStore es(false, DictionaryConfig::Type::BTREE); EnumIndex e1 = es.insert("Aa"); EnumIndex e2 = es.insert("aa"); EnumIndex e3 = es.insert("aB"); auto aa_utf32 = as_utf32("aa"); - DfaStringComparator cmp1(es.get_data_store(), aa_utf32); + DfaStringComparator cmp1(es.get_data_store(), aa_utf32, false); EXPECT_FALSE(cmp1.less(EnumIndex(), e1)); EXPECT_FALSE(cmp1.less(EnumIndex(), e2)); EXPECT_TRUE(cmp1.less(EnumIndex(), e3)); @@ -282,7 +282,7 @@ TEST(DfaStringComparatorTest, require_that_less_is_working) EXPECT_FALSE(cmp1.less(e2, EnumIndex())); EXPECT_FALSE(cmp1.less(e3, EnumIndex())); auto Aa_utf32 = as_utf32("Aa"); - DfaStringComparator cmp2(es.get_data_store(), Aa_utf32); + DfaStringComparator cmp2(es.get_data_store(), Aa_utf32, false); EXPECT_TRUE(cmp2.less(EnumIndex(), e1)); EXPECT_TRUE(cmp2.less(EnumIndex(), e2)); EXPECT_TRUE(cmp2.less(EnumIndex(), e3)); @@ -291,6 +291,39 @@ TEST(DfaStringComparatorTest, require_that_less_is_working) EXPECT_FALSE(cmp2.less(e3, EnumIndex())); } +TEST(DfaStringComparatorTest, require_that_cased_less_is_working) +{ + StringEnumStore es(false, DictionaryConfig(DictionaryConfig::Type::BTREE, DictionaryConfig::Match::CASED)); + auto e1 = es.insert("Aa"); + auto e2 = es.insert("aa"); + auto e3 = es.insert("aB"); + auto uaa_utf32 = as_utf32("Aa"); + auto aa_utf32 = as_utf32("aa"); + DfaStringComparator cmp1(es.get_data_store(), uaa_utf32, true); + DfaStringComparator cmp2(es.get_data_store(), aa_utf32, true); + EXPECT_FALSE(cmp1.less(e1, e1)); + EXPECT_TRUE(cmp1.less(e1, e2)); + EXPECT_TRUE(cmp1.less(e1, e3)); + EXPECT_FALSE(cmp1.less(e2, e1)); + EXPECT_FALSE(cmp1.less(e2, e2)); + EXPECT_FALSE(cmp1.less(e2, e3)); + EXPECT_FALSE(cmp1.less(e3, e1)); + EXPECT_TRUE(cmp1.less(e3, e2)); + EXPECT_FALSE(cmp1.less(e3, e3)); + EXPECT_FALSE(cmp1.less(EnumIndex(), e1)); + EXPECT_TRUE(cmp1.less(EnumIndex(), e2)); + EXPECT_TRUE(cmp1.less(EnumIndex(), e3)); + EXPECT_FALSE(cmp2.less(EnumIndex(), e1)); + EXPECT_FALSE(cmp2.less(EnumIndex(), e2)); + EXPECT_FALSE(cmp2.less(EnumIndex(), e3)); + EXPECT_FALSE(cmp1.less(e1, EnumIndex())); + EXPECT_FALSE(cmp1.less(e2, EnumIndex())); + EXPECT_FALSE(cmp1.less(e3, EnumIndex())); + EXPECT_TRUE(cmp2.less(e1, EnumIndex())); + EXPECT_FALSE(cmp2.less(e2, EnumIndex())); + EXPECT_TRUE(cmp2.less(e3, EnumIndex())); +} + } GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/tests/attribute/imported_search_context/imported_search_context_test.cpp b/searchlib/src/tests/attribute/imported_search_context/imported_search_context_test.cpp index 17a393d97eb..79bdd83dc88 100644 --- a/searchlib/src/tests/attribute/imported_search_context/imported_search_context_test.cpp +++ b/searchlib/src/tests/attribute/imported_search_context/imported_search_context_test.cpp @@ -272,7 +272,7 @@ TEST_F("Strict iterator is marked as strict", Fixture) { TEST_F("Non-strict blueprint with high hit rate is strict", Fixture(false, FastSearchConfig::ExplicitlyEnabled)) { auto ctx = f.create_context(word_term("5678")); - ctx->fetchPostings(queryeval::ExecuteInfo::create(false, 0.02)); + ctx->fetchPostings(queryeval::ExecuteInfo::createForTest(false, 0.02)); TermFieldMatchData match; auto iter = f.create_iterator(*ctx, match, false); @@ -281,7 +281,7 @@ TEST_F("Non-strict blueprint with high hit rate is strict", Fixture(false, FastS TEST_F("Non-strict blueprint with low hit rate is non-strict", Fixture(false, FastSearchConfig::ExplicitlyEnabled)) { auto ctx = f.create_context(word_term("5678")); - ctx->fetchPostings(queryeval::ExecuteInfo::create(false, 0.01)); + ctx->fetchPostings(queryeval::ExecuteInfo::createForTest(false, 0.01)); TermFieldMatchData match; auto iter = f.create_iterator(*ctx, match, false); diff --git a/searchlib/src/tests/attribute/searchable/attribute_searchable_adapter_test.cpp b/searchlib/src/tests/attribute/searchable/attribute_searchable_adapter_test.cpp index 03e338ee284..029c3130609 100644 --- a/searchlib/src/tests/attribute/searchable/attribute_searchable_adapter_test.cpp +++ b/searchlib/src/tests/attribute/searchable/attribute_searchable_adapter_test.cpp @@ -220,7 +220,7 @@ Result do_search(IAttributeManager &attribute_manager, const Node &node, bool st Blueprint::UP bp = source.createBlueprint(requestContext, FieldSpec(field, fieldId, handle), node); ASSERT_TRUE(bp); Result result(bp->getState().estimate().estHits, bp->getState().estimate().empty); - bp->fetchPostings(queryeval::ExecuteInfo::create(strict, 1.0)); + bp->fetchPostings(queryeval::ExecuteInfo::createForTest(strict)); SearchIterator::UP iterator = bp->createSearch(*match_data, strict); ASSERT_TRUE(iterator); iterator->initRange(1, num_docs); diff --git a/searchlib/src/tests/attribute/searchable/attribute_weighted_set_blueprint_test.cpp b/searchlib/src/tests/attribute/searchable/attribute_weighted_set_blueprint_test.cpp index 4ec73a1d313..16597b8b615 100644 --- a/searchlib/src/tests/attribute/searchable/attribute_weighted_set_blueprint_test.cpp +++ b/searchlib/src/tests/attribute/searchable/attribute_weighted_set_blueprint_test.cpp @@ -109,7 +109,7 @@ struct WS { FieldSpecList fields; fields.add(FieldSpec(field, fieldId, handle, ac.getAttribute(field)->getIsFilter())); queryeval::Blueprint::UP bp = searchable.createBlueprint(requestContext, fields, *node); - bp->fetchPostings(queryeval::ExecuteInfo::create(strict)); + bp->fetchPostings(queryeval::ExecuteInfo::createForTest(strict)); SearchIterator::UP sb = bp->createSearch(*md, strict); return sb; } @@ -125,7 +125,7 @@ struct WS { FieldSpecList fields; fields.add(FieldSpec(field, fieldId, handle)); queryeval::Blueprint::UP bp = searchable.createBlueprint(requestContext, fields, *node); - bp->fetchPostings(queryeval::ExecuteInfo::create(strict)); + bp->fetchPostings(queryeval::ExecuteInfo::createForTest(strict)); SearchIterator::UP sb = bp->createSearch(*md, strict); FakeResult result; sb->initRange(1, 10); diff --git a/searchlib/src/tests/attribute/searchcontext/searchcontext_test.cpp b/searchlib/src/tests/attribute/searchcontext/searchcontext_test.cpp index 91b6efc26a6..343a5c8e38b 100644 --- a/searchlib/src/tests/attribute/searchcontext/searchcontext_test.cpp +++ b/searchlib/src/tests/attribute/searchcontext/searchcontext_test.cpp @@ -216,7 +216,7 @@ private: // test search iterator unpacking void fillForSearchIteratorUnpackingTest(IntegerAttribute * ia, bool extra); void testSearchIteratorUnpacking(const AttributePtr & ptr, SearchContext & sc, bool extra, bool strict) { - sc.fetchPostings(queryeval::ExecuteInfo::create(strict, 1.0)); + sc.fetchPostings(queryeval::ExecuteInfo::createForTest(strict)); for (bool withElementId : {false, true}) { testSearchIteratorUnpacking(ptr, sc, extra, strict, withElementId); } @@ -649,7 +649,7 @@ public: ~Verifier() override; SearchIterator::UP create(bool strict) const override { - _sc->fetchPostings(queryeval::ExecuteInfo::create(strict, 1.0)); + _sc->fetchPostings(queryeval::ExecuteInfo::createForTest(strict)); return _sc->createIterator(&_dummy, strict); } private: diff --git a/searchlib/src/tests/queryeval/blueprint/intermediate_blueprints_test.cpp b/searchlib/src/tests/queryeval/blueprint/intermediate_blueprints_test.cpp index 7a8250a2e16..a663944938c 100644 --- a/searchlib/src/tests/queryeval/blueprint/intermediate_blueprints_test.cpp +++ b/searchlib/src/tests/queryeval/blueprint/intermediate_blueprints_test.cpp @@ -122,7 +122,7 @@ TEST("test And propagates updated histestimate") { bp.addChild(ap(MyLeafSpec(2000).create<RememberExecuteInfo>()->setSourceId(2))); bp.optimize_self(); bp.setDocIdLimit(5000); - bp.fetchPostings(ExecuteInfo::create(true)); + bp.fetchPostings(ExecuteInfo::TRUE); EXPECT_EQUAL(3u, bp.childCnt()); for (uint32_t i = 0; i < bp.childCnt(); i++) { const RememberExecuteInfo & child = dynamic_cast<const RememberExecuteInfo &>(bp.getChild(i)); diff --git a/searchlib/src/tests/queryeval/dot_product/dot_product_test.cpp b/searchlib/src/tests/queryeval/dot_product/dot_product_test.cpp index 4305f7da116..1538a9ce0df 100644 --- a/searchlib/src/tests/queryeval/dot_product/dot_product_test.cpp +++ b/searchlib/src/tests/queryeval/dot_product/dot_product_test.cpp @@ -73,7 +73,7 @@ struct DP { FieldSpecList fields; fields.add(FieldSpec(field, fieldId, handle, field_is_filter)); queryeval::Blueprint::UP bp = searchable.createBlueprint(requestContext, fields, *node); - bp->fetchPostings(ExecuteInfo::create(strict)); + bp->fetchPostings(ExecuteInfo::createForTest(strict)); SearchIterator::UP sb = bp->createSearch(*md, strict); EXPECT_TRUE(dynamic_cast<DotProductSearch*>(sb.get()) != 0); sb->initFullRange(); diff --git a/searchlib/src/tests/queryeval/equiv/equiv_test.cpp b/searchlib/src/tests/queryeval/equiv/equiv_test.cpp index 9b74f8a650f..c570c06a60b 100644 --- a/searchlib/src/tests/queryeval/equiv/equiv_test.cpp +++ b/searchlib/src/tests/queryeval/equiv/equiv_test.cpp @@ -1,12 +1,14 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/log/log.h> -LOG_SETUP("equiv_test"); + #include <vespa/searchlib/queryeval/leaf_blueprints.h> #include <vespa/searchlib/queryeval/intermediate_blueprints.h> #include <vespa/searchlib/queryeval/equiv_blueprint.h> #include <vespa/searchlib/fef/matchdatalayout.h> #include <vespa/vespalib/gtest/gtest.h> +#include <vespa/log/log.h> +LOG_SETUP("equiv_test"); + using namespace search::queryeval; using search::fef::MatchData; using search::fef::MatchDataLayout; @@ -17,7 +19,7 @@ using search::fef::FieldPositionsIterator; class EquivTest : public ::testing::Test { protected: EquivTest(); - ~EquivTest(); + ~EquivTest() override; void test_equiv(bool strict, bool unpack_normal_features, bool unpack_interleaved_features); }; @@ -57,7 +59,7 @@ EquivTest::test_equiv(bool strict, bool unpack_normal_features, bool unpack_inte data.setNeedNormalFeatures(unpack_normal_features); data.setNeedInterleavedFeatures(unpack_interleaved_features); } - bp->fetchPostings(ExecuteInfo::create(strict)); + bp->fetchPostings(ExecuteInfo::createForTest(strict)); SearchIterator::UP search = bp->createSearch(*md, strict); search->initFullRange(); diff --git a/searchlib/src/tests/queryeval/fake_searchable/fake_searchable_test.cpp b/searchlib/src/tests/queryeval/fake_searchable/fake_searchable_test.cpp index 3bfdcd21ce6..8ddac327643 100644 --- a/searchlib/src/tests/queryeval/fake_searchable/fake_searchable_test.cpp +++ b/searchlib/src/tests/queryeval/fake_searchable/fake_searchable_test.cpp @@ -6,7 +6,6 @@ #include <vespa/searchlib/queryeval/fake_requestcontext.h> #include <vespa/searchlib/queryeval/blueprint.h> #include <vespa/searchlib/query/tree/intermediatenodes.h> -#include <vespa/searchlib/query/tree/termnodes.h> #include <vespa/searchlib/query/tree/simplequery.h> #include <vespa/searchlib/fef/matchdata.h> @@ -63,7 +62,7 @@ TEST_F(FakeSearchableTest, require_that_term_search_works) { bool strict = (i == 0); SCOPED_TRACE(strict ? "strict" : "non-strict"); MatchData::UP md = MatchData::makeTestInstance(100, 10); - bp->fetchPostings(ExecuteInfo::create(strict)); + bp->fetchPostings(ExecuteInfo::createForTest(strict)); SearchIterator::UP search = bp->createSearch(*md, strict); search->initFullRange(); @@ -117,7 +116,7 @@ TEST_F(FakeSearchableTest, require_that_phrase_search_works) { bool strict = (i == 0); SCOPED_TRACE(strict ? "strict" : "non-strict"); MatchData::UP md = MatchData::makeTestInstance(100, 10); - bp->fetchPostings(ExecuteInfo::create(strict)); + bp->fetchPostings(ExecuteInfo::createForTest(strict)); SearchIterator::UP search = bp->createSearch(*md, strict); search->initFullRange(); @@ -168,7 +167,7 @@ TEST_F(FakeSearchableTest, require_that_weigheted_set_search_works) { bool strict = (i == 0); SCOPED_TRACE(strict ? "strict" : "non-strict"); MatchData::UP md = MatchData::makeTestInstance(100, 10); - bp->fetchPostings(ExecuteInfo::create(strict)); + bp->fetchPostings(ExecuteInfo::createForTest(strict)); SearchIterator::UP search = bp->createSearch(*md, strict); search->initFullRange(); @@ -239,7 +238,7 @@ TEST_F(FakeSearchableTest, require_that_multi_field_search_works) { bool strict = (i == 0); SCOPED_TRACE(strict ? "strict" : "non-strict"); MatchData::UP md = MatchData::makeTestInstance(100, 10); - bp->fetchPostings(ExecuteInfo::create(strict)); + bp->fetchPostings(ExecuteInfo::createForTest(strict)); SearchIterator::UP search = bp->createSearch(*md, strict); search->initFullRange(); @@ -323,7 +322,7 @@ TEST_F(FakeSearchableTest, require_that_phrase_with_empty_child_works) { bool strict = (i == 0); SCOPED_TRACE(strict ? "strict" : "non-strict"); MatchData::UP md = MatchData::makeTestInstance(100, 10); - bp->fetchPostings(ExecuteInfo::create(strict)); + bp->fetchPostings(ExecuteInfo::createForTest(strict)); SearchIterator::UP search = bp->createSearch(*md, strict); search->initFullRange(); diff --git a/searchlib/src/tests/queryeval/same_element/same_element_test.cpp b/searchlib/src/tests/queryeval/same_element/same_element_test.cpp index 7c5a4648925..d05e6c8e4f4 100644 --- a/searchlib/src/tests/queryeval/same_element/same_element_test.cpp +++ b/searchlib/src/tests/queryeval/same_element/same_element_test.cpp @@ -6,7 +6,6 @@ #include <vespa/searchlib/queryeval/simpleresult.h> #include <vespa/searchlib/queryeval/same_element_blueprint.h> #include <vespa/searchlib/queryeval/same_element_search.h> -#include <vespa/searchlib/queryeval/emptysearch.h> #include <vespa/searchcommon/attribute/i_search_context.h> #include <vespa/searchlib/attribute/searchcontextelementiterator.h> #include <vespa/vespalib/test/insertion_operators.h> @@ -48,7 +47,7 @@ std::unique_ptr<SameElementBlueprint> make_blueprint(const std::vector<FakeResul Blueprint::UP finalize(Blueprint::UP bp, bool strict) { Blueprint::UP result = Blueprint::optimize(std::move(bp)); - result->fetchPostings(ExecuteInfo::create(strict)); + result->fetchPostings(ExecuteInfo::createForTest(strict)); result->freeze(); return result; } @@ -87,7 +86,7 @@ TEST("require that matching elements can be identified") { auto md = make_match_data(); auto search = bp->createSearch(*md, false); search->initRange(1, 1000); - SameElementSearch *se = dynamic_cast<SameElementSearch*>(search.get()); + auto *se = dynamic_cast<SameElementSearch*>(search.get()); ASSERT_TRUE(se != nullptr); TEST_DO(verify_elements(*se, 5, {3, 7})); TEST_DO(verify_elements(*se, 10, {})); @@ -148,7 +147,7 @@ TEST("require that attribute iterators are wrapped for element unpacking") { auto bp = finalize(make_blueprint({a,b}, true), true); auto md = make_match_data(); auto search = bp->createSearch(*md, false); - SameElementSearch *se = dynamic_cast<SameElementSearch*>(search.get()); + auto *se = dynamic_cast<SameElementSearch*>(search.get()); ASSERT_TRUE(se != nullptr); ASSERT_EQUAL(se->children().size(), 2u); EXPECT_TRUE(dynamic_cast<SearchContextElementIterator*>(se->children()[0].get()) != nullptr); diff --git a/searchlib/src/tests/queryeval/simple_phrase/simple_phrase_test.cpp b/searchlib/src/tests/queryeval/simple_phrase/simple_phrase_test.cpp index 29d4dd2c457..cbe497d6363 100644 --- a/searchlib/src/tests/queryeval/simple_phrase/simple_phrase_test.cpp +++ b/searchlib/src/tests/queryeval/simple_phrase/simple_phrase_test.cpp @@ -36,7 +36,7 @@ struct MyTerm : public search::queryeval::SimpleLeafBlueprint { setEstimate(HitEstimate(hits, (hits == 0))); } SearchIterator::UP createLeafSearch(const search::fef::TermFieldMatchDataArray &, bool) const override { - return SearchIterator::UP(); + return {}; } SearchIteratorUP createFilterSearch(bool strict, FilterConstraint constraint) const override { return create_default_filter(strict, constraint); @@ -143,13 +143,13 @@ public: void fetchPostings(bool useBlueprint) { - ExecuteInfo execInfo = ExecuteInfo::create(_strict); + ExecuteInfo execInfo = ExecuteInfo::createForTest(_strict); if (useBlueprint) { _phrase.fetchPostings(execInfo); return; } - for (size_t i = 0; i < _children.size(); ++i) { - _children[i]->fetchPostings(execInfo); + for (const auto & i : _children) { + i->fetchPostings(execInfo); } } @@ -167,8 +167,8 @@ public: childMatch.add(child_term_field_match_data); } SimplePhraseSearch::Children children; - for (size_t i = 0; i < _children.size(); ++i) { - children.push_back(_children[i]->createSearch(*_md, _strict)); + for (const auto & i : _children) { + children.push_back(i->createSearch(*_md, _strict)); } search = std::make_unique<SimplePhraseSearch>(std::move(children), MatchData::UP(), childMatch, _order, diff --git a/searchlib/src/tests/queryeval/sourceblender/sourceblender_test.cpp b/searchlib/src/tests/queryeval/sourceblender/sourceblender_test.cpp index 77c9e1a8039..bb2e559106f 100644 --- a/searchlib/src/tests/queryeval/sourceblender/sourceblender_test.cpp +++ b/searchlib/src/tests/queryeval/sourceblender/sourceblender_test.cpp @@ -74,7 +74,7 @@ TEST("test strictness") { blend_b->addChild(std::move(a_b)); blend_b->addChild(std::move(b_b)); Blueprint::UP bp(blend_b); - bp->fetchPostings(ExecuteInfo::create(strict)); + bp->fetchPostings(ExecuteInfo::createForTest(strict)); SearchIterator::UP search = bp->createSearch(*md, strict); search->initFullRange(); SearchIterator &blend = *search; diff --git a/searchlib/src/tests/queryeval/weighted_set_term/weighted_set_term_test.cpp b/searchlib/src/tests/queryeval/weighted_set_term/weighted_set_term_test.cpp index 2f3f0eb7392..fed4e7c9bbd 100644 --- a/searchlib/src/tests/queryeval/weighted_set_term/weighted_set_term_test.cpp +++ b/searchlib/src/tests/queryeval/weighted_set_term/weighted_set_term_test.cpp @@ -68,7 +68,7 @@ struct WS { FieldSpecList fields; fields.add(FieldSpec(field, fieldId, handle)); auto bp = searchable.createBlueprint(requestContext, fields, *node); - bp->fetchPostings(ExecuteInfo::create(strict)); + bp->fetchPostings(ExecuteInfo::createForTest(strict)); auto sb = bp->createSearch(*md, strict); return (dynamic_cast<WeightedSetTermSearch*>(sb.get()) != nullptr); } @@ -83,7 +83,7 @@ struct WS { FieldSpecList fields; fields.add(FieldSpec(field, fieldId, handle, field_is_filter)); auto bp = searchable.createBlueprint(requestContext, fields, *node); - bp->fetchPostings(ExecuteInfo::create(strict)); + bp->fetchPostings(ExecuteInfo::createForTest(strict)); auto sb = bp->createSearch(*md, strict); sb->initFullRange(); FakeResult result; diff --git a/searchlib/src/vespa/searchlib/attribute/dfa_fuzzy_matcher.h b/searchlib/src/vespa/searchlib/attribute/dfa_fuzzy_matcher.h index a6467cfc91d..9de77085fae 100644 --- a/searchlib/src/vespa/searchlib/attribute/dfa_fuzzy_matcher.h +++ b/searchlib/src/vespa/searchlib/attribute/dfa_fuzzy_matcher.h @@ -62,7 +62,7 @@ public: return true; } } - DfaStringComparator cmp(data_store, _successor); + DfaStringComparator cmp(data_store, _successor, _cased); itr.seek(vespalib::datastore::AtomicEntryRef(), cmp); return false; } diff --git a/searchlib/src/vespa/searchlib/attribute/dfa_string_comparator.cpp b/searchlib/src/vespa/searchlib/attribute/dfa_string_comparator.cpp index 762ad1e9042..9204510c0d2 100644 --- a/searchlib/src/vespa/searchlib/attribute/dfa_string_comparator.cpp +++ b/searchlib/src/vespa/searchlib/attribute/dfa_string_comparator.cpp @@ -5,9 +5,10 @@ namespace search::attribute { -DfaStringComparator::DfaStringComparator(const DataStoreType& data_store, const std::vector<uint32_t>& candidate) +DfaStringComparator::DfaStringComparator(const DataStoreType& data_store, const std::vector<uint32_t>& candidate, bool cased) : ParentType(data_store), - _candidate(std::cref(candidate)) + _candidate(std::cref(candidate)), + _cased(cased) { } @@ -16,13 +17,25 @@ DfaStringComparator::less(const vespalib::datastore::EntryRef lhs, const vespali { if (lhs.valid()) { if (rhs.valid()) { - return FoldedStringCompare::compareFolded<true, true>(get(lhs), get(rhs)) < 0; + if (_cased) { + return FoldedStringCompare::compareFolded<false, false>(get(lhs), get(rhs)) < 0; + } else { + return FoldedStringCompare::compareFolded<true, true>(get(lhs), get(rhs)) < 0; + } } else { - return FoldedStringCompare::compareFolded<true, false>(get(lhs), _candidate) < 0; + if (_cased) { + return FoldedStringCompare::compareFolded<false, false>(get(lhs), _candidate) < 0; + } else { + return FoldedStringCompare::compareFolded<true, false>(get(lhs), _candidate) < 0; + } } } else { if (rhs.valid()) { - return FoldedStringCompare::compareFolded<false, true>(_candidate, get(rhs)) < 0; + if (_cased) { + return FoldedStringCompare::compareFolded<false, false>(_candidate, get(rhs)) < 0; + } else { + return FoldedStringCompare::compareFolded<false, true>(_candidate, get(rhs)) < 0; + } } else { return false; } diff --git a/searchlib/src/vespa/searchlib/attribute/dfa_string_comparator.h b/searchlib/src/vespa/searchlib/attribute/dfa_string_comparator.h index f9eaf281ff7..13a337f565c 100644 --- a/searchlib/src/vespa/searchlib/attribute/dfa_string_comparator.h +++ b/searchlib/src/vespa/searchlib/attribute/dfa_string_comparator.h @@ -26,9 +26,10 @@ public: private: using ParentType::get; std::reference_wrapper<const std::vector<uint32_t>> _candidate; + bool _cased; public: - DfaStringComparator(const DataStoreType& data_store, const std::vector<uint32_t>& candidate); + DfaStringComparator(const DataStoreType& data_store, const std::vector<uint32_t>& candidate, bool cased); bool less(const vespalib::datastore::EntryRef lhs, const vespalib::datastore::EntryRef rhs) const override; }; diff --git a/searchlib/src/vespa/searchlib/queryeval/executeinfo.h b/searchlib/src/vespa/searchlib/queryeval/executeinfo.h index 9e42bb4da8e..362b0826f67 100644 --- a/searchlib/src/vespa/searchlib/queryeval/executeinfo.h +++ b/searchlib/src/vespa/searchlib/queryeval/executeinfo.h @@ -22,16 +22,13 @@ public: static ExecuteInfo create(bool strict, const ExecuteInfo & org) noexcept { return {strict, org._hitRate, org.getDoom()}; } - static ExecuteInfo create(bool strict, const vespalib::Doom * doom) noexcept { - return create(strict, 1.0F, doom); - } static ExecuteInfo create(bool strict, float hitRate, const vespalib::Doom * doom) noexcept { return {strict, hitRate, doom}; } - static ExecuteInfo create(bool strict) noexcept { - return create(strict, 1.0F); + static ExecuteInfo createForTest(bool strict) noexcept { + return createForTest(strict, 1.0F); } - static ExecuteInfo create(bool strict, float hitRate) noexcept { + static ExecuteInfo createForTest(bool strict, float hitRate) noexcept { return create(strict, hitRate, nullptr); } private: diff --git a/searchlib/src/vespa/searchlib/queryeval/same_element_blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/same_element_blueprint.cpp index 4674aee8314..6b2faac847e 100644 --- a/searchlib/src/vespa/searchlib/queryeval/same_element_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/same_element_blueprint.cpp @@ -56,8 +56,13 @@ SameElementBlueprint::optimize_self() void SameElementBlueprint::fetchPostings(const ExecuteInfo &execInfo) { - for (size_t i = 0; i < _terms.size(); ++i) { - _terms[i]->fetchPostings(ExecuteInfo::create(execInfo.isStrict() && (i == 0), execInfo)); + if (_terms.empty()) return; + _terms[0]->fetchPostings(execInfo); + double hit_rate = execInfo.hitRate() * _terms[0]->hit_ratio(); + for (size_t i = 1; i < _terms.size(); ++i) { + Blueprint & term = *_terms[i]; + term.fetchPostings(ExecuteInfo::create(false, hit_rate, execInfo.getDoom())); + hit_rate = hit_rate * term.hit_ratio(); } } diff --git a/vespajlib/src/main/java/com/yahoo/slime/SlimeUtils.java b/vespajlib/src/main/java/com/yahoo/slime/SlimeUtils.java index 56ef0c76408..11299814274 100644 --- a/vespajlib/src/main/java/com/yahoo/slime/SlimeUtils.java +++ b/vespajlib/src/main/java/com/yahoo/slime/SlimeUtils.java @@ -131,7 +131,7 @@ public class SlimeUtils { } public static Optional<String> optionalString(Inspector inspector) { - return Optional.of(inspector).filter(SlimeUtils::isPresent).map(Inspector::asString); + return isPresent(inspector) ? Optional.of(inspector.asString()) : Optional.empty(); } public static OptionalLong optionalLong(Inspector field) { @@ -147,11 +147,11 @@ public class SlimeUtils { } public static Optional<Instant> optionalInstant(Inspector field) { - return optionalLong(field).stream().mapToObj(Instant::ofEpochMilli).findFirst(); + return isPresent(field) ? Optional.of(Instant.ofEpochMilli(field.asLong())) : Optional.empty(); } public static Optional<Duration> optionalDuration(Inspector field) { - return optionalLong(field).stream().mapToObj(Duration::ofMillis).findFirst(); + return isPresent(field) ? Optional.of(Duration.ofMillis(field.asLong())) : Optional.empty(); } public static Iterator<Inspector> entriesIterator(Inspector inspector) { @@ -165,8 +165,9 @@ public class SlimeUtils { /** Returns stream of entries for given inspector. If the inspector is not an array, empty stream is returned */ public static Stream<Inspector> entriesStream(Inspector inspector) { int characteristics = Spliterator.NONNULL | Spliterator.SIZED | Spliterator.ORDERED; - return StreamSupport.stream(Spliterators.spliteratorUnknownSize(entriesIterator(inspector), - characteristics), + return StreamSupport.stream(Spliterators.spliterator(entriesIterator(inspector), + inspector.entries(), + characteristics), false); } |