diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2020-06-07 23:16:54 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2020-06-07 23:16:54 +0200 |
commit | 8b85b62224800ef740ea80c223e5e7c2baadadae (patch) | |
tree | 0456ec2cec34c515d681934e803d9ab075348ebc | |
parent | 633d97a8c892bbff4cb1c8bb58c5797435dd2ee0 (diff) | |
parent | c6696dd4623ae421d557558fe8ac24e31dffd621 (diff) |
Merge branch 'master' into balder/fetch-and-merge-a-cacheline
149 files changed, 2300 insertions, 2535 deletions
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index ee83c9fa804..82fbfe87de3 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -178,7 +178,7 @@ public abstract class ContainerCluster<CONTAINER extends Container> addComponent(new StatisticsComponent()); addSimpleComponent(AccessLog.class); - addSimpleComponent(ThreadPoolProvider.class); + addComponent(new ThreadPoolExecutorComponent.Builder("default-pool").build()); addSimpleComponent(com.yahoo.concurrent.classlock.ClassLocking.class); addSimpleComponent(SecurityFilterInvoker.class); addSimpleComponent("com.yahoo.container.jdisc.metric.MetricConsumerProviderProvider"); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ThreadPoolExecutorComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ThreadPoolExecutorComponent.java new file mode 100644 index 00000000000..2926cb3ee6c --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ThreadPoolExecutorComponent.java @@ -0,0 +1,70 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container; + +import com.yahoo.container.bundle.BundleInstantiationSpecification; +import com.yahoo.container.handler.ThreadPoolProvider; +import com.yahoo.container.handler.ThreadpoolConfig; +import com.yahoo.osgi.provider.model.ComponentModel; +import com.yahoo.vespa.model.container.component.SimpleComponent; + +import java.time.Duration; + +/** + * Component definition for a {@link java.util.concurrent.Executor} using {@link ThreadPoolProvider}. + * + * @author bjorncs + */ +public class ThreadPoolExecutorComponent extends SimpleComponent implements ThreadpoolConfig.Producer { + + private final String name; + private final Integer maxPoolSize; + private final Integer corePoolSize; + private final Duration keepAliveTime; + private final Integer queueSize; + private final Duration maxThreadExecutionTime; + + private ThreadPoolExecutorComponent(Builder builder) { + super(new ComponentModel( + BundleInstantiationSpecification.getFromStrings( + "threadpool-provider@" + builder.name, + ThreadPoolProvider.class.getName(), + null))); + this.name = builder.name; + this.maxPoolSize = builder.maxPoolSize; + this.corePoolSize = builder.corePoolSize; + this.keepAliveTime = builder.keepAliveTime; + this.queueSize = builder.queueSize; + this.maxThreadExecutionTime = builder.maxThreadExecutionTime; + } + + @Override + public void getConfig(ThreadpoolConfig.Builder builder) { + builder.name(this.name); + if (maxPoolSize != null) builder.maxthreads(maxPoolSize); + if (corePoolSize != null) builder.corePoolSize(corePoolSize); + if (keepAliveTime != null) builder.keepAliveTime(keepAliveTime.toMillis() / 1000D); + if (queueSize != null) builder.queueSize(queueSize); + if (maxThreadExecutionTime != null) builder.maxThreadExecutionTimeSeconds((int)maxThreadExecutionTime.toMillis() / 1000); + } + + public static class Builder { + + private final String name; + private Integer maxPoolSize; + private Integer corePoolSize; + private Duration keepAliveTime; + private Integer queueSize; + private Duration maxThreadExecutionTime; + + public Builder(String name) { this.name = name; } + + public Builder maxPoolSize(int size) { this.maxPoolSize = size; return this; } + public Builder corePoolSize(int size) { this.corePoolSize = size; return this; } + public Builder keepAliveTime(Duration time) { this.keepAliveTime = time; return this; } + public Builder queueSize(int size) { this.queueSize = size; return this; } + public Builder maxThreadExecutionTime(Duration time) { this.maxThreadExecutionTime = time; return this; } + + public ThreadPoolExecutorComponent build() { return new ThreadPoolExecutorComponent(this); } + + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 9e8010cf8a7..9404bc9c279 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -223,10 +223,6 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye } } - public PrepareResult deploy(CompressedApplicationInputStream in, PrepareParams prepareParams) { - return deploy(in, prepareParams, false, clock.instant()); - } - public PrepareResult deploy(CompressedApplicationInputStream in, PrepareParams prepareParams, boolean ignoreSessionStaleFailure, Instant now) { File tempDir = uncheck(() -> Files.createTempDirectory("deploy")).toFile(); @@ -664,7 +660,9 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye public long createSession(ApplicationId applicationId, TimeoutBudget timeoutBudget, File applicationDirectory) { Tenant tenant = tenantRepository.getTenant(applicationId.tenant()); tenant.getApplicationRepo().createApplication(applicationId); - LocalSession session = tenant.getSessionFactory().createSession(applicationDirectory, applicationId, timeoutBudget); + Optional<Long> activeSessionId = tenant.getApplicationRepo().activeSessionOf(applicationId); + LocalSession session = tenant.getSessionFactory().createSession(applicationDirectory, applicationId, + timeoutBudget, activeSessionId); tenant.getLocalSessionRepo().addSession(session); return session.getSessionId(); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelController.java b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelController.java index 42c80acd80d..657e113475b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelController.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelController.java @@ -16,7 +16,6 @@ import com.yahoo.vespa.config.protocol.DefContent; import com.yahoo.vespa.config.server.model.SuperModelConfigProvider; import com.yahoo.vespa.config.server.rpc.ConfigResponseFactory; -import java.io.IOException; import java.io.StringReader; /** @@ -74,7 +73,7 @@ public class SuperModelController { long getGeneration() { return generation; } - public <CONFIGTYPE extends ConfigInstance> CONFIGTYPE getConfig(Class<CONFIGTYPE> configClass, ApplicationId applicationId, String configId) throws IOException { + public <CONFIGTYPE extends ConfigInstance> CONFIGTYPE getConfig(Class<CONFIGTYPE> configClass, ApplicationId applicationId, String configId) { return model.getConfig(configClass, applicationId, configId); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelGenerationCounter.java b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelGenerationCounter.java index 6adbcc8dae9..6d5c4a81c92 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelGenerationCounter.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelGenerationCounter.java @@ -2,11 +2,9 @@ package com.yahoo.vespa.config.server; import com.yahoo.path.Path; -import com.yahoo.transaction.AbstractTransaction; -import com.yahoo.transaction.Transaction; import com.yahoo.vespa.config.GenerationCounter; -import com.yahoo.vespa.curator.recipes.CuratorCounter; import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.curator.recipes.CuratorCounter; /** * Distributed global generation counter for the super model. diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelRequestHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelRequestHandler.java index 6fcfde80510..aa06c07f8af 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelRequestHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelRequestHandler.java @@ -3,21 +3,20 @@ package com.yahoo.vespa.config.server; import com.google.inject.Inject; import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.component.Version; import com.yahoo.config.ConfigInstance; import com.yahoo.config.FileReference; import com.yahoo.config.model.api.ConfigDefinitionRepo; -import com.yahoo.component.Version; -import java.util.logging.Level; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.GetConfigRequest; import com.yahoo.vespa.config.protocol.ConfigResponse; -import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.config.server.application.ApplicationSet; import com.yahoo.vespa.config.server.rpc.ConfigResponseFactory; -import java.io.IOException; import java.util.Optional; import java.util.Set; +import java.util.logging.Level; /** * Handles request for supermodel config. @@ -84,7 +83,7 @@ public class SuperModelRequestHandler implements RequestHandler { return null; } - public <CONFIGTYPE extends ConfigInstance> CONFIGTYPE getConfig(Class<CONFIGTYPE> configClass, ApplicationId applicationId, String configId) throws IOException { + public <CONFIGTYPE extends ConfigInstance> CONFIGTYPE getConfig(Class<CONFIGTYPE> configClass, ApplicationId applicationId, String configId) { return handler.getConfig(configClass, applicationId, configId); } @@ -128,4 +127,5 @@ public class SuperModelRequestHandler implements RequestHandler { superModelManager.markAsComplete(); updateHandler(); } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java index 4f04724a0a8..22de36e98aa 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java @@ -1,16 +1,28 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.application; +import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.component.Version; import com.yahoo.concurrent.StripedExecutor; +import com.yahoo.config.FileReference; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; -import java.util.logging.Level; import com.yahoo.path.Path; import com.yahoo.text.Utf8; import com.yahoo.transaction.Transaction; +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.GetConfigRequest; +import com.yahoo.vespa.config.protocol.ConfigResponse; import com.yahoo.vespa.config.server.GlobalComponentRegistry; import com.yahoo.vespa.config.server.NotFoundException; import com.yahoo.vespa.config.server.ReloadHandler; +import com.yahoo.vespa.config.server.ReloadListener; +import com.yahoo.vespa.config.server.RequestHandler; +import com.yahoo.vespa.config.server.host.HostRegistry; +import com.yahoo.vespa.config.server.host.HostValidator; +import com.yahoo.vespa.config.server.monitoring.MetricUpdater; +import com.yahoo.vespa.config.server.monitoring.Metrics; +import com.yahoo.vespa.config.server.rpc.ConfigResponseFactory; import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.Lock; @@ -19,15 +31,21 @@ import com.yahoo.vespa.curator.transaction.CuratorTransaction; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; +import java.time.Clock; import java.time.Duration; +import java.util.Collection; +import java.util.LinkedHashSet; import java.util.List; import java.util.Optional; import java.util.Set; import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; +import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; +import static java.util.stream.Collectors.toSet; + /** * The applications of a tenant, backed by ZooKeeper. * @@ -38,7 +56,7 @@ import java.util.stream.Collectors; * @author Ulf Lilleengen * @author jonmv */ -public class TenantApplications { +public class TenantApplications implements RequestHandler, ReloadHandler, HostValidator<ApplicationId> { private static final Logger log = Logger.getLogger(TenantApplications.class.getName()); @@ -46,24 +64,44 @@ public class TenantApplications { private final Path applicationsPath; private final Path locksPath; private final Curator.DirectoryCache directoryCache; - private final ReloadHandler reloadHandler; private final Executor zkWatcherExecutor; + private final Metrics metrics; + private final TenantName tenant; + private final ReloadListener reloadListener; + private final ConfigResponseFactory responseFactory; + private final HostRegistry<ApplicationId> hostRegistry; + private final ApplicationMapper applicationMapper = new ApplicationMapper(); + private final MetricUpdater tenantMetricUpdater; + private final Clock clock = Clock.systemUTC(); - private TenantApplications(Curator curator, ReloadHandler reloadHandler, TenantName tenant, - ExecutorService zkCacheExecutor, StripedExecutor<TenantName> zkWatcherExecutor) { + public TenantApplications(TenantName tenant, Curator curator, StripedExecutor<TenantName> zkWatcherExecutor, + ExecutorService zkCacheExecutor, Metrics metrics, ReloadListener reloadListener, + ConfigserverConfig configserverConfig, HostRegistry<ApplicationId> hostRegistry) { this.curator = curator; this.applicationsPath = TenantRepository.getApplicationsPath(tenant); this.locksPath = TenantRepository.getLocksPath(tenant); - this.reloadHandler = reloadHandler; + this.tenant = tenant; this.zkWatcherExecutor = command -> zkWatcherExecutor.execute(tenant, command); this.directoryCache = curator.createDirectoryCache(applicationsPath.getAbsolute(), false, false, zkCacheExecutor); this.directoryCache.start(); this.directoryCache.addListener(this::childEvent); + this.metrics = metrics; + this.reloadListener = reloadListener; + this.responseFactory = ConfigResponseFactory.create(configserverConfig); + this.tenantMetricUpdater = metrics.getOrCreateMetricUpdater(Metrics.createDimensions(tenant)); + this.hostRegistry = hostRegistry; } - public static TenantApplications create(GlobalComponentRegistry registry, ReloadHandler reloadHandler, TenantName tenant) { - return new TenantApplications(registry.getCurator(), reloadHandler, tenant, - registry.getZkCacheExecutor(), registry.getZkWatcherExecutor()); + // For testing only + public static TenantApplications create(GlobalComponentRegistry componentRegistry, TenantName tenantName) { + return new TenantApplications(tenantName, + componentRegistry.getCurator(), + componentRegistry.getZkWatcherExecutor(), + componentRegistry.getZkCacheExecutor(), + componentRegistry.getMetrics(), + componentRegistry.getReloadListener(), + componentRegistry.getConfigserverConfig(), + componentRegistry.getHostRegistries().createApplicationHostRegistry(tenantName)); } /** @@ -132,7 +170,7 @@ public class TenantApplications { * Removes all applications not known to this from the config server state. */ public void removeUnusedApplications() { - reloadHandler.removeApplicationsExcept(Set.copyOf(activeApplications())); + removeApplicationsExcept(Set.copyOf(activeApplications())); } /** @@ -170,7 +208,7 @@ public class TenantApplications { } private void applicationRemoved(ApplicationId applicationId) { - reloadHandler.removeApplication(applicationId); + removeApplication(applicationId); log.log(Level.INFO, TenantRepository.logPre(applicationId) + "Application removed: " + applicationId); } @@ -186,4 +224,195 @@ public class TenantApplications { return locksPath.append(id.serializedForm()); } + + /** + * Gets a config for the given app, or null if not found + */ + @Override + public ConfigResponse resolveConfig(ApplicationId appId, GetConfigRequest req, Optional<Version> vespaVersion) { + Application application = getApplication(appId, vespaVersion); + if (log.isLoggable(Level.FINE)) { + log.log(Level.FINE, TenantRepository.logPre(appId) + "Resolving for tenant '" + tenant + "' with handler for application '" + application + "'"); + } + return application.resolveConfig(req, responseFactory); + } + + // For testing only + long getApplicationGeneration(ApplicationId appId, Optional<Version> vespaVersion) { + Application application = getApplication(appId, vespaVersion); + return application.getApplicationGeneration(); + } + + private void notifyReloadListeners(ApplicationSet applicationSet) { + reloadListener.hostsUpdated(tenant, hostRegistry.getAllHosts()); + reloadListener.configActivated(applicationSet); + } + + /** + * Activates the config of the given app. Notifies listeners + * + * @param applicationSet the {@link ApplicationSet} to be reloaded + */ + @Override + public void reloadConfig(ApplicationSet applicationSet) { + ApplicationId id = applicationSet.getId(); + try (Lock lock = lock(id)) { + if ( ! exists(id)) + return; // Application was deleted before activation. + if (applicationSet.getApplicationGeneration() != requireActiveSessionOf(id)) + return; // Application activated a new session before we got here. + + setLiveApp(applicationSet); + notifyReloadListeners(applicationSet); + } + } + + @Override + public void removeApplication(ApplicationId applicationId) { + try (Lock lock = lock(applicationId)) { + if (exists(applicationId)) + return; // Application was deployed again. + + if (applicationMapper.hasApplication(applicationId, clock.instant())) { + applicationMapper.remove(applicationId); + hostRegistry.removeHostsForKey(applicationId); + reloadListenersOnRemove(applicationId); + tenantMetricUpdater.setApplications(applicationMapper.numApplications()); + metrics.removeMetricUpdater(Metrics.createDimensions(applicationId)); + } + } + } + + @Override + public void removeApplicationsExcept(Set<ApplicationId> applications) { + for (ApplicationId activeApplication : applicationMapper.listApplicationIds()) { + if ( ! applications.contains(activeApplication)) { + log.log(Level.INFO, "Will remove deleted application " + activeApplication.toShortString()); + removeApplication(activeApplication); + } + } + } + + private void reloadListenersOnRemove(ApplicationId applicationId) { + reloadListener.hostsUpdated(tenant, hostRegistry.getAllHosts()); + reloadListener.applicationRemoved(applicationId); + } + + private void setLiveApp(ApplicationSet applicationSet) { + ApplicationId id = applicationSet.getId(); + Collection<String> hostsForApp = applicationSet.getAllHosts(); + hostRegistry.update(id, hostsForApp); + applicationSet.updateHostMetrics(); + tenantMetricUpdater.setApplications(applicationMapper.numApplications()); + applicationMapper.register(id, applicationSet); + } + + @Override + public Set<ConfigKey<?>> listNamedConfigs(ApplicationId appId, Optional<Version> vespaVersion, ConfigKey<?> keyToMatch, boolean recursive) { + Application application = getApplication(appId, vespaVersion); + return listConfigs(application, keyToMatch, recursive); + } + + private Set<ConfigKey<?>> listConfigs(Application application, ConfigKey<?> keyToMatch, boolean recursive) { + Set<ConfigKey<?>> ret = new LinkedHashSet<>(); + for (ConfigKey<?> key : application.allConfigsProduced()) { + String configId = key.getConfigId(); + if (recursive) { + key = new ConfigKey<>(key.getName(), configId, key.getNamespace()); + } else { + // Include first part of id as id + key = new ConfigKey<>(key.getName(), configId.split("/")[0], key.getNamespace()); + } + if (keyToMatch != null) { + String n = key.getName(); // Never null + String ns = key.getNamespace(); // Never null + if (n.equals(keyToMatch.getName()) && + ns.equals(keyToMatch.getNamespace()) && + configId.startsWith(keyToMatch.getConfigId()) && + !(configId.equals(keyToMatch.getConfigId()))) { + + if (!recursive) { + // For non-recursive, include the id segment we were searching for, and first part of the rest + key = new ConfigKey<>(key.getName(), appendOneLevelOfId(keyToMatch.getConfigId(), configId), key.getNamespace()); + } + ret.add(key); + } + } else { + ret.add(key); + } + } + return ret; + } + + @Override + public Set<ConfigKey<?>> listConfigs(ApplicationId appId, Optional<Version> vespaVersion, boolean recursive) { + Application application = getApplication(appId, vespaVersion); + return listConfigs(application, null, recursive); + } + + /** + * Given baseIdSegment search/ and id search/qrservers/default.0, return search/qrservers + * @return id segment with one extra level from the id appended + */ + String appendOneLevelOfId(String baseIdSegment, String id) { + if ("".equals(baseIdSegment)) return id.split("/")[0]; + String theRest = id.substring(baseIdSegment.length()); + if ("".equals(theRest)) return id; + theRest = theRest.replaceFirst("/", ""); + String theRestFirstSeg = theRest.split("/")[0]; + return baseIdSegment+"/"+theRestFirstSeg; + } + + @Override + public Set<ConfigKey<?>> allConfigsProduced(ApplicationId appId, Optional<Version> vespaVersion) { + Application application = getApplication(appId, vespaVersion); + return application.allConfigsProduced(); + } + + private Application getApplication(ApplicationId appId, Optional<Version> vespaVersion) { + try { + return applicationMapper.getForVersion(appId, vespaVersion, clock.instant()); + } catch (VersionDoesNotExistException ex) { + throw new NotFoundException(String.format("%sNo such application (id %s): %s", TenantRepository.logPre(tenant), appId, ex.getMessage())); + } + } + + @Override + public Set<String> allConfigIds(ApplicationId appId, Optional<Version> vespaVersion) { + Application application = getApplication(appId, vespaVersion); + return application.allConfigIds(); + } + + @Override + public boolean hasApplication(ApplicationId appId, Optional<Version> vespaVersion) { + return hasHandler(appId, vespaVersion); + } + + private boolean hasHandler(ApplicationId appId, Optional<Version> vespaVersion) { + return applicationMapper.hasApplicationForVersion(appId, vespaVersion, clock.instant()); + } + + @Override + public ApplicationId resolveApplicationId(String hostName) { + ApplicationId applicationId = hostRegistry.getKeyForHost(hostName); + if (applicationId == null) { + applicationId = ApplicationId.defaultId(); + } + return applicationId; + } + + @Override + public Set<FileReference> listFileReferences(ApplicationId applicationId) { + return applicationMapper.listApplications(applicationId).stream() + .flatMap(app -> app.getModel().fileReferences().stream()) + .collect(toSet()); + } + + @Override + public void verifyHosts(ApplicationId key, Collection<String> newHosts) { + hostRegistry.verifyHosts(key, newHosts); + reloadListener.verifyHostsAreAvailable(tenant, newHosts); + } + + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java index e2cf84d6715..7e83d7013e0 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java @@ -21,7 +21,6 @@ import com.yahoo.vespa.config.server.zookeeper.ZKApplicationPackage; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; -import java.io.Reader; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HostHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HostHandler.java index ffe3d39b524..2c888df6658 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HostHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HostHandler.java @@ -9,13 +9,14 @@ import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.jdisc.Response; import com.yahoo.jdisc.application.BindingMatch; -import java.util.logging.Level; import com.yahoo.vespa.config.server.GlobalComponentRegistry; import com.yahoo.vespa.config.server.host.HostRegistries; import com.yahoo.vespa.config.server.host.HostRegistry; -import com.yahoo.vespa.config.server.http.*; +import com.yahoo.vespa.config.server.http.HttpErrorResponse; +import com.yahoo.vespa.config.server.http.HttpHandler; +import com.yahoo.vespa.config.server.http.JSONResponse; -import java.util.concurrent.Executor; +import java.util.logging.Level; /** diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java index 7828ce8963f..6b42ca7fa95 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java @@ -36,7 +36,6 @@ import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.flags.FlagSource; import java.net.URI; -import java.time.Instant; import java.util.Map; import java.util.Optional; import java.util.logging.Level; @@ -88,8 +87,8 @@ public class ActivatedModelsBuilder extends ModelsBuilder<Application> { ApplicationId applicationId, Optional<DockerImage> wantedDockerImageRepository, Version wantedNodeVespaVersion, - Optional<AllocatedHosts> ignored, // Ignored since we have this in the app package for activated models - Instant now) { + Optional<AllocatedHosts> ignored // Ignored since we have this in the app package for activated models + ) { log.log(Level.FINE, String.format("Loading model version %s for session %s application %s", modelFactory.version(), appGeneration, applicationId)); ModelContext.Properties modelContextProperties = createModelContextProperties(applicationId); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java index 455731d9cb6..245b9db020b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java @@ -166,8 +166,7 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> { applicationId, wantedDockerImageRepository, wantedNodeVespaVersion, - allocatedHosts.asOptional(), - now); + allocatedHosts.asOptional()); allocatedHosts.set(latestModelVersion.getModel().allocatedHosts()); // Update with additional clusters allocated allApplicationVersions.add(latestModelVersion); } @@ -189,8 +188,7 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> { applicationId, wantedDockerImageRepository, wantedNodeVespaVersion, - allocatedHosts.asOptional(), - now); + allocatedHosts.asOptional()); allocatedHosts.set(modelVersion.getModel().allocatedHosts()); // Update with additional clusters allocated allApplicationVersions.add(modelVersion); } catch (RuntimeException e) { @@ -245,8 +243,7 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> { protected abstract MODELRESULT buildModelVersion(ModelFactory modelFactory, ApplicationPackage applicationPackage, ApplicationId applicationId, Optional<DockerImage> dockerImageRepository, - Version wantedNodeVespaVersion, Optional<AllocatedHosts> allocatedHosts, - Instant now); + Version wantedNodeVespaVersion, Optional<AllocatedHosts> allocatedHosts); /** * Returns a host provisioner returning the previously allocated hosts if available and when on hosted Vespa, diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java index e6d275fccdd..2397dba6b5e 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java @@ -32,7 +32,6 @@ import com.yahoo.vespa.config.server.session.PrepareParams; import java.io.File; import java.io.IOException; -import java.time.Instant; import java.util.List; import java.util.Optional; import java.util.logging.Level; @@ -78,13 +77,12 @@ public class PreparedModelsBuilder extends ModelsBuilder<PreparedModelsBuilder.P } @Override - protected PreparedModelResult buildModelVersion(ModelFactory modelFactory, + protected PreparedModelResult buildModelVersion(ModelFactory modelFactory, ApplicationPackage applicationPackage, ApplicationId applicationId, Optional<DockerImage> wantedDockerImageRepository, Version wantedNodeVespaVersion, - Optional<AllocatedHosts> allocatedHosts, - Instant now) { + Optional<AllocatedHosts> allocatedHosts) { Version modelVersion = modelFactory.version(); log.log(Level.FINE, "Building model " + modelVersion + " for " + applicationId); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/monitoring/ZKMetricUpdater.java b/configserver/src/main/java/com/yahoo/vespa/config/server/monitoring/ZKMetricUpdater.java index 62da6fcffbe..52ca73c68b9 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/monitoring/ZKMetricUpdater.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/monitoring/ZKMetricUpdater.java @@ -14,8 +14,6 @@ import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.Map; import java.util.Optional; -import java.util.Timer; -import java.util.TimerTask; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; import java.util.concurrent.ScheduledExecutorService; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/provision/HostProvisionerProvider.java b/configserver/src/main/java/com/yahoo/vespa/config/server/provision/HostProvisionerProvider.java index eb14947f73f..13c21a065ff 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/provision/HostProvisionerProvider.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/provision/HostProvisionerProvider.java @@ -5,10 +5,8 @@ import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.ComponentId; import com.yahoo.component.provider.ComponentRegistry; import com.yahoo.config.provision.Provisioner; -import java.util.logging.Level; import java.util.Optional; -import java.util.logging.Logger; /** * This class is necessary to support both having and not having a host provisioner. We inject diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/provision/ProvisionerAdapter.java b/configserver/src/main/java/com/yahoo/vespa/config/server/provision/ProvisionerAdapter.java index 307ec5c0c3c..6a681ae143d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/provision/ProvisionerAdapter.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/provision/ProvisionerAdapter.java @@ -9,7 +9,6 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.ProvisionLogger; import com.yahoo.config.provision.Provisioner; -import com.yahoo.config.provision.NetworkPorts; import java.util.*; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java index 7809000695a..62f7d3ce5d0 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java @@ -3,10 +3,9 @@ package com.yahoo.vespa.config.server.rpc; import com.yahoo.cloud.config.SentinelConfig; import com.yahoo.collections.Pair; -import com.yahoo.config.provision.TenantName; import com.yahoo.component.Version; +import com.yahoo.config.provision.TenantName; import com.yahoo.jrt.Request; -import java.util.logging.Level; import com.yahoo.net.HostName; import com.yahoo.vespa.config.ConfigPayload; import com.yahoo.vespa.config.ErrorCode; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/DefaultRpcAuthorizerProvider.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/DefaultRpcAuthorizerProvider.java index 5c760f0a25a..8d1d4f58e37 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/DefaultRpcAuthorizerProvider.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/DefaultRpcAuthorizerProvider.java @@ -8,8 +8,6 @@ import com.yahoo.container.di.componentgraph.Provider; import com.yahoo.security.tls.TransportSecurityUtils; import com.yahoo.vespa.config.server.host.HostRegistries; import com.yahoo.vespa.config.server.rpc.RequestHandlerProvider; -import com.yahoo.vespa.flags.FlagSource; -import com.yahoo.vespa.flags.Flags; /** * A provider for {@link RpcAuthorizer}. The instance provided is dependent on the configuration of the configserver. diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionRepo.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionRepo.java index b6a9c8c0854..e23552dee44 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionRepo.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionRepo.java @@ -10,8 +10,6 @@ import com.yahoo.vespa.config.server.deploy.TenantFileSystemDirs; import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; import com.yahoo.vespa.curator.Curator; -import com.yahoo.vespa.flags.Flags; -import com.yahoo.vespa.flags.LongFlag; import java.io.File; import java.io.FilenameFilter; @@ -44,7 +42,6 @@ public class LocalSessionRepo { private final Curator curator; private final Executor zkWatcherExecutor; private final TenantFileSystemDirs tenantFileSystemDirs; - private final LongFlag expiryTimeFlag; public LocalSessionRepo(TenantName tenantName, GlobalComponentRegistry componentRegistry, SessionFactory sessionFactory) { sessionCache = new SessionCache<>(); @@ -53,7 +50,6 @@ public class LocalSessionRepo { this.sessionLifetime = Duration.ofSeconds(componentRegistry.getConfigserverConfig().sessionLifetime()); this.zkWatcherExecutor = command -> componentRegistry.getZkWatcherExecutor().execute(tenantName, command); this.tenantFileSystemDirs = new TenantFileSystemDirs(componentRegistry.getConfigServerDB(), tenantName); - this.expiryTimeFlag = Flags.CONFIGSERVER_LOCAL_SESSIONS_EXPIRY_INTERVAL_IN_DAYS.bindTo(componentRegistry.getFlagSource()); loadSessions(sessionFactory); } @@ -98,7 +94,7 @@ public class LocalSessionRepo { // Sessions with state other than ACTIVATED if (hasExpired(candidate) && !isActiveSession(candidate)) { deleteSession(candidate); - } else if (createTime.plus(Duration.ofDays(expiryTimeFlag.value())).isBefore(clock.instant())) { + } else if (createTime.plus(Duration.ofDays(1)).isBefore(clock.instant())) { // Sessions with state ACTIVATE, but which are not actually active ApplicationId applicationId = candidate.getApplicationId(); Long activeSession = activeSessions.get(applicationId); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactory.java index 16bb32a19f2..fc4071916ed 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactory.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactory.java @@ -80,11 +80,11 @@ public class SessionFactory { * @param timeoutBudget Timeout for creating session and waiting for other servers. * @return a new session */ - public LocalSession createSession(File applicationDirectory, ApplicationId applicationId, TimeoutBudget timeoutBudget) { - return create(applicationDirectory, applicationId, nonExistingActiveSession, false, timeoutBudget); + public LocalSession createSession(File applicationDirectory, ApplicationId applicationId, + TimeoutBudget timeoutBudget, Optional<Long> activeSessionId) { + return create(applicationDirectory, applicationId, activeSessionId.orElse(nonExistingActiveSession), false, timeoutBudget); } - public RemoteSession createRemoteSession(long sessionId) { Path sessionPath = sessionsPath.append(String.valueOf(sessionId)); SessionZooKeeperClient sessionZKClient = createSessionZooKeeperClient(sessionPath); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java index 9a5a45bc03b..4a2e7cb405b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java @@ -153,8 +153,6 @@ public class SessionPreparer { final DeployLogger logger; final PrepareParams params; - final Optional<ApplicationSet> currentActiveApplicationSet; - final Path tenantPath; final ApplicationId applicationId; /** The repository part of docker image to be used for this deployment */ @@ -187,8 +185,6 @@ public class SessionPreparer { SessionZooKeeperClient sessionZooKeeperClient) { this.logger = logger; this.params = params; - this.currentActiveApplicationSet = currentActiveApplicationSet; - this.tenantPath = tenantPath; this.applicationPackage = preprocessedApplicationPackage; this.sessionZooKeeperClient = sessionZooKeeperClient; this.applicationId = params.getApplicationId(); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java index 43b25826507..90a03153d30 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java @@ -12,9 +12,7 @@ import com.yahoo.vespa.config.server.GlobalComponentRegistry; import com.yahoo.vespa.config.server.ReloadHandler; import com.yahoo.vespa.config.server.RequestHandler; import com.yahoo.vespa.config.server.application.TenantApplications; -import com.yahoo.vespa.config.server.host.HostValidator; import com.yahoo.vespa.config.server.monitoring.MetricUpdater; -import com.yahoo.vespa.config.server.rpc.ConfigResponseFactory; import com.yahoo.vespa.config.server.session.LocalSessionRepo; import com.yahoo.vespa.config.server.session.RemoteSessionRepo; import com.yahoo.vespa.config.server.session.SessionFactory; @@ -75,7 +73,7 @@ public class TenantRepository { private static final Logger log = Logger.getLogger(TenantRepository.class.getName()); private final Map<TenantName, Tenant> tenants = Collections.synchronizedMap(new LinkedHashMap<>()); - private final GlobalComponentRegistry globalComponentRegistry; + private final GlobalComponentRegistry componentRegistry; private final List<TenantListener> tenantListeners = Collections.synchronizedList(new ArrayList<>()); private final Curator curator; @@ -90,30 +88,30 @@ public class TenantRepository { /** * Creates a new tenant repository * - * @param globalComponentRegistry a {@link com.yahoo.vespa.config.server.GlobalComponentRegistry} + * @param componentRegistry a {@link com.yahoo.vespa.config.server.GlobalComponentRegistry} */ @Inject - public TenantRepository(GlobalComponentRegistry globalComponentRegistry) { - this(globalComponentRegistry, true); + public TenantRepository(GlobalComponentRegistry componentRegistry) { + this(componentRegistry, true); } /** * Creates a new tenant repository * - * @param globalComponentRegistry a {@link com.yahoo.vespa.config.server.GlobalComponentRegistry} + * @param componentRegistry a {@link com.yahoo.vespa.config.server.GlobalComponentRegistry} * @param useZooKeeperWatchForTenantChanges set to false for tests where you want to control adding and deleting * tenants yourself */ - public TenantRepository(GlobalComponentRegistry globalComponentRegistry, boolean useZooKeeperWatchForTenantChanges) { - this.globalComponentRegistry = globalComponentRegistry; - ConfigserverConfig configserverConfig = globalComponentRegistry.getConfigserverConfig(); + public TenantRepository(GlobalComponentRegistry componentRegistry, boolean useZooKeeperWatchForTenantChanges) { + this.componentRegistry = componentRegistry; + ConfigserverConfig configserverConfig = componentRegistry.getConfigserverConfig(); this.bootstrapExecutor = Executors.newFixedThreadPool(configserverConfig.numParallelTenantLoaders()); this.throwExceptionIfBootstrappingFails = configserverConfig.throwIfBootstrappingTenantRepoFails(); - this.curator = globalComponentRegistry.getCurator(); - metricUpdater = globalComponentRegistry.getMetrics().getOrCreateMetricUpdater(Collections.emptyMap()); - this.tenantListeners.add(globalComponentRegistry.getTenantListener()); - this.zkCacheExecutor = globalComponentRegistry.getZkCacheExecutor(); - this.zkWatcherExecutor = globalComponentRegistry.getZkWatcherExecutor(); + this.curator = componentRegistry.getCurator(); + metricUpdater = componentRegistry.getMetrics().getOrCreateMetricUpdater(Collections.emptyMap()); + this.tenantListeners.add(componentRegistry.getTenantListener()); + this.zkCacheExecutor = componentRegistry.getZkCacheExecutor(); + this.zkWatcherExecutor = componentRegistry.getZkWatcherExecutor(); curator.framework().getConnectionStateListenable().addListener(this::stateChanged); curator.create(tenantsPath); @@ -210,34 +208,29 @@ public class TenantRepository { private void createTenant(TenantName tenantName, RequestHandler requestHandler, ReloadHandler reloadHandler) { if (tenants.containsKey(tenantName)) return; - TenantRequestHandler tenantRequestHandler = null; - if (requestHandler == null) { - tenantRequestHandler = new TenantRequestHandler(globalComponentRegistry.getMetrics(), - tenantName, - List.of(globalComponentRegistry.getReloadListener()), - ConfigResponseFactory.create(globalComponentRegistry.getConfigserverConfig()), - globalComponentRegistry); - requestHandler = tenantRequestHandler; - } - - if (reloadHandler == null && tenantRequestHandler != null) - reloadHandler = tenantRequestHandler; - - HostValidator<ApplicationId> hostValidator = tenantRequestHandler; - TenantApplications applicationRepo = TenantApplications.create(globalComponentRegistry, - reloadHandler, - tenantName); - - SessionFactory sessionFactory = new SessionFactory(globalComponentRegistry, applicationRepo, hostValidator, tenantName); - LocalSessionRepo localSessionRepo = new LocalSessionRepo(tenantName, globalComponentRegistry, sessionFactory); - RemoteSessionRepo remoteSessionRepo = new RemoteSessionRepo(globalComponentRegistry, + TenantApplications applicationRepo = + new TenantApplications(tenantName, + curator, + componentRegistry.getZkWatcherExecutor(), + componentRegistry.getZkCacheExecutor(), + componentRegistry.getMetrics(), + componentRegistry.getReloadListener(), + componentRegistry.getConfigserverConfig(), + componentRegistry.getHostRegistries().createApplicationHostRegistry(tenantName)); + if (requestHandler == null) + requestHandler = applicationRepo; + if (reloadHandler == null) + reloadHandler = applicationRepo; + SessionFactory sessionFactory = new SessionFactory(componentRegistry, applicationRepo, applicationRepo, tenantName); + LocalSessionRepo localSessionRepo = new LocalSessionRepo(tenantName, componentRegistry, sessionFactory); + RemoteSessionRepo remoteSessionRepo = new RemoteSessionRepo(componentRegistry, sessionFactory, reloadHandler, tenantName, applicationRepo); log.log(Level.INFO, "Creating tenant '" + tenantName + "'"); - Tenant tenant = new Tenant(tenantName, sessionFactory, localSessionRepo, remoteSessionRepo, requestHandler, - reloadHandler, applicationRepo, globalComponentRegistry.getCurator()); + Tenant tenant = new Tenant(tenantName, sessionFactory, localSessionRepo, remoteSessionRepo, requestHandler, + reloadHandler, applicationRepo, componentRegistry.getCurator()); notifyNewTenant(tenant); tenants.putIfAbsent(tenantName, tenant); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandler.java deleted file mode 100644 index 25d6f194fdc..00000000000 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandler.java +++ /dev/null @@ -1,270 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.tenant; - -import com.yahoo.component.Version; -import com.yahoo.config.FileReference; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.TenantName; -import com.yahoo.vespa.config.ConfigKey; -import com.yahoo.vespa.config.GetConfigRequest; -import com.yahoo.vespa.config.protocol.ConfigResponse; -import com.yahoo.vespa.config.server.GlobalComponentRegistry; -import com.yahoo.vespa.config.server.NotFoundException; -import com.yahoo.vespa.config.server.ReloadHandler; -import com.yahoo.vespa.config.server.ReloadListener; -import com.yahoo.vespa.config.server.RequestHandler; -import com.yahoo.vespa.config.server.application.Application; -import com.yahoo.vespa.config.server.application.ApplicationMapper; -import com.yahoo.vespa.config.server.application.ApplicationSet; -import com.yahoo.vespa.config.server.application.TenantApplications; -import com.yahoo.vespa.config.server.application.VersionDoesNotExistException; -import com.yahoo.vespa.config.server.host.HostRegistry; -import com.yahoo.vespa.config.server.host.HostValidator; -import com.yahoo.vespa.config.server.monitoring.MetricUpdater; -import com.yahoo.vespa.config.server.monitoring.Metrics; -import com.yahoo.vespa.config.server.rpc.ConfigResponseFactory; -import com.yahoo.vespa.curator.Lock; - -import java.time.Clock; -import java.util.Collection; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Optional; -import java.util.Set; -import java.util.logging.Level; - -import static java.util.stream.Collectors.toSet; - -/** - * A per tenant request handler, for handling reload (activate application) and getConfig requests for - * a set of applications belonging to a tenant. - * - * @author Harald Musum - */ -public class TenantRequestHandler implements RequestHandler, ReloadHandler, HostValidator<ApplicationId> { - - private static final java.util.logging.Logger log = java.util.logging.Logger.getLogger(TenantRequestHandler.class.getName()); - - private final Metrics metrics; - private final TenantName tenant; - private final List<ReloadListener> reloadListeners; - private final ConfigResponseFactory responseFactory; - private final HostRegistry<ApplicationId> hostRegistry; - private final ApplicationMapper applicationMapper = new ApplicationMapper(); - private final MetricUpdater tenantMetricUpdater; - private final Clock clock = Clock.systemUTC(); - private final TenantApplications applications; - - public TenantRequestHandler(Metrics metrics, - TenantName tenant, - List<ReloadListener> reloadListeners, - ConfigResponseFactory responseFactory, - GlobalComponentRegistry registry) { // TODO jvenstad: Merge this class with TenantApplications, and straighten this out. - this.metrics = metrics; - this.tenant = tenant; - this.reloadListeners = List.copyOf(reloadListeners); - this.responseFactory = responseFactory; - this.tenantMetricUpdater = metrics.getOrCreateMetricUpdater(Metrics.createDimensions(tenant)); - this.hostRegistry = registry.getHostRegistries().createApplicationHostRegistry(tenant); - this.applications = TenantApplications.create(registry, this, tenant); - - } - - /** - * Gets a config for the given app, or null if not found - */ - @Override - public ConfigResponse resolveConfig(ApplicationId appId, GetConfigRequest req, Optional<Version> vespaVersion) { - Application application = getApplication(appId, vespaVersion); - if (log.isLoggable(Level.FINE)) { - log.log(Level.FINE, TenantRepository.logPre(appId) + "Resolving for tenant '" + tenant + "' with handler for application '" + application + "'"); - } - return application.resolveConfig(req, responseFactory); - } - - // For testing only - long getApplicationGeneration(ApplicationId appId, Optional<Version> vespaVersion) { - Application application = getApplication(appId, vespaVersion); - return application.getApplicationGeneration(); - } - - private void notifyReloadListeners(ApplicationSet applicationSet) { - for (ReloadListener reloadListener : reloadListeners) { - reloadListener.hostsUpdated(tenant, hostRegistry.getAllHosts()); - reloadListener.configActivated(applicationSet); - } - } - - /** - * Activates the config of the given app. Notifies listeners - * - * @param applicationSet the {@link ApplicationSet} to be reloaded - */ - @Override - public void reloadConfig(ApplicationSet applicationSet) { - ApplicationId id = applicationSet.getId(); - try (Lock lock = applications.lock(id)) { - if ( ! applications.exists(id)) - return; // Application was deleted before activation. - if (applicationSet.getApplicationGeneration() != applications.requireActiveSessionOf(id)) - return; // Application activated a new session before we got here. - - setLiveApp(applicationSet); - notifyReloadListeners(applicationSet); - } - } - - @Override - public void removeApplication(ApplicationId applicationId) { - try (Lock lock = applications.lock(applicationId)) { - if (applications.exists(applicationId)) - return; // Application was deployed again. - - if (applicationMapper.hasApplication(applicationId, clock.instant())) { - applicationMapper.remove(applicationId); - hostRegistry.removeHostsForKey(applicationId); - reloadListenersOnRemove(applicationId); - tenantMetricUpdater.setApplications(applicationMapper.numApplications()); - metrics.removeMetricUpdater(Metrics.createDimensions(applicationId)); - } - } - } - - @Override - public void removeApplicationsExcept(Set<ApplicationId> applications) { - for (ApplicationId activeApplication : applicationMapper.listApplicationIds()) { - if ( ! applications.contains(activeApplication)) { - log.log(Level.INFO, "Will remove deleted application " + activeApplication.toShortString()); - removeApplication(activeApplication); - } - } - } - - private void reloadListenersOnRemove(ApplicationId applicationId) { - for (ReloadListener listener : reloadListeners) { - listener.hostsUpdated(tenant, hostRegistry.getAllHosts()); - listener.applicationRemoved(applicationId); - } - } - - private void setLiveApp(ApplicationSet applicationSet) { - ApplicationId id = applicationSet.getId(); - Collection<String> hostsForApp = applicationSet.getAllHosts(); - hostRegistry.update(id, hostsForApp); - applicationSet.updateHostMetrics(); - tenantMetricUpdater.setApplications(applicationMapper.numApplications()); - applicationMapper.register(id, applicationSet); - } - - @Override - public Set<ConfigKey<?>> listNamedConfigs(ApplicationId appId, Optional<Version> vespaVersion, ConfigKey<?> keyToMatch, boolean recursive) { - Application application = getApplication(appId, vespaVersion); - return listConfigs(application, keyToMatch, recursive); - } - - private Set<ConfigKey<?>> listConfigs(Application application, ConfigKey<?> keyToMatch, boolean recursive) { - Set<ConfigKey<?>> ret = new LinkedHashSet<>(); - for (ConfigKey<?> key : application.allConfigsProduced()) { - String configId = key.getConfigId(); - if (recursive) { - key = new ConfigKey<>(key.getName(), configId, key.getNamespace()); - } else { - // Include first part of id as id - key = new ConfigKey<>(key.getName(), configId.split("/")[0], key.getNamespace()); - } - if (keyToMatch != null) { - String n = key.getName(); // Never null - String ns = key.getNamespace(); // Never null - if (n.equals(keyToMatch.getName()) && - ns.equals(keyToMatch.getNamespace()) && - configId.startsWith(keyToMatch.getConfigId()) && - !(configId.equals(keyToMatch.getConfigId()))) { - - if (!recursive) { - // For non-recursive, include the id segment we were searching for, and first part of the rest - key = new ConfigKey<>(key.getName(), appendOneLevelOfId(keyToMatch.getConfigId(), configId), key.getNamespace()); - } - ret.add(key); - } - } else { - ret.add(key); - } - } - return ret; - } - - @Override - public Set<ConfigKey<?>> listConfigs(ApplicationId appId, Optional<Version> vespaVersion, boolean recursive) { - Application application = getApplication(appId, vespaVersion); - return listConfigs(application, null, recursive); - } - - /** - * Given baseIdSegment search/ and id search/qrservers/default.0, return search/qrservers - * @return id segment with one extra level from the id appended - */ - String appendOneLevelOfId(String baseIdSegment, String id) { - if ("".equals(baseIdSegment)) return id.split("/")[0]; - String theRest = id.substring(baseIdSegment.length()); - if ("".equals(theRest)) return id; - theRest = theRest.replaceFirst("/", ""); - String theRestFirstSeg = theRest.split("/")[0]; - return baseIdSegment+"/"+theRestFirstSeg; - } - - @Override - public Set<ConfigKey<?>> allConfigsProduced(ApplicationId appId, Optional<Version> vespaVersion) { - Application application = getApplication(appId, vespaVersion); - return application.allConfigsProduced(); - } - - private Application getApplication(ApplicationId appId, Optional<Version> vespaVersion) { - try { - return applicationMapper.getForVersion(appId, vespaVersion, clock.instant()); - } catch (VersionDoesNotExistException ex) { - throw new NotFoundException(String.format("%sNo such application (id %s): %s", TenantRepository.logPre(tenant), appId, ex.getMessage())); - } - } - - @Override - public Set<String> allConfigIds(ApplicationId appId, Optional<Version> vespaVersion) { - Application application = getApplication(appId, vespaVersion); - return application.allConfigIds(); - } - - @Override - public boolean hasApplication(ApplicationId appId, Optional<Version> vespaVersion) { - return hasHandler(appId, vespaVersion); - } - - private boolean hasHandler(ApplicationId appId, Optional<Version> vespaVersion) { - return applicationMapper.hasApplicationForVersion(appId, vespaVersion, clock.instant()); - } - - @Override - public ApplicationId resolveApplicationId(String hostName) { - ApplicationId applicationId = hostRegistry.getKeyForHost(hostName); - if (applicationId == null) { - applicationId = ApplicationId.defaultId(); - } - return applicationId; - } - - @Override - public Set<FileReference> listFileReferences(ApplicationId applicationId) { - return applicationMapper.listApplications(applicationId).stream() - .flatMap(app -> app.getModel().fileReferences().stream()) - .collect(toSet()); - } - - @Override - public void verifyHosts(ApplicationId key, Collection<String> newHosts) { - hostRegistry.verifyHosts(key, newHosts); - for (ReloadListener reloadListener : reloadListeners) { - reloadListener.verifyHostsAreAvailable(tenant, newHosts); - } - } - - TenantApplications applications() { return applications; } - -} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ConfigCurator.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ConfigCurator.java index f3f9c914be8..35e8b0917cf 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ConfigCurator.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ConfigCurator.java @@ -111,11 +111,6 @@ public class ConfigCurator { return (data == null) ? null : Utf8.toString(data); } - /** Returns the data at a path and node. Replaces / by # in node names. Returns null if the path doesn't exist. */ - public byte[] getBytes(String path, String node) { - return getBytes(createFullPath(path, node)); - } - /** * Returns the data at a path, or null if the path does not exist. * diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java index d44be294713..a3d072be38b 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java @@ -2,7 +2,11 @@ package com.yahoo.vespa.config.server; import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.component.Version; +import com.yahoo.config.ConfigInstance; +import com.yahoo.config.SimpletypesConfig; import com.yahoo.config.application.api.ApplicationMetaData; +import com.yahoo.config.model.NullConfigModelRegistry; import com.yahoo.config.model.api.ApplicationRoles; import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.config.provision.AllocatedHosts; @@ -20,7 +24,14 @@ import com.yahoo.io.IOUtils; import com.yahoo.jdisc.Metric; import com.yahoo.test.ManualClock; import com.yahoo.text.Utf8; +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.ConfigPayload; +import com.yahoo.vespa.config.GetConfigRequest; +import com.yahoo.vespa.config.protocol.ConfigResponse; +import com.yahoo.vespa.config.protocol.DefContent; +import com.yahoo.vespa.config.protocol.VespaVersion; import com.yahoo.vespa.config.server.application.OrchestratorMock; +import com.yahoo.vespa.config.server.application.TenantApplications; import com.yahoo.vespa.config.server.deploy.DeployTester; import com.yahoo.vespa.config.server.http.InternalServerException; import com.yahoo.vespa.config.server.http.SessionHandlerTest; @@ -36,6 +47,9 @@ import com.yahoo.vespa.config.server.tenant.Tenant; import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.model.VespaModelFactory; +import org.hamcrest.core.Is; +import org.jetbrains.annotations.NotNull; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -76,6 +90,8 @@ public class ApplicationRepositoryTest { private final static File testAppJdiscOnly = new File("src/test/apps/app-jdisc-only"); private final static File testAppJdiscOnlyRestart = new File("src/test/apps/app-jdisc-only-restart"); private final static File testAppLogServerWithContainer = new File("src/test/apps/app-logserver-with-container"); + private final static File app1 = new File("src/test/apps/cs1"); + private final static File app2 = new File("src/test/apps/cs2"); private final static TenantName tenant1 = TenantName.from("test1"); private final static TenantName tenant2 = TenantName.from("test2"); @@ -94,12 +110,21 @@ public class ApplicationRepositoryTest { @Rule public ExpectedException exceptionRule = ExpectedException.none(); + @Rule + public TemporaryFolder tempFolder = new TemporaryFolder(); + @Before - public void setup() { + public void setup() throws IOException { Curator curator = new MockCurator(); - tenantRepository = new TenantRepository(new TestComponentRegistry.Builder() - .curator(curator) - .build()); + TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder() + .curator(curator) + .configServerConfig(new ConfigserverConfig.Builder() + .payloadCompressionType(ConfigserverConfig.PayloadCompressionType.Enum.UNCOMPRESSED) + .configServerDBDir(tempFolder.newFolder("configserverdb").getAbsolutePath()) + .configDefinitionsDir(tempFolder.newFolder("configdefinitions").getAbsolutePath()) + .build()) + .build(); + tenantRepository = new TenantRepository(componentRegistry, false); tenantRepository.addTenant(TenantRepository.HOSTED_VESPA_TENANT); tenantRepository.addTenant(tenant1); tenantRepository.addTenant(tenant2); @@ -142,6 +167,23 @@ public class ApplicationRepositoryTest { } @Test + public void redeploy() { + PrepareResult result = deployApp(testApp); + + long firstSessionId = result.sessionId(); + + PrepareResult result2 = deployApp(testApp); + long secondSessionId = result2.sessionId(); + assertNotEquals(firstSessionId, secondSessionId); + + TenantName tenantName = applicationId().tenant(); + Tenant tenant = tenantRepository.getTenant(tenantName); + LocalSession session = tenant.getLocalSessionRepo().getSession( + tenant.getApplicationRepo().requireActiveSessionOf(applicationId())); + assertEquals(firstSessionId, session.getMetaData().getPreviousActiveGeneration()); + } + + @Test public void createFromActiveSession() { PrepareResult result = deployApp(testApp); long sessionId = applicationRepository.createSessionFromExisting(applicationId(), @@ -529,6 +571,93 @@ public class ApplicationRepositoryTest { assertEquals(Session.Status.DEACTIVATE, firstSession.getStatus()); } + @Test + public void testResolveForAppId() { + Version vespaVersion = new VespaModelFactory(new NullConfigModelRegistry()).version(); + applicationRepository.deploy(app1, new PrepareParams.Builder() + .applicationId(applicationId()) + .vespaVersion(vespaVersion) + .build()); + + // TODO: Need to reload config before resolving works + RequestHandler requestHandler = reloadConfig(applicationId()); + SimpletypesConfig config = resolve(SimpletypesConfig.class, requestHandler, applicationId(), vespaVersion); + assertEquals(1337 , config.intval()); + } + + @Test + public void testResolveConfigForMultipleApps() { + Version vespaVersion = new VespaModelFactory(new NullConfigModelRegistry()).version(); + applicationRepository.deploy(app1, new PrepareParams.Builder() + .applicationId(applicationId()) + .vespaVersion(vespaVersion) + .build()); + + ApplicationId appId2 = new ApplicationId.Builder() + .tenant(tenant1) + .applicationName("myapp2") + .instanceName("default") + .build(); + applicationRepository.deploy(app2, new PrepareParams.Builder() + .applicationId(appId2) + .vespaVersion(vespaVersion) + .build()); + + // TODO: Need to reload config before resolving works + RequestHandler requestHandler = reloadConfig(applicationId()); + SimpletypesConfig config = resolve(SimpletypesConfig.class, requestHandler, applicationId(), vespaVersion); + assertEquals(1337, config.intval()); + + // TODO: Need to reload config before resolving works + RequestHandler requestHandler2 = reloadConfig(appId2); + SimpletypesConfig config2 = resolve(SimpletypesConfig.class, requestHandler2, appId2, vespaVersion); + assertEquals(1330, config2.intval()); + + assertTrue(requestHandler.hasApplication(applicationId(), Optional.of(vespaVersion))); + assertThat(requestHandler.resolveApplicationId("doesnotexist"), Is.is(ApplicationId.defaultId())); + assertThat(requestHandler.resolveApplicationId("mytesthost"), Is.is(new ApplicationId.Builder() + .tenant(tenant1) + .applicationName("testapp").build())); // Host set in application package. + } + + @Test + public void testResolveMultipleVersions() { + Version vespaVersion = new VespaModelFactory(new NullConfigModelRegistry()).version(); + applicationRepository.deploy(app1, new PrepareParams.Builder() + .applicationId(applicationId()) + .vespaVersion(vespaVersion) + .build()); + + // TODO: Need to reload config before resolving works + RequestHandler requestHandler = reloadConfig(applicationId()); + SimpletypesConfig config = resolve(SimpletypesConfig.class, requestHandler, applicationId(), vespaVersion); + assertEquals(1337, config.intval()); + + // TODO: Revisit this test, I cannot see that we create a model for version 3.2.1 + config = resolve(SimpletypesConfig.class, requestHandler, applicationId(), new Version(3, 2, 1)); + assertThat(config.intval(), Is.is(1337)); + } + + @Test + public void testResolveForDeletedApp() { + Version vespaVersion = new VespaModelFactory(new NullConfigModelRegistry()).version(); + applicationRepository.deploy(app1, new PrepareParams.Builder() + .applicationId(applicationId()) + .vespaVersion(vespaVersion) + .build()); + + // TODO: Need to reload config before resolving works + RequestHandler requestHandler = reloadConfig(applicationId()); + SimpletypesConfig config = resolve(SimpletypesConfig.class, requestHandler, applicationId(), vespaVersion); + assertEquals(1337 , config.intval()); + + applicationRepository.delete(applicationId()); + + exceptionRule.expect(com.yahoo.vespa.config.server.NotFoundException.class); + exceptionRule.expectMessage(containsString("No such application id: test1.testapp")); + resolve(SimpletypesConfig.class, requestHandler, applicationId(), vespaVersion); + } + private ApplicationRepository createApplicationRepository() { return new ApplicationRepository(tenantRepository, provisioner, @@ -569,7 +698,6 @@ public class ApplicationRepositoryTest { return applicationRepository.getMetadataFromLocalSession(tenant, sessionId); } - /** Stores all added or set values for each metric and context. */ static class MockMetric implements Metric { @@ -591,7 +719,6 @@ public class ApplicationRepositoryTest { return new Context(properties); } - private static class Context implements Metric.Context { private final Map<String, ?> point; @@ -604,4 +731,48 @@ public class ApplicationRepositoryTest { } + private <T extends ConfigInstance> T resolve(Class<T> clazz, + RequestHandler applications, + ApplicationId appId, + Version vespaVersion) { + String configId = ""; + ConfigResponse response = getConfigResponse(clazz, applications, appId, vespaVersion, configId); + return ConfigPayload.fromUtf8Array(response.getPayload()).toInstance(clazz, configId); + } + + private <T extends ConfigInstance> ConfigResponse getConfigResponse(Class<T> clazz, + RequestHandler applications, + ApplicationId appId, + Version vespaVersion, + String configId) { + return applications.resolveConfig(appId, new GetConfigRequest() { + @Override + public ConfigKey<T> getConfigKey() { + return new ConfigKey<>(clazz, configId); + } + + @Override + public DefContent getDefContent() { + return DefContent.fromClass(clazz); + } + + @Override + public Optional<VespaVersion> getVespaVersion() { + return Optional.of(VespaVersion.fromString(vespaVersion.toFullString())); + } + + @Override + public boolean noCache() { + return false; + } + }, Optional.empty()); + } + + @NotNull + private RequestHandler reloadConfig(ApplicationId applicationId) { + RequestHandler requestHandler = tenantRepository.getTenant(applicationId.tenant()).getRequestHandler(); + ((TenantApplications) requestHandler).reloadConfig(applicationRepository.getActiveSession(applicationId).ensureApplicationLoaded()); + return requestHandler; + } + } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelRequestHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelRequestHandlerTest.java index 4346d83e85e..f91205af44d 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelRequestHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelRequestHandlerTest.java @@ -2,19 +2,16 @@ package com.yahoo.vespa.config.server; import com.yahoo.cloud.config.ConfigserverConfig; -import com.yahoo.config.model.application.provider.FilesApplicationPackage; -import com.yahoo.config.provision.NodeFlavors; import com.yahoo.component.Version; +import com.yahoo.config.model.application.provider.FilesApplicationPackage; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Zone; -import com.yahoo.config.provisioning.FlavorsConfig; import com.yahoo.vespa.config.server.application.Application; -import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.config.server.application.ApplicationSet; import com.yahoo.vespa.config.server.monitoring.MetricUpdater; import com.yahoo.vespa.curator.mock.MockCurator; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.model.VespaModel; - import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -28,7 +25,11 @@ import java.util.Arrays; import java.util.Optional; import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; /** * @author Ulf Lilleengen @@ -129,10 +130,6 @@ public class SuperModelRequestHandlerTest { } - public static NodeFlavors emptyNodeFlavors() { - return new NodeFlavors(new FlavorsConfig(new FlavorsConfig.Builder())); - } - private ApplicationId applicationId(String tenantName, String applicationName) { return ApplicationId.from(tenantName, applicationName, "default"); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/TestComponentRegistry.java b/configserver/src/test/java/com/yahoo/vespa/config/server/TestComponentRegistry.java index b3dca7c73f3..ec5648757f1 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/TestComponentRegistry.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/TestComponentRegistry.java @@ -11,6 +11,7 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; import com.yahoo.container.jdisc.secretstore.SecretStore; import com.yahoo.vespa.config.server.application.PermanentApplicationPackage; +import com.yahoo.vespa.config.server.application.TenantApplicationsTest; import com.yahoo.vespa.config.server.host.HostRegistries; import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry; import com.yahoo.vespa.config.server.monitoring.Metrics; @@ -20,7 +21,6 @@ import com.yahoo.vespa.config.server.session.MockFileDistributionFactory; import com.yahoo.vespa.config.server.session.SessionPreparer; import com.yahoo.vespa.config.server.tenant.MockTenantListener; import com.yahoo.vespa.config.server.tenant.TenantListener; -import com.yahoo.vespa.config.server.tenant.TenantRequestHandlerTest; import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; @@ -106,7 +106,7 @@ public class TestComponentRegistry implements GlobalComponentRegistry { .configDefinitionsDir(uncheck(() -> Files.createTempDirectory("configdefinitions")).toString()) .sessionLifetime(5)); private ConfigDefinitionRepo defRepo = new StaticConfigDefinitionRepo(); - private TenantRequestHandlerTest.MockReloadListener reloadListener = new TenantRequestHandlerTest.MockReloadListener(); + private ReloadListener reloadListener = new TenantApplicationsTest.MockReloadListener(); private MockTenantListener tenantListener = new MockTenantListener(); private Optional<PermanentApplicationPackage> permanentApplicationPackage = Optional.empty(); private HostRegistries hostRegistries = new HostRegistries(); @@ -156,6 +156,11 @@ public class TestComponentRegistry implements GlobalComponentRegistry { return this; } + public Builder reloadListener(ReloadListener reloadListener) { + this.reloadListener = reloadListener; + return this; + } + public TestComponentRegistry build() { final PermanentApplicationPackage permApp = this.permanentApplicationPackage .orElse(new PermanentApplicationPackage(configserverConfig)); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/HttpProxyTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/HttpProxyTest.java index 4cccafef266..e64921e3ea0 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/HttpProxyTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/HttpProxyTest.java @@ -59,7 +59,7 @@ public class HttpProxyTest { public void testFetchException() { when(fetcher.get(any(), any())).thenThrow(new RequestTimeoutException("timed out")); - HttpResponse actualResponse = proxy.get(applicationMock, hostname, CLUSTERCONTROLLER_CONTAINER.serviceName, - "clustercontroller-status/v1/clusterName"); + proxy.get(applicationMock, hostname, CLUSTERCONTROLLER_CONTAINER.serviceName, + "clustercontroller-status/v1/clusterName"); } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/TenantApplicationsTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/TenantApplicationsTest.java index 33932a678b7..969040174dd 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/TenantApplicationsTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/TenantApplicationsTest.java @@ -1,23 +1,49 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.application; +import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.component.Version; +import com.yahoo.config.model.NullConfigModelRegistry; +import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; import com.yahoo.text.Utf8; -import com.yahoo.vespa.config.server.MockReloadHandler; - +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.server.ReloadListener; +import com.yahoo.vespa.config.server.ServerCache; import com.yahoo.vespa.config.server.TestComponentRegistry; +import com.yahoo.vespa.config.server.model.TestModelFactory; +import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry; +import com.yahoo.vespa.config.server.monitoring.MetricUpdater; import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.VespaModelFactory; import org.apache.curator.framework.CuratorFramework; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; - +import org.junit.rules.TemporaryFolder; +import org.xml.sax.SAXException; + +import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; import static org.hamcrest.Matchers.is; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; /** * @author Ulf Lilleengen @@ -25,14 +51,34 @@ import static org.junit.Assert.*; public class TenantApplicationsTest { private static final TenantName tenantName = TenantName.from("tenant"); + private static final Version vespaVersion = new VespaModelFactory(new NullConfigModelRegistry()).version(); - private Curator curator; + private MockReloadListener listener = new MockReloadListener(); private CuratorFramework curatorFramework; + private TestComponentRegistry componentRegistry; + private TenantApplications applications; + + @Rule + public TemporaryFolder tempFolder = new TemporaryFolder(); @Before - public void setup() { - curator = new MockCurator(); + public void setup() throws IOException { + Curator curator = new MockCurator(); curatorFramework = curator.framework(); + componentRegistry = new TestComponentRegistry.Builder() + .curator(curator) + .configServerConfig(new ConfigserverConfig.Builder() + .payloadCompressionType(ConfigserverConfig.PayloadCompressionType.Enum.UNCOMPRESSED) + .configServerDBDir(tempFolder.newFolder("configserverdb").getAbsolutePath()) + .configDefinitionsDir(tempFolder.newFolder("configdefinitions").getAbsolutePath()) + .build()) + .modelFactoryRegistry(createRegistry()) + .reloadListener(listener) + .build(); + TenantRepository tenantRepository = new TenantRepository(componentRegistry, false); + tenantRepository.addTenant(TenantRepository.HOSTED_VESPA_TENANT); + tenantRepository.addTenant(tenantName); + applications = TenantApplications.create(componentRegistry, tenantName); } @Test @@ -94,29 +140,71 @@ public class TenantApplicationsTest { assertThat(repo.activeApplications().size(), is(0)); } - @Test - public void require_that_reload_handler_is_called_when_apps_are_removed() throws Exception { - ApplicationId foo = createApplicationId("foo"); - writeApplicationData(foo, 3L); - writeApplicationData(createApplicationId("bar"), 4L); - MockReloadHandler reloadHandler = new MockReloadHandler(); - TenantApplications repo = createZKAppRepo(reloadHandler); - assertNull(reloadHandler.lastRemoved); - repo.createDeleteTransaction(foo).commit(); - long endTime = System.currentTimeMillis() + 60_000; - while (System.currentTimeMillis() < endTime && reloadHandler.lastRemoved == null) { - Thread.sleep(100); + public static class MockReloadListener implements ReloadListener { + public AtomicInteger reloaded = new AtomicInteger(0); + AtomicInteger removed = new AtomicInteger(0); + Map<String, Collection<String>> tenantHosts = new LinkedHashMap<>(); + + @Override + public void configActivated(ApplicationSet application) { + reloaded.incrementAndGet(); + } + + @Override + public void hostsUpdated(TenantName tenant, Collection<String> newHosts) { + tenantHosts.put(tenant.value(), newHosts); + } + + @Override + public void verifyHostsAreAvailable(TenantName tenant, Collection<String> newHosts) { + } + + @Override + public void applicationRemoved(ApplicationId applicationId) { + removed.incrementAndGet(); } - assertNotNull(reloadHandler.lastRemoved); - assertThat(reloadHandler.lastRemoved.serializedForm(), is(foo.serializedForm())); } - private TenantApplications createZKAppRepo() { - return createZKAppRepo(new MockReloadHandler()); + private void assertdefaultAppNotFound() { + assertFalse(applications.hasApplication(ApplicationId.defaultId(), Optional.of(vespaVersion))); + } + + @Test + public void testListConfigs() throws IOException, SAXException { + assertdefaultAppNotFound(); + + VespaModel model = new VespaModel(FilesApplicationPackage.fromFile(new File("src/test/apps/app"))); + applications.createApplication(ApplicationId.defaultId()); + applications.createPutTransaction(ApplicationId.defaultId(), 1).commit(); + applications.reloadConfig(ApplicationSet.fromSingle(new Application(model, + new ServerCache(), + 1, + false, + vespaVersion, + MetricUpdater.createTestUpdater(), + ApplicationId.defaultId()))); + Set<ConfigKey<?>> configNames = applications.listConfigs(ApplicationId.defaultId(), Optional.of(vespaVersion), false); + assertTrue(configNames.contains(new ConfigKey<>("sentinel", "hosts", "cloud.config"))); + + configNames = applications.listConfigs(ApplicationId.defaultId(), Optional.of(vespaVersion), true); + assertTrue(configNames.contains(new ConfigKey<>("documentmanager", "container", "document.config"))); + assertTrue(configNames.contains(new ConfigKey<>("documentmanager", "", "document.config"))); + assertTrue(configNames.contains(new ConfigKey<>("documenttypes", "", "document"))); + assertTrue(configNames.contains(new ConfigKey<>("documentmanager", "container", "document.config"))); + assertTrue(configNames.contains(new ConfigKey<>("health-monitor", "container", "container.jdisc.config"))); + assertTrue(configNames.contains(new ConfigKey<>("specific", "container", "project"))); } - private TenantApplications createZKAppRepo(MockReloadHandler reloadHandler) { - return TenantApplications.create(new TestComponentRegistry.Builder().curator(curator).build(), reloadHandler, tenantName); + @Test + public void testAppendIdsInNonRecursiveListing() { + assertEquals(applications.appendOneLevelOfId("search/music", "search/music/qrservers/default/qr.0"), "search/music/qrservers"); + assertEquals(applications.appendOneLevelOfId("search", "search/music/qrservers/default/qr.0"), "search/music"); + assertEquals(applications.appendOneLevelOfId("search/music/qrservers/default/qr.0", "search/music/qrservers/default/qr.0"), "search/music/qrservers/default/qr.0"); + assertEquals(applications.appendOneLevelOfId("", "search/music/qrservers/default/qr.0"), "search"); + } + + private TenantApplications createZKAppRepo() { + return TenantApplications.create(componentRegistry, tenantName); } private static ApplicationId createApplicationId(String name) { @@ -134,4 +222,10 @@ public class TenantApplicationsTest { .forPath(TenantRepository.getApplicationsPath(tenantName).append(applicationId).getAbsolute(), Utf8.toAsciiBytes(sessionId)); } + + private ModelFactoryRegistry createRegistry() { + return new ModelFactoryRegistry(Arrays.asList(new TestModelFactory(vespaVersion), + new TestModelFactory(new Version(3, 2, 1)))); + } + } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActionsBuilder.java b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActionsBuilder.java index 2566b1029a8..e2c3369d49e 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActionsBuilder.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/configchange/ConfigChangeActionsBuilder.java @@ -1,4 +1,4 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.configchange; import com.google.common.collect.ImmutableMap; @@ -6,7 +6,6 @@ import com.yahoo.config.model.api.ConfigChangeAction; import com.yahoo.config.model.api.ServiceInfo; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; /** diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/MockDeployer.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/MockDeployer.java deleted file mode 100644 index 967e2321b95..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/MockDeployer.java +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.deploy; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.Deployment; - -import java.time.Duration; -import java.time.Instant; -import java.util.Optional; - -/** - * @author Ulf Lilleengen - */ -public class MockDeployer implements com.yahoo.config.provision.Deployer { - - @Override - public Optional<Deployment> deployFromLocalActive(ApplicationId application) { - return deployFromLocalActive(application, Duration.ofSeconds(60)); - } - - @Override - public Optional<Deployment> deployFromLocalActive(ApplicationId application, boolean bootstrap) { - return Optional.empty(); - } - - @Override - public Optional<Deployment> deployFromLocalActive(ApplicationId application, Duration timeout) { - return Optional.empty(); - } - - @Override - public Optional<Deployment> deployFromLocalActive(ApplicationId application, Duration timeout, boolean bootstrap) { - return Optional.empty(); - } - - @Override - public Optional<Instant> lastDeployTime(ApplicationId application) { - return Optional.empty(); - } - -} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/ApplicationMetricsRetrieverTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/ApplicationMetricsRetrieverTest.java index 492767728e5..49de9b41ca1 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/ApplicationMetricsRetrieverTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/ApplicationMetricsRetrieverTest.java @@ -1,7 +1,6 @@ // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.metrics; -import com.yahoo.component.Version; import com.yahoo.config.FileReference; import com.yahoo.config.model.api.FileDistribution; import com.yahoo.config.model.api.HostInfo; @@ -23,7 +22,7 @@ import java.util.List; import java.util.Map; import java.util.Set; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; /** * @author olaa diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/MockRpc.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/MockRpc.java index 07f6e9cf222..d923f4c1856 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/MockRpc.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/MockRpc.java @@ -55,14 +55,6 @@ public class MockRpc extends RpcServer { this(port, true, tempDir); } - /** Reset fields used to assert on the calls made to this */ - public void resetChecks() { - forced = false; - tryResolveConfig = false; - tryRespond = false; - latestRequest = null; - } - private static ConfigserverConfig createConfig(int port) { ConfigserverConfig.Builder b = new ConfigserverConfig.Builder(); b.rpcport(port); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionRepoTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionRepoTest.java index 2c119a119b6..a758698d3b5 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionRepoTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionRepoTest.java @@ -59,7 +59,7 @@ public class LocalSessionRepoTest { .build()) .build(); SessionFactory sessionFactory = new SessionFactory(globalComponentRegistry, - TenantApplications.create(globalComponentRegistry, new MockReloadHandler(), tenantName), + TenantApplications.create(globalComponentRegistry, tenantName), new HostRegistry<>(), tenantName); repo = new LocalSessionRepo(tenantName, globalComponentRegistry, sessionFactory); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java index ce3b119d852..b072f20414f 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java @@ -12,7 +12,6 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.path.Path; import com.yahoo.slime.Slime; import com.yahoo.transaction.NestedTransaction; -import com.yahoo.vespa.config.server.MockReloadHandler; import com.yahoo.vespa.config.server.TestComponentRegistry; import com.yahoo.vespa.config.server.application.TenantApplications; import com.yahoo.vespa.config.server.deploy.DeployHandlerLogger; @@ -138,7 +137,7 @@ public class LocalSessionTest { File sessionDir = new File(tenantFileSystemDirs.sessionsPath(), String.valueOf(sessionId)); sessionDir.createNewFile(); TenantApplications applications = TenantApplications.create( - new TestComponentRegistry.Builder().curator(curator).build(), new MockReloadHandler(), tenant); + new TestComponentRegistry.Builder().curator(curator).build(), tenant); applications.createApplication(zkc.readApplicationId()); return new LocalSession(tenant, sessionId, preparer, FilesApplicationPackage.fromFile(testApp), zkc, sessionDir, applications, new HostRegistry<>()); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java index c3237c9fabb..51b0a36d8f4 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java @@ -2,21 +2,20 @@ package com.yahoo.vespa.config.server.tenant; import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.component.Version; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; -import com.yahoo.component.Version; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.config.server.GlobalComponentRegistry; -import com.yahoo.vespa.config.server.ReloadHandler; -import com.yahoo.vespa.config.server.RequestHandler; -import com.yahoo.vespa.config.server.application.ApplicationSet; import com.yahoo.vespa.config.server.ServerCache; import com.yahoo.vespa.config.server.TestComponentRegistry; import com.yahoo.vespa.config.server.application.Application; +import com.yahoo.vespa.config.server.application.ApplicationSet; +import com.yahoo.vespa.config.server.application.TenantApplicationsTest; import com.yahoo.vespa.config.server.monitoring.MetricUpdater; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; @@ -47,7 +46,7 @@ public class TenantRepositoryTest { private TenantRepository tenantRepository; private TestComponentRegistry globalComponentRegistry; - private TenantRequestHandlerTest.MockReloadListener listener; + private TenantApplicationsTest.MockReloadListener listener; private MockTenantListener tenantListener; private Curator curator; @@ -61,7 +60,7 @@ public class TenantRepositoryTest { public void setupSessions() { curator = new MockCurator(); globalComponentRegistry = new TestComponentRegistry.Builder().curator(curator).build(); - listener = (TenantRequestHandlerTest.MockReloadListener)globalComponentRegistry.getReloadListener(); + listener = (TenantApplicationsTest.MockReloadListener)globalComponentRegistry.getReloadListener(); tenantListener = (MockTenantListener)globalComponentRegistry.getTenantListener(); assertFalse(tenantListener.tenantsLoaded); tenantRepository = new TenantRepository(globalComponentRegistry); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandlerTest.java deleted file mode 100644 index fa9efec1255..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandlerTest.java +++ /dev/null @@ -1,373 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.server.tenant; - -import com.yahoo.component.Version; -import com.yahoo.config.ConfigInstance; -import com.yahoo.config.SimpletypesConfig; -import com.yahoo.config.application.api.ApplicationPackage; -import com.yahoo.config.model.NullConfigModelRegistry; -import com.yahoo.config.model.application.provider.BaseDeployLogger; -import com.yahoo.config.model.application.provider.DeployData; -import com.yahoo.config.model.application.provider.FilesApplicationPackage; -import com.yahoo.config.model.application.provider.MockFileRegistry; -import com.yahoo.config.provision.AllocatedHosts; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ApplicationName; -import com.yahoo.config.provision.TenantName; -import com.yahoo.io.IOUtils; -import com.yahoo.vespa.config.ConfigKey; -import com.yahoo.vespa.config.ConfigPayload; -import com.yahoo.vespa.config.GetConfigRequest; -import com.yahoo.vespa.config.protocol.ConfigResponse; -import com.yahoo.vespa.config.protocol.DefContent; -import com.yahoo.vespa.config.protocol.VespaVersion; -import com.yahoo.vespa.config.server.ReloadListener; -import com.yahoo.vespa.config.server.ServerCache; -import com.yahoo.vespa.config.server.TestComponentRegistry; -import com.yahoo.vespa.config.server.application.Application; -import com.yahoo.vespa.config.server.application.ApplicationSet; -import com.yahoo.vespa.config.server.deploy.ZooKeeperDeployer; -import com.yahoo.vespa.config.server.model.TestModelFactory; -import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry; -import com.yahoo.vespa.config.server.monitoring.MetricUpdater; -import com.yahoo.vespa.config.server.monitoring.Metrics; -import com.yahoo.vespa.config.server.rpc.UncompressedConfigResponseFactory; -import com.yahoo.vespa.config.server.session.RemoteSession; -import com.yahoo.vespa.config.server.session.SessionZooKeeperClient; -import com.yahoo.vespa.curator.Curator; -import com.yahoo.vespa.curator.mock.MockCurator; -import com.yahoo.vespa.model.VespaModel; -import com.yahoo.vespa.model.VespaModelFactory; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.xml.sax.SAXException; - -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.atomic.AtomicInteger; - -import static org.hamcrest.core.Is.is; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertThat; -import static org.junit.Assert.assertTrue; - -/** - * @author Ulf Lilleengen - */ -public class TenantRequestHandlerTest { - - private static final Version vespaVersion = new VespaModelFactory(new NullConfigModelRegistry()).version(); - private TenantRequestHandler server; - private MockReloadListener listener = new MockReloadListener(); - private File app1 = new File("src/test/apps/cs1"); - private File app2 = new File("src/test/apps/cs2"); - private TenantName tenant = TenantName.from("mytenant"); - private TestComponentRegistry componentRegistry; - private Curator curator; - - @Rule - public TemporaryFolder tempFolder = new TemporaryFolder(); - - private ApplicationId defaultApp() { - return new ApplicationId.Builder().applicationName(ApplicationName.defaultName()).tenant(tenant).build(); - } - - @Before - public void setUp() throws IOException { - curator = new MockCurator(); - - feedApp(app1, 1, defaultApp(), false); - Metrics sh = Metrics.createTestMetrics(); - List<ReloadListener> listeners = new ArrayList<>(); - listeners.add(listener); - componentRegistry = new TestComponentRegistry.Builder() - .curator(curator) - .modelFactoryRegistry(createRegistry()) - .build(); - server = new TenantRequestHandler(sh, tenant, listeners, new UncompressedConfigResponseFactory(), componentRegistry); - } - - private void feedApp(File appDir, long sessionId, ApplicationId appId, boolean internalRedeploy) throws IOException { - SessionZooKeeperClient zkc = new SessionZooKeeperClient(curator, TenantRepository.getSessionsPath(tenant).append(String.valueOf(sessionId))); - zkc.writeApplicationId(appId); - File app = tempFolder.newFolder(); - IOUtils.copyDirectory(appDir, app); - ZooKeeperDeployer deployer = zkc.createDeployer(new BaseDeployLogger()); - DeployData deployData = new DeployData("user", - appDir.toString(), - appId, - 0L, - internalRedeploy, - 0L, - 0L); - ApplicationPackage appPackage = FilesApplicationPackage.fromFileWithDeployData(appDir, deployData); - deployer.deploy(appPackage, - Collections.singletonMap(vespaVersion, new MockFileRegistry()), - AllocatedHosts.withHosts(Collections.emptySet())); - } - - private ApplicationSet reloadConfig(long sessionId) { - return reloadConfig(sessionId, "default"); - } - - private ApplicationSet reloadConfig(long sessionId, String application) { - SessionZooKeeperClient zkc = new SessionZooKeeperClient(curator, TenantRepository.getSessionsPath(tenant).append(String.valueOf(sessionId))); - zkc.writeApplicationId(new ApplicationId.Builder().tenant(tenant).applicationName(application).build()); - RemoteSession session = new RemoteSession(tenant, sessionId, componentRegistry, zkc); - return session.ensureApplicationLoaded(); - } - - private ModelFactoryRegistry createRegistry() { - return new ModelFactoryRegistry(Arrays.asList(new TestModelFactory(vespaVersion), - new TestModelFactory(new Version(3, 2, 1)))); - } - - private <T extends ConfigInstance> T resolve(Class<T> clazz, - TenantRequestHandler tenantRequestHandler, - ApplicationId appId, - Version vespaVersion, - String configId) { - ConfigResponse response = getConfigResponse(clazz, tenantRequestHandler, appId, vespaVersion, configId); - return ConfigPayload.fromUtf8Array(response.getPayload()).toInstance(clazz, configId); - } - - private <T extends ConfigInstance> ConfigResponse getConfigResponse(Class<T> clazz, - TenantRequestHandler tenantRequestHandler, - ApplicationId appId, - Version vespaVersion, - String configId) { - return tenantRequestHandler.resolveConfig(appId, new GetConfigRequest() { - @Override - public ConfigKey<T> getConfigKey() { - return new ConfigKey<>(clazz, configId); - } - - @Override - public DefContent getDefContent() { - return DefContent.fromClass(clazz); - } - - @Override - public Optional<VespaVersion> getVespaVersion() { - return Optional.of(VespaVersion.fromString(vespaVersion.toFullString())); - } - - @Override - public boolean noCache() { - return false; - } - }, Optional.empty()); - } - - @Test - public void testReloadConfig() throws IOException { - ApplicationId applicationId = new ApplicationId.Builder().applicationName(ApplicationName.defaultName()).tenant(tenant).build(); - - server.applications().createApplication(applicationId); - server.applications().createPutTransaction(applicationId, 1).commit(); - server.reloadConfig(reloadConfig(1)); - assertThat(listener.reloaded.get(), is(1)); - // Using only payload list for this simple test - SimpletypesConfig config = resolve(SimpletypesConfig.class, server, defaultApp(), vespaVersion, ""); - assertThat(config.intval(), is(1337)); - assertThat(server.getApplicationGeneration(applicationId, Optional.of(vespaVersion)), is(1L)); - - server.reloadConfig(reloadConfig(1L)); - ConfigResponse configResponse = getConfigResponse(SimpletypesConfig.class, server, defaultApp(), vespaVersion, ""); - assertFalse(configResponse.isInternalRedeploy()); - config = resolve(SimpletypesConfig.class, server, defaultApp(), vespaVersion, ""); - assertThat(config.intval(), is(1337)); - assertThat(listener.reloaded.get(), is(2)); - assertThat(server.getApplicationGeneration(applicationId, Optional.of(vespaVersion)), is(1L)); - assertThat(listener.tenantHosts.size(), is(1)); - assertThat(server.resolveApplicationId("mytesthost"), is(applicationId)); - - listener.reloaded.set(0); - feedApp(app2, 2, defaultApp(), true); - server.applications().createPutTransaction(applicationId, 2).commit(); - server.reloadConfig(reloadConfig(2L)); - configResponse = getConfigResponse(SimpletypesConfig.class, server, defaultApp(), vespaVersion, ""); - assertTrue(configResponse.isInternalRedeploy()); - config = resolve(SimpletypesConfig.class, server, defaultApp(), vespaVersion,""); - assertThat(config.intval(), is(1330)); - assertThat(listener.reloaded.get(), is(1)); - assertThat(server.getApplicationGeneration(applicationId, Optional.of(vespaVersion)), is(2L)); - } - - @Test - public void testRemoveApplication() { - ApplicationId appId = ApplicationId.from(tenant.value(), "default", "default"); - server.reloadConfig(reloadConfig(1)); - assertThat(listener.reloaded.get(), is(0)); - - server.applications().createApplication(appId); - server.applications().createPutTransaction(appId, 1).commit(); - server.reloadConfig(reloadConfig(1)); - assertThat(listener.reloaded.get(), is(1)); - - assertThat(listener.removed.get(), is(0)); - - server.removeApplication(appId); - assertThat(listener.removed.get(), is(0)); - - server.applications().createDeleteTransaction(appId).commit(); - server.removeApplication(appId); - assertThat(listener.removed.get(), is(1)); - } - - @Test - public void testResolveForAppId() { - long id = 1L; - ApplicationId appId = new ApplicationId.Builder() - .tenant(tenant) - .applicationName("myapp").instanceName("myinst").build(); - server.applications().createApplication(appId); - server.applications().createPutTransaction(appId, 1).commit(); - SessionZooKeeperClient zkc = new SessionZooKeeperClient(curator, TenantRepository.getSessionsPath(tenant).append(String.valueOf(id))); - zkc.writeApplicationId(appId); - RemoteSession session = new RemoteSession(appId.tenant(), id, componentRegistry, zkc); - server.reloadConfig(session.ensureApplicationLoaded()); - SimpletypesConfig config = resolve(SimpletypesConfig.class, server, appId, vespaVersion, ""); - assertThat(config.intval(), is(1337)); - } - - @Test - public void testResolveMultipleApps() throws IOException { - ApplicationId appId1 = new ApplicationId.Builder() - .tenant(tenant) - .applicationName("myapp1").instanceName("myinst1").build(); - ApplicationId appId2 = new ApplicationId.Builder() - .tenant(tenant) - .applicationName("myapp2").instanceName("myinst2").build(); - feedAndReloadApp(app1, 1, appId1); - SimpletypesConfig config = resolve(SimpletypesConfig.class, server, appId1, vespaVersion, ""); - assertThat(config.intval(), is(1337)); - - feedAndReloadApp(app2, 2, appId2); - config = resolve(SimpletypesConfig.class, server, appId2, vespaVersion, ""); - assertThat(config.intval(), is(1330)); - } - - @Test - public void testResolveMultipleVersions() throws IOException { - ApplicationId appId = new ApplicationId.Builder() - .tenant(tenant) - .applicationName("myapp1").instanceName("myinst1").build(); - feedAndReloadApp(app1, 1, appId); - SimpletypesConfig config = resolve(SimpletypesConfig.class, server, appId, vespaVersion, ""); - assertThat(config.intval(), is(1337)); - config = resolve(SimpletypesConfig.class, server, appId, new Version(3, 2, 1), ""); - assertThat(config.intval(), is(1337)); - } - - private void feedAndReloadApp(File appDir, long sessionId, ApplicationId appId) throws IOException { - server.applications().createApplication(appId); - server.applications().createPutTransaction(appId, sessionId).commit(); - feedApp(appDir, sessionId, appId, false); - SessionZooKeeperClient zkc = new SessionZooKeeperClient(curator, TenantRepository.getSessionsPath(tenant).append(String.valueOf(sessionId))); - zkc.writeApplicationId(appId); - RemoteSession session = new RemoteSession(tenant, sessionId, componentRegistry, zkc); - server.reloadConfig(session.ensureApplicationLoaded()); - } - - public static class MockReloadListener implements ReloadListener { - AtomicInteger reloaded = new AtomicInteger(0); - AtomicInteger removed = new AtomicInteger(0); - Map<String, Collection<String>> tenantHosts = new LinkedHashMap<>(); - - @Override - public void configActivated(ApplicationSet application) { - reloaded.incrementAndGet(); - } - - @Override - public void hostsUpdated(TenantName tenant, Collection<String> newHosts) { - tenantHosts.put(tenant.value(), newHosts); - } - - @Override - public void verifyHostsAreAvailable(TenantName tenant, Collection<String> newHosts) { - } - - @Override - public void applicationRemoved(ApplicationId applicationId) { - removed.incrementAndGet(); - } - } - - @Test - public void testHasApplication() { - assertdefaultAppNotFound(); - ApplicationId appId = ApplicationId.from(tenant.value(), "default", "default"); - server.applications().createApplication(appId); - server.applications().createPutTransaction(appId, 1).commit(); - server.reloadConfig(reloadConfig(1)); - assertTrue(server.hasApplication(appId, Optional.of(vespaVersion))); - } - - private void assertdefaultAppNotFound() { - assertFalse(server.hasApplication(ApplicationId.defaultId(), Optional.of(vespaVersion))); - } - - @Test - public void testMultipleApplicationsReload() { - ApplicationId appId = ApplicationId.from(tenant.value(), "foo", "default"); - assertdefaultAppNotFound(); - server.applications().createApplication(appId); - server.applications().createPutTransaction(appId, 1).commit(); - server.reloadConfig(reloadConfig(1, "foo")); - assertdefaultAppNotFound(); - assertTrue(server.hasApplication(appId, - Optional.of(vespaVersion))); - assertThat(server.resolveApplicationId("doesnotexist"), is(ApplicationId.defaultId())); - assertThat(server.resolveApplicationId("mytesthost"), is(new ApplicationId.Builder() - .tenant(tenant) - .applicationName("foo").build())); // Host set in application package. - } - - @Test - public void testListConfigs() throws IOException, SAXException { - assertdefaultAppNotFound(); - - VespaModel model = new VespaModel(FilesApplicationPackage.fromFile(new File("src/test/apps/app"))); - server.applications().createApplication(ApplicationId.defaultId()); - server.applications().createPutTransaction(ApplicationId.defaultId(), 1).commit(); - server.reloadConfig(ApplicationSet.fromSingle(new Application(model, - new ServerCache(), - 1, - false, - vespaVersion, - MetricUpdater.createTestUpdater(), - ApplicationId.defaultId()))); - Set<ConfigKey<?>> configNames = server.listConfigs(ApplicationId.defaultId(), Optional.of(vespaVersion), false); - assertTrue(configNames.contains(new ConfigKey<>("sentinel", "hosts", "cloud.config"))); - - configNames = server.listConfigs(ApplicationId.defaultId(), Optional.of(vespaVersion), true); - assertTrue(configNames.contains(new ConfigKey<>("documentmanager", "container", "document.config"))); - assertTrue(configNames.contains(new ConfigKey<>("documentmanager", "", "document.config"))); - assertTrue(configNames.contains(new ConfigKey<>("documenttypes", "", "document"))); - assertTrue(configNames.contains(new ConfigKey<>("documentmanager", "container", "document.config"))); - assertTrue(configNames.contains(new ConfigKey<>("health-monitor", "container", "container.jdisc.config"))); - assertTrue(configNames.contains(new ConfigKey<>("specific", "container", "project"))); - } - - @Test - public void testAppendIdsInNonRecursiveListing() { - assertEquals(server.appendOneLevelOfId("search/music", "search/music/qrservers/default/qr.0"), "search/music/qrservers"); - assertEquals(server.appendOneLevelOfId("search", "search/music/qrservers/default/qr.0"), "search/music"); - assertEquals(server.appendOneLevelOfId("search/music/qrservers/default/qr.0", "search/music/qrservers/default/qr.0"), "search/music/qrservers/default/qr.0"); - assertEquals(server.appendOneLevelOfId("", "search/music/qrservers/default/qr.0"), "search"); - } -} diff --git a/container-core/src/main/java/com/yahoo/container/handler/ThreadPoolProvider.java b/container-core/src/main/java/com/yahoo/container/handler/ThreadPoolProvider.java index 1ec5455dacd..425387039ff 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/ThreadPoolProvider.java +++ b/container-core/src/main/java/com/yahoo/container/handler/ThreadPoolProvider.java @@ -46,7 +46,7 @@ public class ThreadPoolProvider extends AbstractComponent implements Provider<Ex */ @Override public void deconstruct() { - threadpool.deconstruct(); + threadpool.close(); } } diff --git a/container-core/src/main/java/com/yahoo/container/handler/threadpool/ContainerThreadPool.java b/container-core/src/main/java/com/yahoo/container/handler/threadpool/ContainerThreadPool.java index 3789d5cbedc..6fc9da298a8 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/threadpool/ContainerThreadPool.java +++ b/container-core/src/main/java/com/yahoo/container/handler/threadpool/ContainerThreadPool.java @@ -1,8 +1,6 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.handler.threadpool; -import com.google.inject.Inject; -import com.yahoo.component.AbstractComponent; import com.yahoo.concurrent.ThreadFactoryFactory; import com.yahoo.container.handler.ThreadpoolConfig; import com.yahoo.container.protect.ProcessTerminator; @@ -22,11 +20,10 @@ import java.util.concurrent.TimeUnit; * @author bratseth * @author bjorncs */ -public class ContainerThreadPool extends AbstractComponent implements AutoCloseable { +public class ContainerThreadPool implements AutoCloseable { private final ExecutorServiceWrapper threadpool; - @Inject public ContainerThreadPool(ThreadpoolConfig config, Metric metric) { this(config, metric, new ProcessTerminator()); } @@ -50,7 +47,6 @@ public class ContainerThreadPool extends AbstractComponent implements AutoClosea } public Executor executor() { return threadpool; } - @Override public void deconstruct() { closeInternal(); } @Override public void close() { closeInternal(); } /** @@ -60,7 +56,6 @@ public class ContainerThreadPool extends AbstractComponent implements AutoClosea private void closeInternal() { boolean terminated; - super.deconstruct(); threadpool.shutdown(); try { terminated = threadpool.awaitTermination(1, TimeUnit.SECONDS); diff --git a/container-core/src/test/java/com/yahoo/container/handler/threadpool/ContainerThreadPoolTest.java b/container-core/src/test/java/com/yahoo/container/handler/threadpool/ContainerThreadPoolTest.java index 7998bbc4872..f6a3aebd7ff 100644 --- a/container-core/src/test/java/com/yahoo/container/handler/threadpool/ContainerThreadPoolTest.java +++ b/container-core/src/test/java/com/yahoo/container/handler/threadpool/ContainerThreadPoolTest.java @@ -44,7 +44,7 @@ public class ContainerThreadPoolTest { if (reply.second != Boolean.TRUE) { fail("Executor task seemed to run, but did not get correct value."); } - threadPool.deconstruct(); + threadPool.close(); command = new FlipIt(); try { exec.execute(command); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java index 15eb330e308..68dff26529f 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java @@ -22,7 +22,8 @@ enum PathGroup { /** Paths exclusive to operators (including read), used for system management. */ classifiedOperator(PathPrefix.api, - "/configserver/v1/{*}"), + "/configserver/v1/{*}", + "/deployment/v1/{*}"), /** Paths used for system management by operators. */ operator(PathPrefix.none, @@ -199,15 +200,11 @@ enum PathGroup { "/", "/d/{*}"), - /** Same as classifiedInfo, but with optional /api prefix */ - classifiedApiInfo(PathPrefix.api, - "/deployment/v1/{*}", - "/user/v1/user"), - /** Paths providing public information. */ publicInfo(PathPrefix.api, - "/badge/v1/{*}", - "/zone/v1/{*}"), + "/user/v1/user", // Information about who you are. + "/badge/v1/{*}", // Badges for deployment jobs. + "/zone/v1/{*}"), // Lists environment and regions. /** Paths used for deploying system-wide feature flags. */ systemFlagsDeploy(PathPrefix.none, "/system-flags/v1/deploy"), diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java index 00550387db5..fc904b9d1a0 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java @@ -123,10 +123,6 @@ enum Policy { .on(PathGroup.allExcept(PathGroup.classifiedOperator)) .in(SystemName.main, SystemName.cd, SystemName.dev)), - classifiedApiRead(Privilege.grant(Action.read) - .on(PathGroup.classifiedApiInfo) - .in(SystemName.all())), - /** Read access to public info. */ publicRead(Privilege.grant(Action.read) .on(PathGroup.publicInfo) diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java index 6467050d3f3..ad7b3f68440 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java @@ -27,7 +27,6 @@ public enum RoleDefinition { /** Base role which every user is part of. */ everyone(Policy.classifiedRead, - Policy.classifiedApiRead, Policy.publicRead, Policy.user, Policy.tenantCreate), diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutor.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutor.java index 529acc48cbe..110d994c179 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutor.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutor.java @@ -10,5 +10,7 @@ import com.yahoo.container.jdisc.HttpResponse; * @author Haakon Dybdahl */ public interface ConfigServerRestExecutor { - HttpResponse handle(ProxyRequest proxyRequest) throws ProxyException; + + HttpResponse handle(ProxyRequest request); + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java index d10c4dd226b..f5dcae9c961 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java @@ -1,15 +1,16 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.proxy; +import ai.vespa.util.http.retry.Sleeper; import com.google.inject.Inject; import com.yahoo.component.AbstractComponent; import com.yahoo.jdisc.http.HttpRequest.Method; -import java.util.logging.Level; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider; import com.yahoo.vespa.athenz.tls.AthenzIdentityVerifier; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import org.apache.http.Header; +import org.apache.http.HttpResponse; import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpDelete; @@ -19,11 +20,15 @@ import org.apache.http.client.methods.HttpPost; import org.apache.http.client.methods.HttpPut; import org.apache.http.client.methods.HttpRequestBase; import org.apache.http.entity.InputStreamEntity; +import org.apache.http.impl.DefaultConnectionReuseStrategy; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.protocol.HttpContext; +import org.apache.http.protocol.HttpCoreContext; import org.apache.http.util.EntityUtils; import javax.net.ssl.HostnameVerifier; +import javax.net.ssl.SSLContext; import javax.net.ssl.SSLSession; import java.io.IOException; import java.io.InputStream; @@ -37,6 +42,7 @@ import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -50,75 +56,87 @@ import static com.yahoo.yolean.Exceptions.uncheck; @SuppressWarnings("unused") // Injected public class ConfigServerRestExecutorImpl extends AbstractComponent implements ConfigServerRestExecutor { - private static final Logger log = Logger.getLogger(ConfigServerRestExecutorImpl.class.getName()); - + private static final Logger LOG = Logger.getLogger(ConfigServerRestExecutorImpl.class.getName()); private static final Duration PROXY_REQUEST_TIMEOUT = Duration.ofSeconds(10); + private static final Duration PING_REQUEST_TIMEOUT = Duration.ofMillis(500); + private static final Duration SINGLE_TARGET_WAIT = Duration.ofSeconds(2); + private static final int SINGLE_TARGET_RETRIES = 3; private static final Set<String> HEADERS_TO_COPY = Set.of("X-HTTP-Method-Override", "Content-Type"); private final CloseableHttpClient client; + private final Sleeper sleeper; @Inject public ConfigServerRestExecutorImpl(ZoneRegistry zoneRegistry, ServiceIdentityProvider sslContextProvider) { - RequestConfig config = RequestConfig.custom() - .setConnectTimeout((int) PROXY_REQUEST_TIMEOUT.toMillis()) - .setConnectionRequestTimeout((int) PROXY_REQUEST_TIMEOUT.toMillis()) - .setSocketTimeout((int) PROXY_REQUEST_TIMEOUT.toMillis()).build(); + this(zoneRegistry, sslContextProvider.getIdentitySslContext(), new Sleeper.Default(), + new ConnectionReuseStrategy(zoneRegistry)); + } - this.client = createHttpClient(config, sslContextProvider, - new ControllerOrConfigserverHostnameVerifier(zoneRegistry)); + ConfigServerRestExecutorImpl(ZoneRegistry zoneRegistry, SSLContext sslContext, + Sleeper sleeper, ConnectionReuseStrategy connectionReuseStrategy) { + this.client = createHttpClient(sslContext, + new ControllerOrConfigserverHostnameVerifier(zoneRegistry), + connectionReuseStrategy); + this.sleeper = sleeper; } @Override - public ProxyResponse handle(ProxyRequest proxyRequest) throws ProxyException { - // Make a local copy of the list as we want to manipulate it in case of ping problems. - List<URI> allServers = new ArrayList<>(proxyRequest.getTargets()); + public ProxyResponse handle(ProxyRequest request) { + List<URI> targets = new ArrayList<>(request.getTargets()); StringBuilder errorBuilder = new StringBuilder(); - if (queueFirstServerIfDown(allServers)) { + boolean singleTarget = targets.size() == 1; + if (singleTarget) { + for (int i = 0; i < SINGLE_TARGET_RETRIES - 1; i++) { + targets.add(targets.get(0)); + } + } else if (queueFirstServerIfDown(targets)) { errorBuilder.append("Change ordering due to failed ping."); } - for (URI uri : allServers) { - Optional<ProxyResponse> proxyResponse = proxyCall(uri, proxyRequest, errorBuilder); + + for (URI url : targets) { + Optional<ProxyResponse> proxyResponse = proxy(request, url, errorBuilder); if (proxyResponse.isPresent()) { return proxyResponse.get(); } + if (singleTarget) { + sleeper.sleep(SINGLE_TARGET_WAIT); + } } - // TODO Add logging, for now, experimental and we want to not add more noise. - throw new ProxyException(ErrorResponse.internalServerError("Failed talking to config servers: " - + errorBuilder.toString())); + + throw new RuntimeException("Failed talking to config servers: " + errorBuilder.toString()); } - private Optional<ProxyResponse> proxyCall(URI uri, ProxyRequest proxyRequest, StringBuilder errorBuilder) - throws ProxyException { - final HttpRequestBase requestBase = createHttpBaseRequest( - proxyRequest.getMethod(), proxyRequest.createConfigServerRequestUri(uri), proxyRequest.getData()); + private Optional<ProxyResponse> proxy(ProxyRequest request, URI url, StringBuilder errorBuilder) { + HttpRequestBase requestBase = createHttpBaseRequest( + request.getMethod(), request.createConfigServerRequestUri(url), request.getData()); // Empty list of headers to copy for now, add headers when needed, or rewrite logic. - copyHeaders(proxyRequest.getHeaders(), requestBase); + copyHeaders(request.getHeaders(), requestBase); try (CloseableHttpResponse response = client.execute(requestBase)) { String content = getContent(response); int status = response.getStatusLine().getStatusCode(); if (status / 100 == 5) { - errorBuilder.append("Talking to server ").append(uri.getHost()); + errorBuilder.append("Talking to server ").append(url.getHost()); errorBuilder.append(", got ").append(status).append(" ") .append(content).append("\n"); - log.log(Level.FINE, () -> String.format("Got response from %s with status code %d and content:\n %s", - uri.getHost(), status, content)); + LOG.log(Level.FINE, () -> String.format("Got response from %s with status code %d and content:\n %s", + url.getHost(), status, content)); return Optional.empty(); } - final Header contentHeader = response.getLastHeader("Content-Type"); - final String contentType; + Header contentHeader = response.getLastHeader("Content-Type"); + String contentType; if (contentHeader != null && contentHeader.getValue() != null && ! contentHeader.getValue().isEmpty()) { contentType = contentHeader.getValue().replace("; charset=UTF-8",""); } else { contentType = "application/json"; } // Send response back - return Optional.of(new ProxyResponse(proxyRequest, content, status, uri, contentType)); + return Optional.of(new ProxyResponse(request, content, status, url, contentType)); } catch (Exception e) { - errorBuilder.append("Talking to server ").append(uri.getHost()); + errorBuilder.append("Talking to server ").append(url.getHost()); errorBuilder.append(" got exception ").append(e.getMessage()); - log.log(Level.FINE, e, () -> "Got exception while sending request to " + uri.getHost()); + LOG.log(Level.FINE, e, () -> "Got exception while sending request to " + url.getHost()); return Optional.empty(); } } @@ -129,33 +147,32 @@ public class ConfigServerRestExecutorImpl extends AbstractComponent implements C .orElse(""); } - private static HttpRequestBase createHttpBaseRequest(Method method, URI uri, InputStream data) throws ProxyException { + private static HttpRequestBase createHttpBaseRequest(Method method, URI url, InputStream data) { switch (method) { case GET: - return new HttpGet(uri); + return new HttpGet(url); case POST: - HttpPost post = new HttpPost(uri); + HttpPost post = new HttpPost(url); if (data != null) { post.setEntity(new InputStreamEntity(data)); } return post; case PUT: - HttpPut put = new HttpPut(uri); + HttpPut put = new HttpPut(url); if (data != null) { put.setEntity(new InputStreamEntity(data)); } return put; case DELETE: - return new HttpDelete(uri); + return new HttpDelete(url); case PATCH: - HttpPatch patch = new HttpPatch(uri); + HttpPatch patch = new HttpPatch(url); if (data != null) { patch.setEntity(new InputStreamEntity(data)); } return patch; - default: - throw new ProxyException(ErrorResponse.methodNotAllowed("Will not proxy such calls.")); } + throw new IllegalArgumentException("Refusing to proxy " + method + " " + url + ": Unsupported method"); } private static void copyHeaders(Map<String, List<String>> headers, HttpRequestBase toRequest) { @@ -169,7 +186,7 @@ public class ConfigServerRestExecutorImpl extends AbstractComponent implements C } /** - * During upgrade, one server can be down, this is normal. Therefor we do a quick ping on the first server, + * During upgrade, one server can be down, this is normal. Therefore we do a quick ping on the first server, * if it is not responding, we try the other servers first. False positive/negatives are not critical, * but will increase latency to some extent. */ @@ -178,15 +195,15 @@ public class ConfigServerRestExecutorImpl extends AbstractComponent implements C return false; } URI uri = allServers.get(0); - HttpGet httpget = new HttpGet(uri); + HttpGet httpGet = new HttpGet(uri); - int timeout = 500; RequestConfig config = RequestConfig.custom() - .setConnectTimeout(timeout) - .setConnectionRequestTimeout(timeout) - .setSocketTimeout(timeout).build(); - httpget.setConfig(config); - try (CloseableHttpResponse response = client.execute(httpget)) { + .setConnectTimeout((int) PING_REQUEST_TIMEOUT.toMillis()) + .setConnectionRequestTimeout((int) PING_REQUEST_TIMEOUT.toMillis()) + .setSocketTimeout((int) PING_REQUEST_TIMEOUT.toMillis()).build(); + httpGet.setConfig(config); + + try (CloseableHttpResponse response = client.execute(httpGet)) { if (response.getStatusLine().getStatusCode() == 200) { return false; } @@ -208,18 +225,25 @@ public class ConfigServerRestExecutorImpl extends AbstractComponent implements C } } - private static CloseableHttpClient createHttpClient(RequestConfig config, - ServiceIdentityProvider sslContextProvider, - HostnameVerifier hostnameVerifier) { + private static CloseableHttpClient createHttpClient(SSLContext sslContext, + HostnameVerifier hostnameVerifier, + org.apache.http.ConnectionReuseStrategy connectionReuseStrategy) { + + RequestConfig config = RequestConfig.custom() + .setConnectTimeout((int) PROXY_REQUEST_TIMEOUT.toMillis()) + .setConnectionRequestTimeout((int) PROXY_REQUEST_TIMEOUT.toMillis()) + .setSocketTimeout((int) PROXY_REQUEST_TIMEOUT.toMillis()).build(); return HttpClientBuilder.create() - .setUserAgent("config-server-proxy-client") - .setSSLContext(sslContextProvider.getIdentitySslContext()) - .setSSLHostnameVerifier(hostnameVerifier) - .setDefaultRequestConfig(config) - .setMaxConnPerRoute(10) - .setMaxConnTotal(500) - .setConnectionTimeToLive(1, TimeUnit.MINUTES) - .build(); + .setUserAgent("config-server-proxy-client") + .setSSLContext(sslContext) + .setSSLHostnameVerifier(hostnameVerifier) + .setDefaultRequestConfig(config) + .setMaxConnPerRoute(10) + .setMaxConnTotal(500) + .setConnectionReuseStrategy(connectionReuseStrategy) + .setConnectionTimeToLive(1, TimeUnit.MINUTES) + .build(); + } private static class ControllerOrConfigserverHostnameVerifier implements HostnameVerifier { @@ -242,4 +266,36 @@ public class ConfigServerRestExecutorImpl extends AbstractComponent implements C return "localhost".equals(hostname) || configserverVerifier.verify(hostname, session); } } + + /** + * A connection reuse strategy which avoids reusing connections to VIPs. Since VIPs are TCP-level load balancers, + * a reconnect is needed to (potentially) switch real server. + */ + public static class ConnectionReuseStrategy extends DefaultConnectionReuseStrategy { + + private final Set<String> vips; + + public ConnectionReuseStrategy(ZoneRegistry zoneRegistry) { + this(zoneRegistry.zones().all().ids().stream() + .map(zoneRegistry::getConfigServerVipUri) + .map(URI::getHost) + .collect(Collectors.toUnmodifiableSet())); + } + + public ConnectionReuseStrategy(Set<String> vips) { + this.vips = Set.copyOf(vips); + } + + @Override + public boolean keepAlive(HttpResponse response, HttpContext context) { + HttpCoreContext coreContext = HttpCoreContext.adapt(context); + String host = coreContext.getTargetHost().getHostName(); + if (vips.contains(host)) { + return false; + } + return super.keepAlive(response, context); + } + + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ErrorResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ErrorResponse.java deleted file mode 100644 index 3673c0227a3..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ErrorResponse.java +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.proxy; - -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.JsonFormat; -import com.yahoo.slime.Slime; - -import java.io.IOException; -import java.io.OutputStream; - -import static com.yahoo.jdisc.Response.Status.BAD_REQUEST; -import static com.yahoo.jdisc.Response.Status.INTERNAL_SERVER_ERROR; -import static com.yahoo.jdisc.Response.Status.METHOD_NOT_ALLOWED; -import static com.yahoo.jdisc.Response.Status.NOT_FOUND; - -/** - * Class for generating error responses. - * - * @author Haakon Dybdahl - */ -public class ErrorResponse extends HttpResponse { - - private final Slime slime = new Slime(); - public final String message; - - public ErrorResponse(int code, String errorType, String message) { - super(code); - this.message = message; - Cursor root = slime.setObject(); - root.setString("error-code", errorType); - root.setString("message", message); - } - - public enum errorCodes { - NOT_FOUND, - BAD_REQUEST, - METHOD_NOT_ALLOWED, - INTERNAL_SERVER_ERROR, - - } - - public static ErrorResponse notFoundError(String message) { - return new ErrorResponse(NOT_FOUND, errorCodes.NOT_FOUND.name(), message); - } - - public static ErrorResponse internalServerError(String message) { - return new ErrorResponse(INTERNAL_SERVER_ERROR, errorCodes.INTERNAL_SERVER_ERROR.name(), message); - } - - public static ErrorResponse badRequest(String message) { - return new ErrorResponse(BAD_REQUEST, errorCodes.BAD_REQUEST.name(), message); - } - - public static ErrorResponse methodNotAllowed(String message) { - return new ErrorResponse(METHOD_NOT_ALLOWED, errorCodes.METHOD_NOT_ALLOWED.name(), message); - } - - @Override - public void render(OutputStream stream) throws IOException { - new JsonFormat(true).encode(stream, slime); - } - - @Override - public String getContentType() { return "application/json"; } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyException.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyException.java deleted file mode 100644 index aa828bc0c83..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyException.java +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.proxy; - -/** - * Exceptions related to proxying calls to config servers. - * - * @author Haakon Dybdahl - */ -public class ProxyException extends Exception { - public final ErrorResponse errorResponse; - - public ProxyException(ErrorResponse errorResponse) { - super(errorResponse.message); - this.errorResponse = errorResponse; - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequest.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequest.java index f398683567b..a6314df9581 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequest.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequest.java @@ -28,33 +28,21 @@ public class ProxyRequest { private final List<URI> targets; private final String targetPath; - /** - * The constructor calls exception if the request is invalid. - * - * @param request the request from the jdisc framework. - * @param targets list of targets this request should be proxied to (targets are tried once in order until a response is returned). - * @param targetPath the path to proxy to. - * @throws ProxyException on errors - */ - public ProxyRequest(HttpRequest request, List<URI> targets, String targetPath) throws ProxyException { - this(request.getMethod(), request.getUri(), request.getJDiscRequest().headers(), request.getData(), - targets, targetPath); - } - - ProxyRequest(Method method, URI requestUri, Map<String, List<String>> headers, InputStream body, - List<URI> targets, String targetPath) throws ProxyException { - Objects.requireNonNull(requestUri, "Request must be non-null"); - if (!requestUri.getPath().endsWith(targetPath)) - throw new ProxyException(ErrorResponse.badRequest(String.format( - "Request path '%s' does not end with proxy path '%s'", requestUri.getPath(), targetPath))); - + ProxyRequest(Method method, URI url, Map<String, List<String>> headers, InputStream body, List<URI> targets, + String path) { + Objects.requireNonNull(url); + if (!url.getPath().endsWith(path)) { + throw new IllegalArgumentException(String.format("Request path '%s' does not end with proxy path '%s'", url.getPath(), path)); + } + if (targets.isEmpty()) { + throw new IllegalArgumentException("targets must be non-empty"); + } this.method = Objects.requireNonNull(method); - this.requestUri = Objects.requireNonNull(requestUri); + this.requestUri = Objects.requireNonNull(url); this.headers = Objects.requireNonNull(headers); this.requestData = body; - this.targets = List.copyOf(targets); - this.targetPath = targetPath.startsWith("/") ? targetPath : "/" + targetPath; + this.targetPath = path.startsWith("/") ? path : "/" + path; } @@ -100,4 +88,16 @@ public class ProxyRequest { return "[targets: " + targets + " request: " + targetPath + "]"; } + /** Create a proxy request that tries all given targets in order */ + public static ProxyRequest tryAll(List<URI> targets, String path, HttpRequest request) { + return new ProxyRequest(request.getMethod(), request.getUri(), request.getJDiscRequest().headers(), + request.getData(), targets, path); + } + + /** Create a proxy request that repeatedly tries a single target */ + public static ProxyRequest tryOne(URI target, String path, HttpRequest request) { + return new ProxyRequest(request.getMethod(), request.getUri(), request.getJDiscRequest().headers(), + request.getData(), List.of(target), path); + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/configserver/ConfigServerApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/configserver/ConfigServerApiHandler.java index 6ffdea93a1c..f65f7534476 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/configserver/ConfigServerApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/configserver/ConfigServerApiHandler.java @@ -15,7 +15,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.ServiceRegistry; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import com.yahoo.vespa.hosted.controller.auditlog.AuditLoggingRequestHandler; import com.yahoo.vespa.hosted.controller.proxy.ConfigServerRestExecutor; -import com.yahoo.vespa.hosted.controller.proxy.ProxyException; import com.yahoo.vespa.hosted.controller.proxy.ProxyRequest; import com.yahoo.yolean.Exceptions; @@ -95,11 +94,7 @@ public class ConfigServerApiHandler extends AuditLoggingRequestHandler { "' through /configserver/v1, following APIs are permitted: " + String.join(", ", WHITELISTED_APIS)); } - try { - return proxy.handle(new ProxyRequest(request, List.of(getEndpoint(zoneId)), cfgPath)); - } catch (ProxyException e) { - throw new RuntimeException(e); - } + return proxy.handle(ProxyRequest.tryOne(getEndpoint(zoneId), cfgPath, request)); } private HttpResponse root(HttpRequest request) { @@ -126,4 +121,5 @@ public class ConfigServerApiHandler extends AuditLoggingRequestHandler { private URI getEndpoint(ZoneId zoneId) { return CONTROLLER_ZONE.equals(zoneId) ? CONTROLLER_URI : zoneRegistry.getConfigServerVipUri(zoneId); } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java index a127a44efb2..7c44ae6d1a5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java @@ -16,12 +16,9 @@ import com.yahoo.vespa.hosted.controller.api.integration.ServiceRegistry; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import com.yahoo.vespa.hosted.controller.auditlog.AuditLoggingRequestHandler; import com.yahoo.vespa.hosted.controller.proxy.ConfigServerRestExecutor; -import com.yahoo.vespa.hosted.controller.proxy.ProxyException; import com.yahoo.vespa.hosted.controller.proxy.ProxyRequest; import com.yahoo.yolean.Exceptions; -import java.net.URI; -import java.util.List; import java.util.logging.Level; /** @@ -84,11 +81,7 @@ public class ZoneApiHandler extends AuditLoggingRequestHandler { if ( ! zoneRegistry.hasZone(zoneId)) { throw new IllegalArgumentException("No such zone: " + zoneId.value()); } - try { - return proxy.handle(new ProxyRequest(request, getConfigserverEndpoints(zoneId), path.getRest())); - } catch (ProxyException e) { - throw new RuntimeException(e); - } + return proxy.handle(proxyRequest(zoneId, path.getRest(), request)); } private HttpResponse root(HttpRequest request) { @@ -114,13 +107,12 @@ public class ZoneApiHandler extends AuditLoggingRequestHandler { return ErrorResponse.notFoundError("Nothing at " + path); } - private List<URI> getConfigserverEndpoints(ZoneId zoneId) { + private ProxyRequest proxyRequest(ZoneId zoneId, String path, HttpRequest request) { // TODO: Use config server VIP for all zones that have one if (zoneId.region().value().startsWith("aws-") || zoneId.region().value().contains("-aws-")) { - return List.of(zoneRegistry.getConfigServerVipUri(zoneId)); - } else { - return zoneRegistry.getConfigServerUris(zoneId); + return ProxyRequest.tryOne(zoneRegistry.getConfigServerVipUri(zoneId), path, request); } + return ProxyRequest.tryAll(zoneRegistry.getConfigServerUris(zoneId), path, request); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerProxyMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerProxyMock.java index d6e1af07938..d481aaa2c77 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerProxyMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerProxyMock.java @@ -20,10 +20,10 @@ public class ConfigServerProxyMock extends AbstractComponent implements ConfigSe private volatile String requestBody = null; @Override - public HttpResponse handle(ProxyRequest proxyRequest) { - lastReceived = proxyRequest; + public HttpResponse handle(ProxyRequest request) { + lastReceived = request; // Copy request body as the input stream is drained once the request completes - requestBody = asString(proxyRequest.getData()); + requestBody = asString(request.getData()); return new StringResponse("ok"); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImplTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImplTest.java new file mode 100644 index 00000000000..1fce7ba5695 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImplTest.java @@ -0,0 +1,123 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.proxy; + +import com.github.tomakehurst.wiremock.junit.WireMockRule; +import com.github.tomakehurst.wiremock.stubbing.Scenario; +import com.yahoo.config.provision.SystemName; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.vespa.hosted.controller.integration.ZoneRegistryMock; +import org.apache.http.protocol.HttpContext; +import org.apache.http.protocol.HttpCoreContext; +import org.junit.Rule; +import org.junit.Test; + +import javax.net.ssl.SSLContext; +import java.io.ByteArrayOutputStream; +import java.net.URI; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static com.github.tomakehurst.wiremock.client.WireMock.getRequestedFor; +import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; +import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.options; +import static org.junit.Assert.assertEquals; + +/** + * @author mpolden + */ +public class ConfigServerRestExecutorImplTest { + + @Rule + public final WireMockRule wireMock = new WireMockRule(options().dynamicPort(), true); + + @Test + public void proxy_with_retries() throws Exception { + var connectionReuseStrategy = new CountingConnectionReuseStrategy(Set.of("127.0.0.1")); + var proxy = new ConfigServerRestExecutorImpl(new ZoneRegistryMock(SystemName.cd), SSLContext.getDefault(), + (duration) -> {}, connectionReuseStrategy); + + URI url = url(); + String path = url.getPath(); + stubRequests(path); + + HttpRequest request = HttpRequest.createTestRequest(url.toString(), com.yahoo.jdisc.http.HttpRequest.Method.GET); + ProxyRequest proxyRequest = ProxyRequest.tryOne(url, path, request); + + // Request is retried + HttpResponse response = proxy.handle(proxyRequest); + wireMock.verify(3, getRequestedFor(urlEqualTo(path))); + assertEquals(200, response.getStatus()); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + response.render(out); + assertEquals("OK", out.toString()); + + // No connections are reused as host is a VIP + assertEquals(0, connectionReuseStrategy.reusedConnections.get(url.getHost()).intValue()); + } + + @Test + public void proxy_without_connection_reuse() throws Exception { + var connectionReuseStrategy = new CountingConnectionReuseStrategy(Set.of()); + var proxy = new ConfigServerRestExecutorImpl(new ZoneRegistryMock(SystemName.cd), SSLContext.getDefault(), + (duration) -> {}, connectionReuseStrategy); + + URI url = url(); + String path = url.getPath(); + stubRequests(path); + + HttpRequest request = HttpRequest.createTestRequest(url.toString(), com.yahoo.jdisc.http.HttpRequest.Method.GET); + ProxyRequest proxyRequest = ProxyRequest.tryOne(url, path, request); + + // Connections are reused + assertEquals(200, proxy.handle(proxyRequest).getStatus()); + assertEquals(3, connectionReuseStrategy.reusedConnections.get(url.getHost()).intValue()); + } + + private URI url() { + return URI.create("http://127.0.0.1:" + wireMock.port() + "/"); + } + + private void stubRequests(String path) { + String retryScenario = "Retry scenario"; + String retryRequest = "Retry request 1"; + String retryRequestAgain = "Retry request 2"; + + wireMock.stubFor(get(urlEqualTo(path)).inScenario(retryScenario) + .whenScenarioStateIs(Scenario.STARTED) + .willSetStateTo(retryRequest) + .willReturn(aResponse().withStatus(500))); + + wireMock.stubFor(get(urlEqualTo(path)).inScenario(retryScenario) + .whenScenarioStateIs(retryRequest) + .willSetStateTo(retryRequestAgain) + .willReturn(aResponse().withStatus(500))); + + wireMock.stubFor(get(urlEqualTo(path)).inScenario(retryScenario) + .whenScenarioStateIs(retryRequestAgain) + .willReturn(aResponse().withBody("OK"))); + } + + private static class CountingConnectionReuseStrategy extends ConfigServerRestExecutorImpl.ConnectionReuseStrategy { + + private final Map<String, Integer> reusedConnections = new HashMap<>(); + + public CountingConnectionReuseStrategy(Set<String> vips) { + super(vips); + } + + @Override + public boolean keepAlive(org.apache.http.HttpResponse response, HttpContext context) { + boolean keepAlive = super.keepAlive(response, context); + String host = HttpCoreContext.adapt(context).getTargetHost().getHostName(); + reusedConnections.putIfAbsent(host, 0); + if (keepAlive) reusedConnections.compute(host, (ignored, count) -> ++count); + return keepAlive; + } + + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequestTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequestTest.java index d8373cb8928..15ec138354d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequestTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyRequestTest.java @@ -21,12 +21,6 @@ public class ProxyRequestTest { public final ExpectedException exception = ExpectedException.none(); @Test - public void testEmpty() throws Exception { - exception.expectMessage("Request must be non-null"); - new ProxyRequest(HttpRequest.Method.GET, null, Map.of(), null, List.of(), "/zone/v2"); - } - - @Test public void testBadUri() throws Exception { exception.expectMessage("Request path '/path' does not end with proxy path '/zone/v2/'"); testRequest("http://domain.tld/path", "/zone/v2/"); @@ -67,8 +61,9 @@ public class ProxyRequestTest { } } - private static ProxyRequest testRequest(String url, String pathPrefix) throws ProxyException { - return new ProxyRequest( - HttpRequest.Method.GET, URI.create(url), Map.of(), null, List.of(), pathPrefix); + private static ProxyRequest testRequest(String url, String pathPrefix) { + return new ProxyRequest(HttpRequest.Method.GET, URI.create(url), Map.of(), null, + List.of(URI.create("http://example.com")), pathPrefix); } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponseTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponseTest.java index 0aac59321b5..539d4a6dd75 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponseTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponseTest.java @@ -20,7 +20,7 @@ public class ProxyResponseTest { @Test public void testRewriteUrl() throws Exception { ProxyRequest request = new ProxyRequest(HttpRequest.Method.GET, URI.create("http://domain.tld/zone/v2/dev/us-north-1/configserver"), - Map.of(), null, List.of(), "configserver"); + Map.of(), null, List.of(URI.create("http://example.com")), "configserver"); ProxyResponse proxyResponse = new ProxyResponse( request, "response link is http://configserver:1234/bla/bla/", @@ -38,7 +38,7 @@ public class ProxyResponseTest { @Test public void testRewriteSecureUrl() throws Exception { ProxyRequest request = new ProxyRequest(HttpRequest.Method.GET, URI.create("https://domain.tld/zone/v2/prod/eu-south-3/configserver"), - Map.of(), null, List.of(), "configserver"); + Map.of(), null, List.of(URI.create("http://example.com")), "configserver"); ProxyResponse proxyResponse = new ProxyResponse( request, "response link is http://configserver:1234/bla/bla/", @@ -52,4 +52,5 @@ public class ProxyResponseTest { String document = new String(outputStream.toByteArray(), StandardCharsets.UTF_8); assertEquals("response link is https://domain.tld/zone/v2/prod/eu-south-3/bla/bla/", document); } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java index 32a77137182..f3c24458e6e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java @@ -73,8 +73,8 @@ public class DeploymentApiTest extends ControllerContainerTest { productionApp.submit(multiInstancePackage).failDeployment(JobType.systemTest); tester.controller().updateVersionStatus(censorConfigServers(VersionStatus.compute(tester.controller()))); - tester.assertResponse(authenticatedRequest("http://localhost:8080/deployment/v1/"), new File("root.json")); - tester.assertResponse(authenticatedRequest("http://localhost:8080/api/deployment/v1/"), new File("root.json")); + tester.assertResponse(operatorRequest("http://localhost:8080/deployment/v1/"), new File("root.json")); + tester.assertResponse(operatorRequest("http://localhost:8080/api/deployment/v1/"), new File("root.json")); } private VersionStatus censorConfigServers(VersionStatus versionStatus) { diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index efe86bb6d55..a9ded0f5fd2 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -246,11 +246,6 @@ public class Flags { "Takes effect on next application redeploy", APPLICATION_ID); - public static final UnboundLongFlag CONFIGSERVER_LOCAL_SESSIONS_EXPIRY_INTERVAL_IN_DAYS = defineLongFlag( - "configserver-local-sessions-expiry-interval-in-days", 1, - "Expiry time for expired local sessions in config server", - "Takes effect on next run of config server maintainer SessionsMaintainer"); - public static final UnboundBooleanFlag USE_CLOUD_INIT_FORMAT = defineFeatureFlag( "use-cloud-init", false, "Use the cloud-init format when provisioning hosts", diff --git a/http-utils/src/main/java/ai/vespa/util/http/retry/Sleeper.java b/http-utils/src/main/java/ai/vespa/util/http/retry/Sleeper.java index 06a7359f307..25f5b9eb627 100644 --- a/http-utils/src/main/java/ai/vespa/util/http/retry/Sleeper.java +++ b/http-utils/src/main/java/ai/vespa/util/http/retry/Sleeper.java @@ -8,7 +8,7 @@ import java.time.Duration; * * @author bjorncs */ -interface Sleeper { +public interface Sleeper { void sleep(Duration duration); class Default implements Sleeper { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java index b5e36abd076..b41820a461b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java @@ -650,7 +650,7 @@ public class NodeRepository extends AbstractComponent { try (Mutex lock = lockUnallocated()) { requireRemovable(node, false, force); - if (node.type() == NodeType.host) { + if (node.type().isDockerHost()) { List<Node> children = list().childrenOf(node).asList(); children.forEach(child -> requireRemovable(child, true, force)); db.removeNodes(children); @@ -665,8 +665,9 @@ public class NodeRepository extends AbstractComponent { return removed; } else { - db.removeNodes(List.of(node)); - return List.of(node); + List<Node> removed = List.of(node); + db.removeNodes(removed); + return removed; } } } @@ -681,8 +682,8 @@ public class NodeRepository extends AbstractComponent { /** * Throws if the given node cannot be removed. Removal is allowed if: * - Tenant node: node is unallocated - * - Non-Docker-container node: iff in state provisioned|failed|parked - * - Docker-container-node: + * - Host node: iff in state provisioned|failed|parked + * - Child node: * If only removing the container node: node in state ready * If also removing the parent node: child is in state provisioned|failed|parked|dirty|ready */ @@ -694,7 +695,7 @@ public class NodeRepository extends AbstractComponent { if (node.flavor().getType() == Flavor.Type.DOCKER_CONTAINER && !removingAsChild) { if (node.state() != State.ready) - illegal(node + " can not be removed as it is not in the state [ready]"); + illegal(node + " can not be removed as it is not in the state " + State.ready); } else if (node.flavor().getType() == Flavor.Type.DOCKER_CONTAINER) { // removing a child node Set<State> legalStates = EnumSet.of(State.provisioned, State.failed, State.parked, State.dirty, State.ready); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTest.java index 97f1eda866a..9d5e7691fe8 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTest.java @@ -32,7 +32,7 @@ import static org.junit.Assert.fail; public class NodeRepositoryTest { @Test - public void nodeRepositoryTest() { + public void add_and_remove() { NodeRepositoryTester tester = new NodeRepositoryTester(); assertEquals(0, tester.nodeRepository().getNodes().size()); @@ -120,13 +120,8 @@ public class NodeRepositoryTest { tester.addNode("node11", "node11", "host1", "docker", NodeType.tenant); tester.addNode("node12", "node12", "host1", "docker", NodeType.tenant); tester.addNode("node20", "node20", "host2", "docker", NodeType.tenant); - assertEquals(6, tester.nodeRepository().getNodes().size()); - - Node node = tester.nodeRepository().getNode("host1").get(); - IP.Config cfg = new IP.Config(Set.of("127.0.0.1"), Set.of()); - node = node.with(cfg); - tester.setNodeState("node11", Node.State.active); + assertEquals(6, tester.nodeRepository().getNodes().size()); try { tester.nodeRepository().removeRecursively("host1"); @@ -152,6 +147,32 @@ public class NodeRepositoryTest { } @Test + public void delete_config_host() { + NodeRepositoryTester tester = new NodeRepositoryTester(); + + String cfghost1 = "cfghost1"; + String cfg1 = "cfg1"; + tester.addNode("id1", cfghost1, "default", NodeType.confighost); + tester.addNode("id2", cfg1, cfghost1, "docker", NodeType.config); + tester.setNodeState(cfghost1, Node.State.active); + tester.setNodeState(cfg1, Node.State.active); + assertEquals(2, tester.nodeRepository().getNodes().size()); + + try { + tester.nodeRepository().removeRecursively(cfghost1); + fail("Should not be able to delete host node, one of the children is in state active"); + } catch (IllegalArgumentException ignored) { } + assertEquals(2, tester.nodeRepository().getNodes().size()); + + // Fail host and container + tester.nodeRepository().failRecursively(cfghost1, Agent.system, getClass().getSimpleName()); + + // Remove recursively + tester.nodeRepository().removeRecursively(cfghost1); + assertEquals(0, tester.nodeRepository().list().not().state(Node.State.deprovisioned).size()); + } + + @Test public void relevant_information_from_deprovisioned_hosts_are_merged_into_readded_host() { NodeRepositoryTester tester = new NodeRepositoryTester(); Instant testStart = tester.nodeRepository().clock().instant(); diff --git a/searchcore/src/tests/proton/matching/querynodes_test.cpp b/searchcore/src/tests/proton/matching/querynodes_test.cpp index 896b2f7e07f..c1247b630a3 100644 --- a/searchcore/src/tests/proton/matching/querynodes_test.cpp +++ b/searchcore/src/tests/proton/matching/querynodes_test.cpp @@ -48,6 +48,7 @@ using search::query::QueryBuilder; using search::queryeval::AndNotSearch; using search::queryeval::AndSearch; using search::queryeval::Blueprint; +using search::queryeval::ChildrenIterators; using search::queryeval::ElementIteratorWrapper; using search::queryeval::ElementIterator; using search::queryeval::EmptySearch; @@ -104,12 +105,12 @@ public: explicit Create(bool strict = true) : _strict(strict) {} Create &add(SearchIterator *s) { - _children.push_back(s); + _children.emplace_back(s); return *this; } - operator SearchIterator *() const { - return SearchType::create(_children, _strict); + operator SearchIterator *() { + return SearchType::create(std::move(_children), _strict).release(); } }; typedef Create<OrSearch> MyOr; @@ -143,9 +144,11 @@ public: return *this; } - operator SearchIterator *() const { + operator SearchIterator *() { return SourceBlenderSearch::create( - ISourceSelectorDummy::makeDummyIterator(), _children, _strict); + ISourceSelectorDummy::makeDummyIterator(), + _children, + _strict).release(); } }; @@ -245,20 +248,20 @@ SearchIterator *getLeaf(const string &fld, const string &tag) { template <> SearchIterator *getLeaf<Phrase>(const string &fld, const string &tag) { SimplePhraseSearch::Children children; - children.push_back(getTerm(phrase_term1, fld, tag)); - children.push_back(getTerm(phrase_term2, fld, tag)); + children.emplace_back(getTerm(phrase_term1, fld, tag)); + children.emplace_back(getTerm(phrase_term2, fld, tag)); static TermFieldMatchData tmd; TermFieldMatchDataArray tfmda; tfmda.add(&tmd).add(&tmd); vector<uint32_t> eval_order(2); - return new SimplePhraseSearch(children, MatchData::UP(), tfmda, eval_order, tmd, true); + return new SimplePhraseSearch(std::move(children), MatchData::UP(), tfmda, eval_order, tmd, true); } template <typename NearType> SearchIterator *getNearParent(SearchIterator *a, SearchIterator *b) { typename NearType::Children children; - children.push_back(a); - children.push_back(b); + children.emplace_back(a); + children.emplace_back(b); TermFieldMatchDataArray data; static TermFieldMatchData tmd; // we only check how many term/field combinations @@ -266,15 +269,15 @@ SearchIterator *getNearParent(SearchIterator *a, SearchIterator *b) { // two terms searching in (two index fields + two attribute fields) data.add(&tmd).add(&tmd).add(&tmd).add(&tmd) .add(&tmd).add(&tmd).add(&tmd).add(&tmd); - return new NearType(children, data, distance, true); + return new NearType(std::move(children), data, distance, true); } template <typename SearchType> SearchIterator *getSimpleParent(SearchIterator *a, SearchIterator *b) { typename SearchType::Children children; - children.push_back(a); - children.push_back(b); - return SearchType::create(children, true); + children.emplace_back(a); + children.emplace_back(b); + return SearchType::create(std::move(children), true).release(); } template <typename T> diff --git a/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt index 7a9426ae716..7e1b1fb1e9a 100644 --- a/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt @@ -26,3 +26,7 @@ vespa_add_library(searchcore_pcommon STATIC searchcore_fconfig ${VESPA_STDCXX_FS_LIB} ) + +if(VESPA_OS_DISTRO_COMBINED STREQUAL "rhel 8.2") + set_source_files_properties(hw_info_sampler.cpp PROPERTIES COMPILE_FLAGS -DRHEL_8_2_KLUDGE) +endif() diff --git a/searchcore/src/vespa/searchcore/proton/common/hw_info_sampler.cpp b/searchcore/src/vespa/searchcore/proton/common/hw_info_sampler.cpp index b381ee9122f..cdec0b440c4 100644 --- a/searchcore/src/vespa/searchcore/proton/common/hw_info_sampler.cpp +++ b/searchcore/src/vespa/searchcore/proton/common/hw_info_sampler.cpp @@ -157,3 +157,16 @@ HwInfoSampler::sampleDiskWriteSpeed(const vespalib::string &path, const Config & } } + +#ifdef RHEL_8_2_KLUDGE + +// Kludge to avoid unresolved symbols with gcc-toolset-9 on RHEL 8.2 +#include <codecvt> + +namespace std { + +template class codecvt_utf8<wchar_t>; + +} + +#endif diff --git a/searchlib/src/tests/queryeval/blueprint/blueprint_test.cpp b/searchlib/src/tests/queryeval/blueprint/blueprint_test.cpp index 6625a4a09ce..0672e51378e 100644 --- a/searchlib/src/tests/queryeval/blueprint/blueprint_test.cpp +++ b/searchlib/src/tests/queryeval/blueprint/blueprint_test.cpp @@ -39,11 +39,11 @@ public: return true; } - virtual SearchIterator::UP - createIntermediateSearch(const MultiSearch::Children &subSearches, + SearchIterator::UP + createIntermediateSearch(MultiSearch::Children subSearches, bool strict, MatchData &md) const override { - return SearchIterator::UP(new MySearch("or", subSearches, &md, strict)); + return SearchIterator::UP(new MySearch("or", std::move(subSearches), &md, strict)); } static MyOr& create() { return *(new MyOr()); } @@ -56,11 +56,11 @@ class OtherOr : public OrBlueprint { private: public: - virtual SearchIterator::UP - createIntermediateSearch(const MultiSearch::Children &subSearches, + SearchIterator::UP + createIntermediateSearch(MultiSearch::Children subSearches, bool strict, MatchData &md) const override { - return SearchIterator::UP(new MySearch("or", subSearches, &md, strict)); + return SearchIterator::UP(new MySearch("or", std::move(subSearches), &md, strict)); } static OtherOr& create() { return *(new OtherOr()); } @@ -86,11 +86,11 @@ public: return (i == 0); } - virtual SearchIterator::UP - createIntermediateSearch(const MultiSearch::Children &subSearches, + SearchIterator::UP + createIntermediateSearch(MultiSearch::Children subSearches, bool strict, MatchData &md) const override { - return SearchIterator::UP(new MySearch("and", subSearches, &md, strict)); + return SearchIterator::UP(new MySearch("and", std::move(subSearches), &md, strict)); } static MyAnd& create() { return *(new MyAnd()); } @@ -103,11 +103,11 @@ class OtherAnd : public AndBlueprint { private: public: - virtual SearchIterator::UP - createIntermediateSearch(const MultiSearch::Children &subSearches, + SearchIterator::UP + createIntermediateSearch(MultiSearch::Children subSearches, bool strict, MatchData &md) const override { - return SearchIterator::UP(new MySearch("and", subSearches, &md, strict)); + return SearchIterator::UP(new MySearch("and", std::move(subSearches), &md, strict)); } static OtherAnd& create() { return *(new OtherAnd()); } @@ -118,11 +118,11 @@ public: class OtherAndNot : public AndNotBlueprint { public: - virtual SearchIterator::UP - createIntermediateSearch(const MultiSearch::Children &subSearches, + SearchIterator::UP + createIntermediateSearch(MultiSearch::Children subSearches, bool strict, MatchData &md) const override { - return SearchIterator::UP(new MySearch("andnot", subSearches, &md, strict)); + return SearchIterator::UP(new MySearch("andnot", std::move(subSearches), &md, strict)); } static OtherAndNot& create() { return *(new OtherAndNot()); } diff --git a/searchlib/src/tests/queryeval/blueprint/mysearch.h b/searchlib/src/tests/queryeval/blueprint/mysearch.h index 1f2057a4a56..012e19f26f5 100644 --- a/searchlib/src/tests/queryeval/blueprint/mysearch.h +++ b/searchlib/src/tests/queryeval/blueprint/mysearch.h @@ -9,11 +9,9 @@ namespace search::queryeval { //----------------------------------------------------------------------------- -class MySearch : public SearchIterator +class MySearch : public MultiSearch { public: - typedef MultiSearch::Children Children; - typedef std::vector<SearchIterator::UP> MyChildren; typedef search::fef::TermFieldMatchDataArray TFMDA; typedef search::fef::MatchData MatchData; @@ -21,7 +19,6 @@ private: vespalib::string _tag; bool _isLeaf; bool _isStrict; - MyChildren _children; TFMDA _match; MatchData *_md; @@ -33,21 +30,18 @@ protected: public: MySearch(const std::string &tag, bool leaf, bool strict) - : _tag(tag), _isLeaf(leaf), _isStrict(strict), _children(), + : _tag(tag), _isLeaf(leaf), _isStrict(strict), _match(), _md(0) {} MySearch(const std::string &tag, const TFMDA &tfmda, bool strict) - : _tag(tag), _isLeaf(true), _isStrict(strict), _children(), + : _tag(tag), _isLeaf(true), _isStrict(strict), _match(tfmda), _md(0) {} - MySearch(const std::string &tag, const Children &children, + MySearch(const std::string &tag, Children children, MatchData *md, bool strict) - : _tag(tag), _isLeaf(false), _isStrict(strict), _children(), - _match(), _md(md) { - for (size_t i(0); i < children.size(); i++) { - _children.emplace_back(children[i]); - } - } + : MultiSearch(std::move(children)), + _tag(tag), _isLeaf(false), _isStrict(strict), + _match(), _md(md) {} MySearch &add(SearchIterator *search) { _children.emplace_back(search); @@ -98,7 +92,7 @@ public: visit(visitor, "_tag", _tag); visit(visitor, "_isLeaf", _isLeaf); visit(visitor, "_isStrict", _isStrict); - visit(visitor, "_children", _children); + MultiSearch::visitMembers(visitor); visit(visitor, "_handles", _handles); } diff --git a/searchlib/src/tests/queryeval/monitoring_search_iterator/monitoring_search_iterator_test.cpp b/searchlib/src/tests/queryeval/monitoring_search_iterator/monitoring_search_iterator_test.cpp index e863cbe7a73..0e0840d9013 100644 --- a/searchlib/src/tests/queryeval/monitoring_search_iterator/monitoring_search_iterator_test.cpp +++ b/searchlib/src/tests/queryeval/monitoring_search_iterator/monitoring_search_iterator_test.cpp @@ -77,16 +77,18 @@ struct TreeFixture : _itr() { MultiSearch::Children children; - children.push_back(new MonitoringSearchIterator("child1", - SearchIterator::UP - (new SimpleSearch(SimpleResult().addHit(2).addHit(4).addHit(6))), - false)); - children.push_back(new MonitoringSearchIterator("child2", - SearchIterator::UP - (new SimpleSearch(SimpleResult().addHit(3).addHit(4).addHit(5))), + children.emplace_back( + new MonitoringSearchIterator("child1", + SearchIterator::UP + (new SimpleSearch(SimpleResult().addHit(2).addHit(4).addHit(6))), + false)); + children.emplace_back( + new MonitoringSearchIterator("child2", + SearchIterator::UP + (new SimpleSearch(SimpleResult().addHit(3).addHit(4).addHit(5))), false)); _itr.reset(new MonitoringSearchIterator("and", - SearchIterator::UP(AndSearch::create(children, true)), + SearchIterator::UP(AndSearch::create(std::move(children), true)), false)); _res.search(*_itr); } diff --git a/searchlib/src/tests/queryeval/multibitvectoriterator/multibitvectoriterator_bench.cpp b/searchlib/src/tests/queryeval/multibitvectoriterator/multibitvectoriterator_bench.cpp index 6e0baf3b2e8..e1d0e034a89 100644 --- a/searchlib/src/tests/queryeval/multibitvectoriterator/multibitvectoriterator_bench.cpp +++ b/searchlib/src/tests/queryeval/multibitvectoriterator/multibitvectoriterator_bench.cpp @@ -98,9 +98,9 @@ Test::testSearch(bool strict) TermFieldMatchData tfmd; MultiSearch::Children andd; for (size_t i(0); i < _bvs.size(); i++) { - andd.push_back(BitVectorIterator::create(_bvs[i].get(), tfmd, strict, false).release()); + andd.push_back(BitVectorIterator::create(_bvs[i].get(), tfmd, strict, false)); } - SearchIterator::UP s(T::create(andd, strict)); + SearchIterator::UP s = T::create(std::move(andd), strict); if (_optimize) { LOG(info, "Optimizing iterator"); s = MultiBitVectorIteratorBase::optimize(std::move(s)); diff --git a/searchlib/src/tests/queryeval/multibitvectoriterator/multibitvectoriterator_test.cpp b/searchlib/src/tests/queryeval/multibitvectoriterator/multibitvectoriterator_test.cpp index 296d06d0b7b..565c013a2fe 100644 --- a/searchlib/src/tests/queryeval/multibitvectoriterator/multibitvectoriterator_test.cpp +++ b/searchlib/src/tests/queryeval/multibitvectoriterator/multibitvectoriterator_test.cpp @@ -125,10 +125,10 @@ Test::testAndWith(bool invert) TermFieldMatchData tfmd; { MultiSearch::Children children; - children.push_back(createIter(0, invert, tfmd, false).release()); - children.push_back(createIter(1, invert, tfmd, false).release()); + children.push_back(createIter(0, invert, tfmd, false)); + children.push_back(createIter(1, invert, tfmd, false)); - SearchIterator::UP s(AndSearch::create(children, false)); + SearchIterator::UP s = AndSearch::create(std::move(children), false); s = MultiBitVectorIteratorBase::optimize(std::move(s)); s->initFullRange(); @@ -137,10 +137,10 @@ Test::testAndWith(bool invert) H lastHits2F = seekNoReset(*s, 130, _bvs[0]->size()); children.clear(); - children.push_back(createIter(0, invert, tfmd, false).release()); - children.push_back(createIter(1, invert, tfmd, false).release()); - children.push_back(createIter(2, invert, tfmd, false).release()); - s.reset(AndSearch::create(children, false)); + children.push_back(createIter(0, invert, tfmd, false)); + children.push_back(createIter(1, invert, tfmd, false)); + children.push_back(createIter(2, invert, tfmd, false)); + s = AndSearch::create(std::move(children), false); s = MultiBitVectorIteratorBase::optimize(std::move(s)); s->initFullRange(); H firstHits3 = seekNoReset(*s, 1, 130); @@ -197,12 +197,12 @@ Test::testBug7163266() MultiSearch::Children children; UnpackInfo unpackInfo; for (size_t i(0); i < 28; i++) { - children.push_back(new TrueSearch(tfmd[2])); + children.emplace_back(new TrueSearch(tfmd[2])); unpackInfo.add(i); } - children.push_back(createIter(0, false, tfmd[0], false).release()); - children.push_back(createIter(1, false, tfmd[1], false).release()); - SearchIterator::UP s(AndSearch::create(children, false, unpackInfo)); + children.push_back(createIter(0, false, tfmd[0], false)); + children.push_back(createIter(1, false, tfmd[1], false)); + SearchIterator::UP s = AndSearch::create(std::move(children), false, unpackInfo); const MultiSearch * ms = dynamic_cast<const MultiSearch *>(s.get()); EXPECT_TRUE(ms != nullptr); EXPECT_EQUAL(30u, ms->getChildren().size()); @@ -233,14 +233,14 @@ Test::testThatOptimizePreservesUnpack() _bvs[1]->setBit(1); _bvs[2]->setBit(1); MultiSearch::Children children; - children.push_back(createIter(0, false, tfmd[0], false).release()); - children.push_back(createIter(1, false, tfmd[1], false).release()); - children.push_back(new TrueSearch(tfmd[2])); - children.push_back(createIter(2, false, tfmd[3], false).release()); + children.push_back(createIter(0, false, tfmd[0], false)); + children.push_back(createIter(1, false, tfmd[1], false)); + children.emplace_back(new TrueSearch(tfmd[2])); + children.push_back(createIter(2, false, tfmd[3], false)); UnpackInfo unpackInfo; unpackInfo.add(1); unpackInfo.add(2); - SearchIterator::UP s(T::create(children, false, unpackInfo)); + SearchIterator::UP s = T::create(std::move(children), false, unpackInfo); s->initFullRange(); const MultiSearch * ms = dynamic_cast<const MultiSearch *>(s.get()); EXPECT_TRUE(ms != nullptr); @@ -291,10 +291,10 @@ Test::verifyUnpackOfOr(const UnpackInfo &unpackInfo) { TermFieldMatchData tfmdA[3]; MultiSearch::Children children; - children.push_back(createIter(0, false, tfmdA[0], false).release()); - children.push_back(createIter(1, false, tfmdA[1], false).release()); - children.push_back(createIter(2, false, tfmdA[2], false).release()); - SearchIterator::UP s(OrSearch::create(children, false, unpackInfo)); + children.push_back(createIter(0, false, tfmdA[0], false)); + children.push_back(createIter(1, false, tfmdA[1], false)); + children.push_back(createIter(2, false, tfmdA[2], false)); + SearchIterator::UP s = OrSearch::create(std::move(children), false, unpackInfo); verifyOrUnpack(*s, tfmdA); for (auto & tfmd : tfmdA) { @@ -353,23 +353,23 @@ Test::testSearch(bool strict, bool invert) uint32_t docIdLimit(_bvs[0]->size()); { MultiSearch::Children children; - children.push_back(createIter(0, invert, tfmd, strict).release()); - SearchIterator::UP s(T::create(children, strict)); + children.push_back(createIter(0, invert, tfmd, strict)); + SearchIterator::UP s = T::create(std::move(children), strict); searchAndCompare(std::move(s), docIdLimit); } { MultiSearch::Children children; - children.push_back(createIter(0, invert, tfmd, strict).release()); - children.push_back(createIter(1, invert, tfmd, strict).release()); - SearchIterator::UP s(T::create(children, strict)); + children.push_back(createIter(0, invert, tfmd, strict)); + children.push_back(createIter(1, invert, tfmd, strict)); + SearchIterator::UP s = T::create(std::move(children), strict); searchAndCompare(std::move(s), docIdLimit); } { MultiSearch::Children children; - children.push_back(createIter(0, invert, tfmd, strict).release()); - children.push_back(createIter(1, invert, tfmd, strict).release()); - children.push_back(createIter(2, invert, tfmd, strict).release()); - SearchIterator::UP s(T::create(children, strict)); + children.push_back(createIter(0, invert, tfmd, strict)); + children.push_back(createIter(1, invert, tfmd, strict)); + children.push_back(createIter(2, invert, tfmd, strict)); + SearchIterator::UP s = T::create(std::move(children), strict); searchAndCompare(std::move(s), docIdLimit); } } @@ -382,79 +382,79 @@ Test::testOptimizeCommon(bool isAnd, bool invert) { MultiSearch::Children children; - children.push_back(createIter(0, invert, tfmd, false).release()); + children.push_back(createIter(0, invert, tfmd, false)); - SearchIterator::UP s(T::create(children, false)); + SearchIterator::UP s = T::create(std::move(children), false); s = MultiBitVectorIteratorBase::optimize(std::move(s)); EXPECT_TRUE(dynamic_cast<const T *>(s.get()) != nullptr); const MultiSearch & m(dynamic_cast<const MultiSearch &>(*s)); EXPECT_EQUAL(1u, m.getChildren().size()); - EXPECT_TRUE(dynamic_cast<const BitVectorIterator *>(m.getChildren()[0]) != nullptr); + EXPECT_TRUE(dynamic_cast<const BitVectorIterator *>(m.getChildren()[0].get()) != nullptr); } { MultiSearch::Children children; - children.push_back(createIter(0, invert, tfmd, false).release()); - children.push_back(new EmptySearch()); + children.push_back(createIter(0, invert, tfmd, false)); + children.emplace_back(new EmptySearch()); - SearchIterator::UP s(T::create(children, false)); + SearchIterator::UP s = T::create(std::move(children), false); s = MultiBitVectorIteratorBase::optimize(std::move(s)); EXPECT_TRUE(dynamic_cast<const T *>(s.get()) != nullptr); const MultiSearch & m(dynamic_cast<const MultiSearch &>(*s)); EXPECT_EQUAL(2u, m.getChildren().size()); - EXPECT_TRUE(dynamic_cast<const BitVectorIterator *>(m.getChildren()[0]) != nullptr); - EXPECT_TRUE(dynamic_cast<const EmptySearch *>(m.getChildren()[1]) != nullptr); + EXPECT_TRUE(dynamic_cast<const BitVectorIterator *>(m.getChildren()[0].get()) != nullptr); + EXPECT_TRUE(dynamic_cast<const EmptySearch *>(m.getChildren()[1].get()) != nullptr); } { MultiSearch::Children children; - children.push_back(new EmptySearch()); - children.push_back(createIter(0, invert, tfmd, false).release()); + children.emplace_back(new EmptySearch()); + children.push_back(createIter(0, invert, tfmd, false)); - SearchIterator::UP s(T::create(children, false)); + SearchIterator::UP s = T::create(std::move(children), false); s = MultiBitVectorIteratorBase::optimize(std::move(s)); EXPECT_TRUE(dynamic_cast<const T *>(s.get()) != nullptr); const MultiSearch & m(dynamic_cast<const MultiSearch &>(*s)); EXPECT_EQUAL(2u, m.getChildren().size()); - EXPECT_TRUE(dynamic_cast<const EmptySearch *>(m.getChildren()[0]) != nullptr); - EXPECT_TRUE(dynamic_cast<const BitVectorIterator *>(m.getChildren()[1]) != nullptr); + EXPECT_TRUE(dynamic_cast<const EmptySearch *>(m.getChildren()[0].get()) != nullptr); + EXPECT_TRUE(dynamic_cast<const BitVectorIterator *>(m.getChildren()[1].get()) != nullptr); } { MultiSearch::Children children; - children.push_back(new EmptySearch()); - children.push_back(createIter(0, invert, tfmd, false).release()); - children.push_back(createIter(1, invert, tfmd, false).release()); + children.emplace_back(new EmptySearch()); + children.push_back(createIter(0, invert, tfmd, false)); + children.push_back(createIter(1, invert, tfmd, false)); - SearchIterator::UP s(T::create(children, false)); + SearchIterator::UP s = T::create(std::move(children), false); s = MultiBitVectorIteratorBase::optimize(std::move(s)); EXPECT_TRUE(s); EXPECT_TRUE(dynamic_cast<const T *>(s.get()) != nullptr); const MultiSearch & m(dynamic_cast<const MultiSearch &>(*s)); EXPECT_EQUAL(2u, m.getChildren().size()); - EXPECT_TRUE(dynamic_cast<const EmptySearch *>(m.getChildren()[0]) != nullptr); - EXPECT_TRUE(dynamic_cast<const MultiBitVectorIteratorBase *>(m.getChildren()[1]) != nullptr); + EXPECT_TRUE(dynamic_cast<const EmptySearch *>(m.getChildren()[0].get()) != nullptr); + EXPECT_TRUE(dynamic_cast<const MultiBitVectorIteratorBase *>(m.getChildren()[1].get()) != nullptr); EXPECT_TRUE(Trinary::False == m.getChildren()[1]->is_strict()); } { MultiSearch::Children children; - children.push_back(new EmptySearch()); - children.push_back(createIter(0, invert, tfmd, true).release()); - children.push_back(createIter(1, invert, tfmd, false).release()); + children.emplace_back(new EmptySearch()); + children.push_back(createIter(0, invert, tfmd, true)); + children.push_back(createIter(1, invert, tfmd, false)); - SearchIterator::UP s(T::create(children, false)); + SearchIterator::UP s = T::create(std::move(children), false); s = MultiBitVectorIteratorBase::optimize(std::move(s)); EXPECT_TRUE(s); EXPECT_TRUE(dynamic_cast<const T *>(s.get()) != nullptr); const MultiSearch & m(dynamic_cast<const MultiSearch &>(*s)); EXPECT_EQUAL(2u, m.getChildren().size()); - EXPECT_TRUE(dynamic_cast<const EmptySearch *>(m.getChildren()[0]) != nullptr); - EXPECT_TRUE(dynamic_cast<const MultiBitVectorIteratorBase *>(m.getChildren()[1]) != nullptr); + EXPECT_TRUE(dynamic_cast<const EmptySearch *>(m.getChildren()[0].get()) != nullptr); + EXPECT_TRUE(dynamic_cast<const MultiBitVectorIteratorBase *>(m.getChildren()[1].get()) != nullptr); EXPECT_TRUE(Trinary::True == m.getChildren()[1]->is_strict()); } { MultiSearch::Children children; - children.push_back(createIter(0, invert, tfmd, false).release()); - children.push_back(createIter(1, invert, tfmd, false).release()); + children.push_back(createIter(0, invert, tfmd, false)); + children.push_back(createIter(1, invert, tfmd, false)); - SearchIterator::UP s(T::create(children, false)); + SearchIterator::UP s = T::create(std::move(children), false); s = MultiBitVectorIteratorBase::optimize(std::move(s)); SearchIterator::UP filter(s->andWith(createIter(2, invert, tfmd, false), 9)); @@ -465,9 +465,9 @@ Test::testOptimizeCommon(bool isAnd, bool invert) } children.clear(); - children.push_back(createIter(0, invert, tfmd, false).release()); - children.push_back(createIter(1, invert, tfmd, false).release()); - s.reset(T::create(children, true)); + children.push_back(createIter(0, invert, tfmd, false)); + children.push_back(createIter(1, invert, tfmd, false)); + s = T::create(std::move(children), true); s = MultiBitVectorIteratorBase::optimize(std::move(s)); filter = s->andWith(createIter(2, invert, tfmd, false), 9); @@ -487,10 +487,10 @@ Test::testOptimizeAndOr(bool invert) { MultiSearch::Children children; - children.push_back(createIter(0, invert, tfmd, false).release()); - children.push_back(createIter(1, invert, tfmd, false).release()); + children.push_back(createIter(0, invert, tfmd, false)); + children.push_back(createIter(1, invert, tfmd, false)); - SearchIterator::UP s(T::create(children, false)); + SearchIterator::UP s = T::create(std::move(children), false); s = MultiBitVectorIteratorBase::optimize(std::move(s)); EXPECT_TRUE(s); EXPECT_TRUE(dynamic_cast<const MultiBitVectorIteratorBase *>(s.get()) != nullptr); @@ -498,67 +498,67 @@ Test::testOptimizeAndOr(bool invert) } { MultiSearch::Children children; - children.push_back(createIter(0, invert, tfmd, false).release()); - children.push_back(new EmptySearch()); - children.push_back(createIter(1, invert, tfmd, false).release()); + children.push_back(createIter(0, invert, tfmd, false)); + children.emplace_back(new EmptySearch()); + children.push_back(createIter(1, invert, tfmd, false)); - SearchIterator::UP s(T::create(children, false)); + SearchIterator::UP s = T::create(std::move(children), false); s = MultiBitVectorIteratorBase::optimize(std::move(s)); EXPECT_TRUE(s); EXPECT_TRUE(dynamic_cast<const T *>(s.get()) != nullptr); const MultiSearch & m(dynamic_cast<const MultiSearch &>(*s)); EXPECT_EQUAL(2u, m.getChildren().size()); - EXPECT_TRUE(dynamic_cast<const MultiBitVectorIteratorBase *>(m.getChildren()[0]) != nullptr); + EXPECT_TRUE(dynamic_cast<const MultiBitVectorIteratorBase *>(m.getChildren()[0].get()) != nullptr); EXPECT_TRUE(Trinary::False == m.getChildren()[0]->is_strict()); - EXPECT_TRUE(dynamic_cast<const EmptySearch *>(m.getChildren()[1]) != nullptr); + EXPECT_TRUE(dynamic_cast<const EmptySearch *>(m.getChildren()[1].get()) != nullptr); } { MultiSearch::Children children; - children.push_back(createIter(0, invert, tfmd, false).release()); - children.push_back(createIter(1, invert, tfmd, false).release()); - children.push_back(new EmptySearch()); + children.push_back(createIter(0, invert, tfmd, false)); + children.push_back(createIter(1, invert, tfmd, false)); + children.emplace_back(new EmptySearch()); - SearchIterator::UP s(T::create(children, false)); + SearchIterator::UP s = T::create(std::move(children), false); s = MultiBitVectorIteratorBase::optimize(std::move(s)); EXPECT_TRUE(s); EXPECT_TRUE(dynamic_cast<const T *>(s.get()) != nullptr); const MultiSearch & m(dynamic_cast<const MultiSearch &>(*s)); EXPECT_EQUAL(2u, m.getChildren().size()); - EXPECT_TRUE(dynamic_cast<const MultiBitVectorIteratorBase *>(m.getChildren()[0]) != nullptr); + EXPECT_TRUE(dynamic_cast<const MultiBitVectorIteratorBase *>(m.getChildren()[0].get()) != nullptr); EXPECT_TRUE(Trinary::False == m.getChildren()[0]->is_strict()); - EXPECT_TRUE(dynamic_cast<const EmptySearch *>(m.getChildren()[1]) != nullptr); + EXPECT_TRUE(dynamic_cast<const EmptySearch *>(m.getChildren()[1].get()) != nullptr); } { MultiSearch::Children children; - children.push_back(createIter(0, invert, tfmd, true).release()); - children.push_back(new EmptySearch()); - children.push_back(createIter(1, invert, tfmd, false).release()); + children.push_back(createIter(0, invert, tfmd, true)); + children.emplace_back(new EmptySearch()); + children.push_back(createIter(1, invert, tfmd, false)); - SearchIterator::UP s(T::create(children, false)); + SearchIterator::UP s = T::create(std::move(children), false); s = MultiBitVectorIteratorBase::optimize(std::move(s)); EXPECT_TRUE(s); EXPECT_TRUE(dynamic_cast<const T *>(s.get()) != nullptr); const MultiSearch & m(dynamic_cast<const MultiSearch &>(*s)); EXPECT_EQUAL(2u, m.getChildren().size()); - EXPECT_TRUE(dynamic_cast<const MultiBitVectorIteratorBase *>(m.getChildren()[0]) != nullptr); + EXPECT_TRUE(dynamic_cast<const MultiBitVectorIteratorBase *>(m.getChildren()[0].get()) != nullptr); EXPECT_TRUE(Trinary::True == m.getChildren()[0]->is_strict()); - EXPECT_TRUE(dynamic_cast<const EmptySearch *>(m.getChildren()[1]) != nullptr); + EXPECT_TRUE(dynamic_cast<const EmptySearch *>(m.getChildren()[1].get()) != nullptr); } { MultiSearch::Children children; - children.push_back(createIter(0, invert, tfmd, true).release()); - children.push_back(createIter(1, invert, tfmd, false).release()); - children.push_back(new EmptySearch()); + children.push_back(createIter(0, invert, tfmd, true)); + children.push_back(createIter(1, invert, tfmd, false)); + children.emplace_back(new EmptySearch()); - SearchIterator::UP s(T::create(children, false)); + SearchIterator::UP s = T::create(std::move(children), false); s = MultiBitVectorIteratorBase::optimize(std::move(s)); EXPECT_TRUE(s); EXPECT_TRUE(dynamic_cast<const T *>(s.get()) != nullptr); const MultiSearch & m(dynamic_cast<const MultiSearch &>(*s)); EXPECT_EQUAL(2u, m.getChildren().size()); - EXPECT_TRUE(dynamic_cast<const MultiBitVectorIteratorBase *>(m.getChildren()[0]) != nullptr); + EXPECT_TRUE(dynamic_cast<const MultiBitVectorIteratorBase *>(m.getChildren()[0].get()) != nullptr); EXPECT_TRUE(Trinary::True == m.getChildren()[0]->is_strict()); - EXPECT_TRUE(dynamic_cast<const EmptySearch *>(m.getChildren()[1]) != nullptr); + EXPECT_TRUE(dynamic_cast<const EmptySearch *>(m.getChildren()[1].get()) != nullptr); } } @@ -569,9 +569,9 @@ Test::testEndGuard(bool invert) TermFieldMatchData tfmd; MultiSearch::Children children; - children.push_back(createIter(0, invert, tfmd, true).release()); - children.push_back(createIter(1, invert, tfmd, true).release()); - SearchIterator::UP s(T::create(children, false)); + children.push_back(createIter(0, invert, tfmd, true)); + children.push_back(createIter(1, invert, tfmd, true)); + SearchIterator::UP s = T::create(std::move(children), false); s = MultiBitVectorIteratorBase::optimize(std::move(s)); s->initFullRange(); EXPECT_TRUE(s); @@ -618,11 +618,11 @@ SearchIterator::UP Verifier::create(bool strict) const { MultiSearch::Children bvs; for (const auto & bv : _bvs) { - bvs.push_back(BitVectorIterator::create(bv.get(), getDocIdLimit(), _tfmd, strict, false).release()); + bvs.push_back(BitVectorIterator::create(bv.get(), getDocIdLimit(), _tfmd, strict, false)); } - SearchIterator::UP iter(_is_and ? AndSearch::create(bvs, strict) : OrSearch::create(bvs, strict)); + SearchIterator::UP iter(_is_and ? AndSearch::create(std::move(bvs), strict) : OrSearch::create(std::move(bvs), strict)); auto mbvit = MultiBitVectorIteratorBase::optimize(std::move(iter)); - EXPECT_TRUE((bvs.size() < 2) || (dynamic_cast<const MultiBitVectorIteratorBase *>(mbvit.get()) != nullptr)); + EXPECT_TRUE((_bvs.size() < 2) || (dynamic_cast<const MultiBitVectorIteratorBase *>(mbvit.get()) != nullptr)); EXPECT_EQUAL(strict, Trinary::True == mbvit->is_strict()); return mbvit; } diff --git a/searchlib/src/tests/queryeval/parallel_weak_and/parallel_weak_and_test.cpp b/searchlib/src/tests/queryeval/parallel_weak_and/parallel_weak_and_test.cpp index 7926a518317..9761b0da2d7 100644 --- a/searchlib/src/tests/queryeval/parallel_weak_and/parallel_weak_and_test.cpp +++ b/searchlib/src/tests/queryeval/parallel_weak_and/parallel_weak_and_test.cpp @@ -89,14 +89,17 @@ struct WandTestSpec : public WandSpec WandTestSpec(uint32_t scoresToTrack, uint32_t scoresAdjustFrequency = 1, score_t scoreThreshold = 0, double thresholdBoostFactor = 1); ~WandTestSpec(); - SearchIterator *create() { + SearchIterator::UP create() { MatchData::UP childrenMatchData = createMatchData(); MatchData *tmp = childrenMatchData.get(); - return new TrackedSearch("PWAND", getHistory(), ParallelWeakAndSearch::create(getTerms(tmp), - matchParams, - RankParams(rootMatchData, - std::move(childrenMatchData)), - true)); + return SearchIterator::UP( + new TrackedSearch("PWAND", getHistory(), + ParallelWeakAndSearch::create( + getTerms(tmp), + matchParams, + RankParams(rootMatchData, + std::move(childrenMatchData)), + true))); } }; diff --git a/searchlib/src/tests/queryeval/queryeval.cpp b/searchlib/src/tests/queryeval/queryeval.cpp index 29cdd6a4b84..48b91607ab1 100644 --- a/searchlib/src/tests/queryeval/queryeval.cpp +++ b/searchlib/src/tests/queryeval/queryeval.cpp @@ -56,11 +56,13 @@ SearchIterator *simple(const std::string &tag) { return &((new SimpleSearch(SimpleResult()))->tag(tag)); } -Collect<SearchIterator*, MultiSearch::Children> search2(const std::string &t1, const std::string &t2) { - return Collect<SearchIterator*, MultiSearch::Children>().add(simple(t1)).add(simple(t2)); +MultiSearch::Children search2(const std::string &t1, const std::string &t2) { + MultiSearch::Children children; + children.emplace_back(simple(t1)); + children.emplace_back(simple(t2)); + return children; } - class ISourceSelectorDummy : public ISourceSelector { static SourceStore _sourceStoreDummy; @@ -95,9 +97,9 @@ void testMultiSearch(SearchIterator & search) { TEST("test that OR.andWith is a NOOP") { TermFieldMatchData tfmd; MultiSearch::Children ch; - ch.push_back(new TrueSearch(tfmd)); - ch.push_back(new TrueSearch(tfmd)); - SearchIterator::UP search(OrSearch::create(ch, true)); + ch.emplace_back(new TrueSearch(tfmd)); + ch.emplace_back(new TrueSearch(tfmd)); + SearchIterator::UP search(OrSearch::create(std::move(ch), true)); auto filter = std::make_unique<TrueSearch>(tfmd); EXPECT_TRUE(search->andWith(std::move(filter), 1)); @@ -106,9 +108,9 @@ TEST("test that OR.andWith is a NOOP") { TEST("test that non-strict AND.andWith is a NOOP") { TermFieldMatchData tfmd; MultiSearch::Children ch; - ch.push_back(new TrueSearch(tfmd)); - ch.push_back(new TrueSearch(tfmd)); - SearchIterator::UP search(AndSearch::create(ch, false)); + ch.emplace_back(new TrueSearch(tfmd)); + ch.emplace_back(new TrueSearch(tfmd)); + SearchIterator::UP search(AndSearch::create(std::move(ch), false)); SearchIterator::UP filter = std::make_unique<TrueSearch>(tfmd); filter = search->andWith(std::move(filter), 8); EXPECT_TRUE(filter); @@ -116,10 +118,10 @@ TEST("test that non-strict AND.andWith is a NOOP") { TEST("test that strict AND.andWith steals filter and places it correctly based on estimate") { TermFieldMatchData tfmd; - MultiSearch::Children ch; - ch.push_back(new TrueSearch(tfmd)); - ch.push_back(new TrueSearch(tfmd)); - SearchIterator::UP search(AndSearch::create(ch, true)); + std::vector<SearchIterator *> ch; + ch.emplace_back(new TrueSearch(tfmd)); + ch.emplace_back(new TrueSearch(tfmd)); + SearchIterator::UP search(AndSearch::create({ch[0], ch[1]}, true)); static_cast<AndSearch &>(*search).estimate(7); auto filter = std::make_unique<TrueSearch>(tfmd); SearchIterator * filterP = filter.get(); @@ -127,18 +129,18 @@ TEST("test that strict AND.andWith steals filter and places it correctly based o EXPECT_TRUE(nullptr == search->andWith(std::move(filter), 8).get()); const MultiSearch::Children & andChildren = static_cast<MultiSearch &>(*search).getChildren(); EXPECT_EQUAL(3u, andChildren.size()); - EXPECT_EQUAL(ch[0], andChildren[0]); - EXPECT_EQUAL(filterP, andChildren[1]); - EXPECT_EQUAL(ch[1], andChildren[2]); + EXPECT_EQUAL(ch[0], andChildren[0].get()); + EXPECT_EQUAL(filterP, andChildren[1].get()); + EXPECT_EQUAL(ch[1], andChildren[2].get()); auto filter2 = std::make_unique<TrueSearch>(tfmd); SearchIterator * filter2P = filter2.get(); EXPECT_TRUE(nullptr == search->andWith(std::move(filter2), 6).get()); EXPECT_EQUAL(4u, andChildren.size()); - EXPECT_EQUAL(filter2P, andChildren[0]); - EXPECT_EQUAL(ch[0], andChildren[1]); - EXPECT_EQUAL(filterP, andChildren[2]); - EXPECT_EQUAL(ch[1], andChildren[3]); + EXPECT_EQUAL(filter2P, andChildren[0].get()); + EXPECT_EQUAL(ch[0], andChildren[1].get()); + EXPECT_EQUAL(filterP, andChildren[2].get()); + EXPECT_EQUAL(ch[1], andChildren[3].get()); } class NonStrictTrueSearch : public TrueSearch @@ -150,69 +152,45 @@ public: TEST("test that strict AND.andWith does not place non-strict iterator first") { TermFieldMatchData tfmd; - MultiSearch::Children ch; - ch.push_back(new TrueSearch(tfmd)); - ch.push_back(new TrueSearch(tfmd)); - SearchIterator::UP search(AndSearch::create(ch, true)); + std::vector<SearchIterator *> ch; + ch.emplace_back(new TrueSearch(tfmd)); + ch.emplace_back(new TrueSearch(tfmd)); + SearchIterator::UP search(AndSearch::create({ch[0], ch[1]}, true)); static_cast<AndSearch &>(*search).estimate(7); auto filter = std::make_unique<NonStrictTrueSearch>(tfmd); SearchIterator * filterP = filter.get(); EXPECT_TRUE(nullptr == search->andWith(std::move(filter), 6).get()); const MultiSearch::Children & andChildren = static_cast<MultiSearch &>(*search).getChildren(); EXPECT_EQUAL(3u, andChildren.size()); - EXPECT_EQUAL(ch[0], andChildren[0]); - EXPECT_EQUAL(filterP, andChildren[1]); - EXPECT_EQUAL(ch[1], andChildren[2]); + EXPECT_EQUAL(ch[0], andChildren[0].get()); + EXPECT_EQUAL(filterP, andChildren[1].get()); + EXPECT_EQUAL(ch[1], andChildren[2].get()); } TEST("test that strict rank search forwards to its greedy first child") { TermFieldMatchData tfmd; - SearchIterator::UP search( - RankSearch::create( - Collect<SearchIterator*, MultiSearch::Children>() - .add(AndSearch::create(search2("a", "b"), true)) - .add(new TrueSearch(tfmd)), - true) - ); + SearchIterator::UP search = RankSearch::create({ AndSearch::create(search2("a", "b"), true), new TrueSearch(tfmd) }, true); auto filter = std::make_unique<TrueSearch>(tfmd); EXPECT_TRUE(nullptr == search->andWith(std::move(filter), 8).get()); } TEST("test that non-strict rank search does NOT forward to its greedy first child") { TermFieldMatchData tfmd; - SearchIterator::UP search( - RankSearch::create( - Collect<SearchIterator*, MultiSearch::Children>() - .add(AndSearch::create(search2("a", "b"), true)) - .add(new TrueSearch(tfmd)), - false) - ); + SearchIterator::UP search = RankSearch::create({ AndSearch::create(search2("a", "b"), true), new TrueSearch(tfmd) }, false); auto filter = std::make_unique<TrueSearch>(tfmd); EXPECT_TRUE(nullptr != search->andWith(std::move(filter), 8).get()); } TEST("test that strict andnot search forwards to its greedy first child") { TermFieldMatchData tfmd; - SearchIterator::UP search( - AndNotSearch::create( - Collect<SearchIterator*, MultiSearch::Children>() - .add(AndSearch::create(search2("a", "b"), true)) - .add(new TrueSearch(tfmd)), - true) - ); + SearchIterator::UP search = AndNotSearch::create({ AndSearch::create(search2("a", "b"), true), new TrueSearch(tfmd) }, true); auto filter = std::make_unique<TrueSearch>(tfmd); EXPECT_TRUE(nullptr == search->andWith(std::move(filter), 8).get()); } TEST("test that non-strict andnot search does NOT forward to its greedy first child") { TermFieldMatchData tfmd; - SearchIterator::UP search( - AndNotSearch::create( - Collect<SearchIterator*, MultiSearch::Children>() - .add(AndSearch::create(search2("a", "b"), true)) - .add(new TrueSearch(tfmd)), - false) - ); + SearchIterator::UP search = AndNotSearch::create({ AndSearch::create(search2("a", "b"), true), new TrueSearch(tfmd) }, false); auto filter = std::make_unique<TrueSearch>(tfmd); EXPECT_TRUE(nullptr != search->andWith(std::move(filter), 8).get()); } @@ -298,10 +276,10 @@ TEST("testOr") { { TermFieldMatchData tfmd; MultiSearch::Children ch; - ch.push_back(new TrueSearch(tfmd)); - ch.push_back(new TrueSearch(tfmd)); - ch.push_back(new TrueSearch(tfmd)); - SearchIterator::UP orSearch(OrSearch::create(ch, true)); + ch.emplace_back(new TrueSearch(tfmd)); + ch.emplace_back(new TrueSearch(tfmd)); + ch.emplace_back(new TrueSearch(tfmd)); + SearchIterator::UP orSearch(OrSearch::create(std::move(ch), true)); testMultiSearch(*orSearch); } } @@ -309,8 +287,8 @@ TEST("testOr") { class TestInsertRemoveSearch : public MultiSearch { public: - TestInsertRemoveSearch(const MultiSearch::Children & children) : - MultiSearch(children), + TestInsertRemoveSearch(ChildrenIterators children) : + MultiSearch(std::move(children)), _accumRemove(0), _accumInsert(0) { } @@ -327,31 +305,31 @@ struct MultiSearchRemoveTest { }; TEST("testMultiSearch") { - MultiSearch::Children children; - children.push_back(new EmptySearch()); - children.push_back(new EmptySearch()); - children.push_back(new EmptySearch()); - TestInsertRemoveSearch ms(children); + std::vector<SearchIterator *> orig; + orig.emplace_back(new EmptySearch()); + orig.emplace_back(new EmptySearch()); + orig.emplace_back(new EmptySearch()); + TestInsertRemoveSearch ms({orig[0], orig[1], orig[2]}); EXPECT_EQUAL(3u, ms.getChildren().size()); - EXPECT_EQUAL(children[0], ms.getChildren()[0]); - EXPECT_EQUAL(children[1], ms.getChildren()[1]); - EXPECT_EQUAL(children[2], ms.getChildren()[2]); + EXPECT_EQUAL(orig[0], ms.getChildren()[0].get()); + EXPECT_EQUAL(orig[1], ms.getChildren()[1].get()); + EXPECT_EQUAL(orig[2], ms.getChildren()[2].get()); EXPECT_EQUAL(0u, ms._accumInsert); EXPECT_EQUAL(0u, ms._accumRemove); - EXPECT_EQUAL(children[1], MultiSearchRemoveTest::remove(ms, 1).get()); + EXPECT_EQUAL(orig[1], MultiSearchRemoveTest::remove(ms, 1).get()); EXPECT_EQUAL(2u, ms.getChildren().size()); - EXPECT_EQUAL(children[0], ms.getChildren()[0]); - EXPECT_EQUAL(children[2], ms.getChildren()[1]); + EXPECT_EQUAL(orig[0], ms.getChildren()[0].get()); + EXPECT_EQUAL(orig[2], ms.getChildren()[1].get()); EXPECT_EQUAL(0u, ms._accumInsert); EXPECT_EQUAL(1u, ms._accumRemove); - children.push_back(new EmptySearch()); - ms.insert(1, SearchIterator::UP(children.back())); + orig.emplace_back(new EmptySearch()); + ms.insert(1, SearchIterator::UP(orig.back())); EXPECT_EQUAL(3u, ms.getChildren().size()); - EXPECT_EQUAL(children[0], ms.getChildren()[0]); - EXPECT_EQUAL(children[3], ms.getChildren()[1]); - EXPECT_EQUAL(children[2], ms.getChildren()[2]); + EXPECT_EQUAL(orig[0], ms.getChildren()[0].get()); + EXPECT_EQUAL(orig[3], ms.getChildren()[1].get()); + EXPECT_EQUAL(orig[2], ms.getChildren()[2].get()); EXPECT_EQUAL(1u, ms._accumInsert); EXPECT_EQUAL(1u, ms._accumRemove); } @@ -627,23 +605,19 @@ TEST("testDump") { #ifdef __clang__ #pragma clang diagnostic pop #endif - SearchIterator::UP search( - AndSearch::create( - Collect<SearchIterator*, MultiSearch::Children>() - .add(AndNotSearch::create(search2("+", "-"), true)) - .add(AndSearch::create(search2("and_a", "and_b"), true)) - .add(new BooleanMatchIteratorWrapper(SearchIterator::UP(simple("wrapped")), TermFieldMatchDataArray())) - .add(new NearSearch(search2("near_a", "near_b"), - TermFieldMatchDataArray(), - 5u, true)) - .add(new ONearSearch(search2("onear_a", "onear_b"), - TermFieldMatchDataArray(), 10, true)) - .add(OrSearch::create(search2("or_a", "or_b"), false)) - .add(RankSearch::create(search2("rank_a", "rank_b"),false)) - .add(SourceBlenderSearch::create(selector(), Collect<Source, SourceBlenderSearch::Children>() - .add(Source(simple("blend_a"), 2)) - .add(Source(simple("blend_b"), 4)), true)) - , true)); + + SearchIterator::UP search = AndSearch::create( { + AndNotSearch::create(search2("+", "-"), true), + AndSearch::create(search2("and_a", "and_b"), true), + new BooleanMatchIteratorWrapper(SearchIterator::UP(simple("wrapped")), TermFieldMatchDataArray()), + new NearSearch(search2("near_a", "near_b"), TermFieldMatchDataArray(), 5u, true), + new ONearSearch(search2("onear_a", "onear_b"), TermFieldMatchDataArray(), 10, true), + OrSearch::create(search2("or_a", "or_b"), false), + RankSearch::create(search2("rank_a", "rank_b"),false), + SourceBlenderSearch::create(selector(), Collect<Source, SourceBlenderSearch::Children>() + .add(Source(simple("blend_a"), 2)) + .add(Source(simple("blend_b"), 4)), + true) }, true); vespalib::string sas = search->asString(); EXPECT_TRUE(sas.size() > 50); vespalib::Slime slime; @@ -846,26 +820,26 @@ TEST("test InitRangeVerifier") { TEST("Test multisearch and andsearchstrict iterators adheres to initRange") { InitRangeVerifier ir; - ir.verify( AndSearch::create({ ir.createIterator(ir.getExpectedDocIds(), false).release(), - ir.createFullIterator().release() }, false)); + ir.verify( AndSearch::create({ ir.createIterator(ir.getExpectedDocIds(), false), + ir.createFullIterator() }, false)); - ir.verify( AndSearch::create({ ir.createIterator(ir.getExpectedDocIds(), true).release(), - ir.createFullIterator().release() }, true)); + ir.verify( AndSearch::create({ ir.createIterator(ir.getExpectedDocIds(), true), + ir.createFullIterator() }, true)); } TEST("Test andnotsearchstrict iterators adheres to initRange") { InitRangeVerifier ir; - TEST_DO(ir.verify( AndNotSearch::create({ir.createIterator(ir.getExpectedDocIds(), false).release(), - ir.createEmptyIterator().release() }, false))); - TEST_DO(ir.verify( AndNotSearch::create({ir.createIterator(ir.getExpectedDocIds(), true).release(), - ir.createEmptyIterator().release() }, true))); + TEST_DO(ir.verify( AndNotSearch::create({ir.createIterator(ir.getExpectedDocIds(), false), + ir.createEmptyIterator() }, false))); + TEST_DO(ir.verify( AndNotSearch::create({ir.createIterator(ir.getExpectedDocIds(), true), + ir.createEmptyIterator() }, true))); auto inverted = InitRangeVerifier::invert(ir.getExpectedDocIds(), ir.getDocIdLimit()); - TEST_DO(ir.verify( AndNotSearch::create({ir.createFullIterator().release(), - ir.createIterator(inverted, false).release() }, false))); - TEST_DO(ir.verify( AndNotSearch::create({ir.createFullIterator().release(), - ir.createIterator(inverted, false).release() }, true))); + TEST_DO(ir.verify( AndNotSearch::create({ir.createFullIterator(), + ir.createIterator(inverted, false) }, false))); + TEST_DO(ir.verify( AndNotSearch::create({ir.createFullIterator(), + ir.createIterator(inverted, false) }, true))); } diff --git a/searchlib/src/tests/queryeval/simple_phrase/simple_phrase_test.cpp b/searchlib/src/tests/queryeval/simple_phrase/simple_phrase_test.cpp index dfe2e2edbd9..d76f13afdca 100644 --- a/searchlib/src/tests/queryeval/simple_phrase/simple_phrase_test.cpp +++ b/searchlib/src/tests/queryeval/simple_phrase/simple_phrase_test.cpp @@ -170,10 +170,12 @@ public: } SimplePhraseSearch::Children children; for (size_t i = 0; i < _children.size(); ++i) { - children.push_back(_children[i]->createSearch(*_md, _strict).release()); + children.push_back(_children[i]->createSearch(*_md, _strict)); } - search = std::make_unique<SimplePhraseSearch>(children, MatchData::UP(), childMatch, _order, - *_md->resolveTermField(phrase_handle), _strict); + search = std::make_unique<SimplePhraseSearch>(std::move(children), + MatchData::UP(), childMatch, _order, + *_md->resolveTermField(phrase_handle), + _strict); } search->initFullRange(); return search.release(); diff --git a/searchlib/src/tests/queryeval/sparse_vector_benchmark/sparse_vector_benchmark_test.cpp b/searchlib/src/tests/queryeval/sparse_vector_benchmark/sparse_vector_benchmark_test.cpp index 1cf39183206..0de0815731b 100644 --- a/searchlib/src/tests/queryeval/sparse_vector_benchmark/sparse_vector_benchmark_test.cpp +++ b/searchlib/src/tests/queryeval/sparse_vector_benchmark/sparse_vector_benchmark_test.cpp @@ -139,19 +139,19 @@ constexpr vespalib::duration max_time = 1000s; struct ChildFactory { ChildFactory() {} virtual std::string name() const = 0; - virtual SearchIterator *createChild(uint32_t idx, uint32_t limit) const = 0; + virtual SearchIterator::UP createChild(uint32_t idx, uint32_t limit) const = 0; virtual ~ChildFactory() {} }; struct SparseVectorFactory { virtual std::string name() const = 0; - virtual SearchIterator *createSparseVector(ChildFactory &childFactory, uint32_t childCnt, uint32_t limit) const = 0; + virtual SearchIterator::UP createSparseVector(ChildFactory &childFactory, uint32_t childCnt, uint32_t limit) const = 0; virtual ~SparseVectorFactory() {} }; struct FilterStrategy { virtual std::string name() const = 0; - virtual SearchIterator *createRoot(SparseVectorFactory &vectorFactory, ChildFactory &childFactory, uint32_t childCnt, uint32_t limit) const = 0; + virtual SearchIterator::UP createRoot(SparseVectorFactory &vectorFactory, ChildFactory &childFactory, uint32_t childCnt, uint32_t limit) const = 0; virtual ~FilterStrategy() {} }; @@ -184,8 +184,8 @@ struct ModSearchFactory : ChildFactory { virtual std::string name() const override { return vespalib::make_string("ModSearch(%u)", bias); } - virtual SearchIterator *createChild(uint32_t idx, uint32_t limit) const override { - return new ModSearch(bias + idx, limit); + SearchIterator::UP createChild(uint32_t idx, uint32_t limit) const override { + return SearchIterator::UP(new ModSearch(bias + idx, limit)); } }; @@ -197,7 +197,7 @@ struct VespaWandFactory : SparseVectorFactory { virtual std::string name() const override { return vespalib::make_string("VespaWand(%u)", n); } - virtual SearchIterator *createSparseVector(ChildFactory &childFactory, uint32_t childCnt, uint32_t limit) const override { + SearchIterator::UP createSparseVector(ChildFactory &childFactory, uint32_t childCnt, uint32_t limit) const override { wand::Terms terms; for (size_t i = 0; i < childCnt; ++i) { terms.push_back(wand::Term(childFactory.createChild(i, limit), default_weight, limit / (i + 1))); @@ -212,12 +212,12 @@ struct RiseWandFactory : SparseVectorFactory { virtual std::string name() const override { return vespalib::make_string("RiseWand(%u)", n); } - virtual SearchIterator *createSparseVector(ChildFactory &childFactory, uint32_t childCnt, uint32_t limit) const override { + SearchIterator::UP createSparseVector(ChildFactory &childFactory, uint32_t childCnt, uint32_t limit) const override { wand::Terms terms; for (size_t i = 0; i < childCnt; ++i) { terms.push_back(wand::Term(childFactory.createChild(i, limit), default_weight, limit / (i + 1))); } - return new rise::TermFrequencyRiseWand(terms, n); + return SearchIterator::UP(new rise::TermFrequencyRiseWand(terms, n)); } }; @@ -226,11 +226,12 @@ struct WeightedSetFactory : SparseVectorFactory { virtual std::string name() const override { return vespalib::make_string("WeightedSet"); } - virtual SearchIterator *createSparseVector(ChildFactory &childFactory, uint32_t childCnt, uint32_t limit) const override { - std::vector<SearchIterator*> terms; + SearchIterator::UP createSparseVector(ChildFactory &childFactory, uint32_t childCnt, uint32_t limit) const override { + std::vector<SearchIterator *> terms; std::vector<int32_t> weights; for (size_t i = 0; i < childCnt; ++i) { - terms.push_back(childFactory.createChild(i, limit)); + // TODO: pass ownership with unique_ptr + terms.push_back(childFactory.createChild(i, limit).release()); weights.push_back(default_weight); } return WeightedSetTermSearch::create(terms, tfmd, weights, MatchData::UP(nullptr)); @@ -242,22 +243,22 @@ struct DotProductFactory : SparseVectorFactory { virtual std::string name() const override { return vespalib::make_string("DotProduct"); } - virtual SearchIterator *createSparseVector(ChildFactory &childFactory, uint32_t childCnt, uint32_t limit) const override { + SearchIterator::UP createSparseVector(ChildFactory &childFactory, uint32_t childCnt, uint32_t limit) const override { MatchDataLayout layout; std::vector<TermFieldHandle> handles; for (size_t i = 0; i < childCnt; ++i) { handles.push_back(layout.allocTermField(0)); } - std::vector<SearchIterator*> terms; + std::vector<SearchIterator *> terms; std::vector<TermFieldMatchData*> childMatch; std::vector<int32_t> weights; MatchData::UP md = layout.createMatchData(); for (size_t i = 0; i < childCnt; ++i) { - terms.push_back(childFactory.createChild(i, limit)); + terms.push_back(childFactory.createChild(i, limit).release()); childMatch.push_back(md->resolveTermField(handles[i])); weights.push_back(default_weight); } - return DotProductSearch::create(terms, tfmd, childMatch, weights, std::move(md)).release(); + return DotProductSearch::create(terms, tfmd, childMatch, weights, std::move(md)); } }; @@ -265,12 +266,12 @@ struct OrFactory : SparseVectorFactory { virtual std::string name() const override { return vespalib::make_string("Or"); } - virtual SearchIterator *createSparseVector(ChildFactory &childFactory, uint32_t childCnt, uint32_t limit) const override { + SearchIterator::UP createSparseVector(ChildFactory &childFactory, uint32_t childCnt, uint32_t limit) const override { OrSearch::Children children; for (size_t i = 0; i < childCnt; ++i) { children.push_back(childFactory.createChild(i, limit)); } - return OrSearch::create(children, true); + return OrSearch::create(std::move(children), true); } }; @@ -280,7 +281,7 @@ struct NoFilterStrategy : FilterStrategy { virtual std::string name() const override { return vespalib::make_string("NoFilter"); } - virtual SearchIterator *createRoot(SparseVectorFactory &vectorFactory, ChildFactory &childFactory, uint32_t childCnt, uint32_t limit) const override { + SearchIterator::UP createRoot(SparseVectorFactory &vectorFactory, ChildFactory &childFactory, uint32_t childCnt, uint32_t limit) const override { return vectorFactory.createSparseVector(childFactory, childCnt, limit); } }; @@ -289,11 +290,11 @@ struct PositiveFilterBeforeStrategy : FilterStrategy { virtual std::string name() const override { return vespalib::make_string("PositiveBefore"); } - virtual SearchIterator *createRoot(SparseVectorFactory &vectorFactory, ChildFactory &childFactory, uint32_t childCnt, uint32_t limit) const override { + SearchIterator::UP createRoot(SparseVectorFactory &vectorFactory, ChildFactory &childFactory, uint32_t childCnt, uint32_t limit) const override { AndSearch::Children children; - children.push_back(new ModSearch(2, limit)); // <- 50% hits (hardcoded) + children.emplace_back(new ModSearch(2, limit)); // <- 50% hits (hardcoded) children.push_back(vectorFactory.createSparseVector(childFactory, childCnt, limit)); - return AndSearch::create(children, true); + return AndSearch::create(std::move(children), true); } }; @@ -301,11 +302,11 @@ struct NegativeFilterAfterStrategy : FilterStrategy { virtual std::string name() const override { return vespalib::make_string("NegativeAfter"); } - virtual SearchIterator *createRoot(SparseVectorFactory &vectorFactory, ChildFactory &childFactory, uint32_t childCnt, uint32_t limit) const override { + SearchIterator::UP createRoot(SparseVectorFactory &vectorFactory, ChildFactory &childFactory, uint32_t childCnt, uint32_t limit) const override { AndNotSearch::Children children; children.push_back(vectorFactory.createSparseVector(childFactory, childCnt, limit)); - children.push_back(new ModSearch(2, limit)); // <- 50% hits (hardcoded) - return AndNotSearch::create(children, true); + children.emplace_back(new ModSearch(2, limit)); // <- 50% hits (hardcoded) + return AndNotSearch::create(std::move(children), true); } }; diff --git a/searchlib/src/tests/queryeval/termwise_eval/termwise_eval_test.cpp b/searchlib/src/tests/queryeval/termwise_eval/termwise_eval_test.cpp index 8d70daa4ca8..dee1bdb0b9a 100644 --- a/searchlib/src/tests/queryeval/termwise_eval/termwise_eval_test.cpp +++ b/searchlib/src/tests/queryeval/termwise_eval/termwise_eval_test.cpp @@ -120,32 +120,32 @@ SearchIterator *TERM(std::initializer_list<uint32_t> hits, bool strict) { return new MyTerm(hits, strict); } -SearchIterator *ANDNOT(std::initializer_list<SearchIterator *> children, bool strict) { - return AndNotSearch::create(children, strict); +SearchIterator::UP ANDNOT(ChildrenIterators children, bool strict) { + return AndNotSearch::create(std::move(children), strict); } -SearchIterator *AND(std::initializer_list<SearchIterator *> children, bool strict) { - return AndSearch::create(children, strict); +SearchIterator::UP AND(ChildrenIterators children, bool strict) { + return AndSearch::create(std::move(children), strict); } -SearchIterator *ANDz(std::initializer_list<SearchIterator *> children, bool strict) { - return AndSearch::create(children, strict, no_unpack()); +SearchIterator::UP ANDz(ChildrenIterators children, bool strict) { + return AndSearch::create(std::move(children), strict, no_unpack()); } -SearchIterator *ANDs(std::initializer_list<SearchIterator *> children, bool strict) { - return AndSearch::create(children, strict, selective_unpack()); +SearchIterator::UP ANDs(ChildrenIterators children, bool strict) { + return AndSearch::create(std::move(children), strict, selective_unpack()); } -SearchIterator *OR(std::initializer_list<SearchIterator *> children, bool strict) { - return OrSearch::create(children, strict); +SearchIterator::UP OR(ChildrenIterators children, bool strict) { + return OrSearch::create(std::move(children), strict); } -SearchIterator *ORz(std::initializer_list<SearchIterator *> children, bool strict) { - return OrSearch::create(children, strict, no_unpack()); +SearchIterator::UP ORz(ChildrenIterators children, bool strict) { + return OrSearch::create(std::move(children), strict, no_unpack()); } -SearchIterator *ORs(std::initializer_list<SearchIterator *> children, bool strict) { - return OrSearch::create(children, strict, selective_unpack()); +SearchIterator::UP ORs(ChildrenIterators children, bool strict) { + return OrSearch::create(std::move(children), strict, selective_unpack()); } //----------------------------------------------------------------------------- @@ -156,22 +156,22 @@ std::unique_ptr<T> UP(T *t) { return std::unique_ptr<T>(t); } //----------------------------------------------------------------------------- SearchIterator::UP make_search(bool strict) { - return UP(AND({OR({TERM({2,7}, true), - TERM({4,8}, true), - TERM({5,6,9}, true)}, true), - OR({TERM({1,4,7}, false), - TERM({2,5,8}, true), - TERM({3,6}, false)}, false), - OR({TERM({1,2,3}, false), - TERM({4,6}, false), - TERM({8,9}, false)}, false)}, strict)); + return AND({OR({ TERM({2,7}, true), + TERM({4,8}, true), + TERM({5,6,9}, true) }, true), + OR({ TERM({1,4,7}, false), + TERM({2,5,8}, true), + TERM({3,6}, false) }, false), + OR({ TERM({1,2,3}, false), + TERM({4,6}, false), + TERM({8,9}, false) }, false)}, strict); } SearchIterator::UP make_filter_search(bool strict) { - return UP(ANDNOT({TERM({1,2,3,4,5,6,7,8,9}, true), - TERM({1,9}, false), - TERM({3,7}, true), - TERM({5}, false)}, strict)); + return ANDNOT({ TERM({1,2,3,4,5,6,7,8,9}, true), + TERM({1,9}, false), + TERM({3,7}, true), + TERM({5}, false) }, strict); } void add_if_inside(uint32_t docid, uint32_t begin, uint32_t end, std::vector<uint32_t> &expect) { @@ -262,7 +262,7 @@ TEST("require that termwise filter search produces appropriate results") { } TEST("require that termwise ANDNOT with single term works") { - TEST_DO(verify({2,3,4}, *make_termwise(UP(ANDNOT({TERM({1,2,3,4,5}, true)}, true)), true), 2, 5)); + TEST_DO(verify({2,3,4}, *make_termwise(ANDNOT({ TERM({1,2,3,4,5}, true) }, true), true), 2, 5)); } TEST("require that pseudo term is rewindable") { @@ -361,20 +361,20 @@ TEST("require that match data keeps track of the termwise limit") { //----------------------------------------------------------------------------- TEST("require that terwise test search string dump is detailed enough") { - EXPECT_EQUAL(make_termwise(UP(OR({TERM({1,2,3}, true), TERM({2,3}, true), TERM({3}, true)}, true)), true)->asString(), - make_termwise(UP(OR({TERM({1,2,3}, true), TERM({2,3}, true), TERM({3}, true)}, true)), true)->asString()); + EXPECT_EQUAL(make_termwise(OR({ TERM({1,2,3}, true), TERM({2,3}, true), TERM({3}, true) }, true), true)->asString(), + make_termwise(OR({ TERM({1,2,3}, true), TERM({2,3}, true), TERM({3}, true) }, true), true)->asString()); - EXPECT_NOT_EQUAL(make_termwise(UP(OR({TERM({1,2,3}, true), TERM({2,3}, true), TERM({3}, true)}, true)), true)->asString(), - make_termwise(UP(OR({TERM({1,2,3}, true), TERM({2,3}, false), TERM({3}, true)}, true)), true)->asString()); + EXPECT_NOT_EQUAL(make_termwise(OR({ TERM({1,2,3}, true), TERM({2,3}, true), TERM({3}, true) }, true), true)->asString(), + make_termwise(OR({ TERM({1,2,3}, true), TERM({2,3}, false), TERM({3}, true) }, true), true)->asString()); - EXPECT_NOT_EQUAL(make_termwise(UP(OR({TERM({1,2,3}, true), TERM({2,3}, true), TERM({3}, true)}, true)), true)->asString(), - make_termwise(UP(OR({TERM({1,2,3}, true), TERM({2,3}, true), TERM({3}, true)}, false)), true)->asString()); + EXPECT_NOT_EQUAL(make_termwise(OR({ TERM({1,2,3}, true), TERM({2,3}, true), TERM({3}, true) }, true), true)->asString(), + make_termwise(OR({ TERM({1,2,3}, true), TERM({2,3}, true), TERM({3}, true) }, false), true)->asString()); - EXPECT_NOT_EQUAL(make_termwise(UP(OR({TERM({1,2,3}, true), TERM({2,3}, true), TERM({3}, true)}, true)), true)->asString(), - make_termwise(UP(OR({TERM({1,2,3}, true), TERM({2,3}, true), TERM({3}, true)}, true)), false)->asString()); + EXPECT_NOT_EQUAL(make_termwise(OR({ TERM({1,2,3}, true), TERM({2,3}, true), TERM({3}, true) }, true), true)->asString(), + make_termwise(OR({ TERM({1,2,3}, true), TERM({2,3}, true), TERM({3}, true) }, true), false)->asString()); - EXPECT_NOT_EQUAL(make_termwise(UP(OR({TERM({1,2,3}, true), TERM({2,3}, true), TERM({3}, true)}, true)), true)->asString(), - make_termwise(UP(OR({TERM({1,2,3}, true), TERM({3}, true), TERM({2,3}, true)}, true)), true)->asString()); + EXPECT_NOT_EQUAL(make_termwise(OR({ TERM({1,2,3}, true), TERM({2,3}, true), TERM({3}, true) }, true), true)->asString(), + make_termwise(OR({ TERM({1,2,3}, true), TERM({3}, true), TERM({2,3}, true) }, true), true)->asString()); } //----------------------------------------------------------------------------- @@ -389,7 +389,7 @@ TEST("require that basic termwise evaluation works") { my_or.addChild(UP(new MyBlueprint({2}, true, 2))); for (bool strict: {true, false}) { EXPECT_EQUAL(my_or.createSearch(*md, strict)->asString(), - make_termwise(UP(OR({TERM({1}, strict), TERM({2}, strict)}, strict)), strict)->asString()); + make_termwise(OR({ TERM({1}, strict), TERM({2}, strict) }, strict), strict)->asString()); } } @@ -434,7 +434,9 @@ TEST("require that termwise evaluation can be multi-level, but not duplicated") my_or.addChild(std::move(child)); for (bool strict: {true, false}) { EXPECT_EQUAL(my_or.createSearch(*md, strict)->asString(), - make_termwise(UP(OR({TERM({1}, strict), ORz({TERM({2}, strict), TERM({3}, strict)}, strict)}, strict)), strict)->asString()); + make_termwise(OR({ TERM({1}, strict), + ORz({ TERM({2}, strict), TERM({3}, strict) }, strict) }, + strict), strict)->asString()); } } @@ -450,7 +452,7 @@ TEST("require that OR can be completely termwise") { my_or.addChild(UP(new MyBlueprint({2}, true, 2))); for (bool strict: {true, false}) { EXPECT_EQUAL(my_or.createSearch(*md, strict)->asString(), - make_termwise(UP(OR({TERM({1}, strict), TERM({2}, strict)}, strict)), strict)->asString()); + make_termwise(OR({ TERM({1}, strict), TERM({2}, strict) }, strict), strict)->asString()); } } @@ -465,7 +467,8 @@ TEST("require that OR can be partially termwise") { my_or.addChild(UP(new MyBlueprint({3}, true, 3))); for (bool strict: {true, false}) { EXPECT_EQUAL(my_or.createSearch(*md, strict)->asString(), - UP(ORs({make_termwise(UP(OR({TERM({1}, strict), TERM({3}, strict)}, strict)), strict).release(), TERM({2}, strict)}, strict))->asString()); + ORs({ make_termwise(OR({ TERM({1}, strict), TERM({3}, strict) }, strict), strict), + TERM({2}, strict) }, strict)->asString()); } } @@ -480,7 +483,9 @@ TEST("require that OR puts termwise subquery at the right place") { my_or.addChild(UP(new MyBlueprint({3}, true, 3))); for (bool strict: {true, false}) { EXPECT_EQUAL(my_or.createSearch(*md, strict)->asString(), - UP(ORs({TERM({1}, strict), make_termwise(UP(OR({TERM({2}, strict), TERM({3}, strict)}, strict)), strict).release()}, strict))->asString()); + ORs({ TERM({1}, strict), + make_termwise(OR({ TERM({2}, strict), TERM({3}, strict) }, strict), + strict) }, strict)->asString()); } } @@ -496,7 +501,10 @@ TEST("require that OR can use termwise eval also when having non-termwise childr my_or.addChild(UP(new MyBlueprint({3}, true, 3))); for (bool strict: {true, false}) { EXPECT_EQUAL(my_or.createSearch(*md, strict)->asString(), - UP(ORz({TERM({1}, strict), make_termwise(UP(OR({TERM({2}, strict), TERM({3}, strict)}, strict)), strict).release()}, strict))->asString()); + ORz({ TERM({1}, strict), + make_termwise(OR({ TERM({2}, strict), TERM({3}, strict) }, strict), + strict)}, + strict)->asString()); } } @@ -512,7 +520,7 @@ TEST("require that AND can be completely termwise") { my_and.addChild(UP(new MyBlueprint({2}, true, 2))); for (bool strict: {true, false}) { EXPECT_EQUAL(my_and.createSearch(*md, strict)->asString(), - make_termwise(UP(AND({TERM({1}, strict), TERM({2}, false)}, strict)), strict)->asString()); + make_termwise(AND({ TERM({1}, strict), TERM({2}, false) }, strict), strict)->asString()); } } @@ -527,7 +535,10 @@ TEST("require that AND can be partially termwise") { my_and.addChild(UP(new MyBlueprint({3}, true, 3))); for (bool strict: {true, false}) { EXPECT_EQUAL(my_and.createSearch(*md, strict)->asString(), - UP(ANDs({make_termwise(UP(AND({TERM({1}, strict), TERM({3}, false)}, strict)), strict).release(), TERM({2}, false)}, strict))->asString()); + ANDs({ make_termwise(AND({ TERM({1}, strict), TERM({3}, false) }, + strict), + strict), + TERM({2}, false) }, strict)->asString()); } } @@ -542,7 +553,9 @@ TEST("require that AND puts termwise subquery at the right place") { my_and.addChild(UP(new MyBlueprint({3}, true, 3))); for (bool strict: {true, false}) { EXPECT_EQUAL(my_and.createSearch(*md, strict)->asString(), - UP(ANDs({TERM({1}, strict), make_termwise(UP(AND({TERM({2}, false), TERM({3}, false)}, false)), false).release()}, strict))->asString()); + ANDs({ TERM({1}, strict), + make_termwise(AND({ TERM({2}, false), TERM({3}, false) }, false), + false) }, strict)->asString()); } } @@ -558,7 +571,9 @@ TEST("require that AND can use termwise eval also when having non-termwise child my_and.addChild(UP(new MyBlueprint({3}, true, 3))); for (bool strict: {true, false}) { EXPECT_EQUAL(my_and.createSearch(*md, strict)->asString(), - UP(ANDz({TERM({1}, strict), make_termwise(UP(AND({TERM({2}, false), TERM({3}, false)}, false)), false).release()}, strict))->asString()); + ANDz({ TERM({1}, strict), + make_termwise(AND({ TERM({2}, false), TERM({3}, false) }, false), + false) }, strict)->asString()); } } @@ -573,7 +588,8 @@ TEST("require that ANDNOT can be completely termwise") { my_andnot.addChild(UP(new MyBlueprint({2}, true, 2))); for (bool strict: {true, false}) { EXPECT_EQUAL(my_andnot.createSearch(*md, strict)->asString(), - make_termwise(UP(ANDNOT({TERM({1}, strict), TERM({2}, false)}, strict)), strict)->asString()); + make_termwise(ANDNOT({ TERM({1}, strict), TERM({2}, false) }, + strict), strict)->asString()); } } @@ -586,7 +602,9 @@ TEST("require that ANDNOT can be partially termwise") { my_andnot.addChild(UP(new MyBlueprint({3}, true, 3))); for (bool strict: {true, false}) { EXPECT_EQUAL(my_andnot.createSearch(*md, strict)->asString(), - UP(ANDNOT({TERM({1}, strict), make_termwise(UP(OR({TERM({2}, false), TERM({3}, false)}, false)), false).release()}, strict))->asString()); + ANDNOT({ TERM({1}, strict), + make_termwise(OR({ TERM({2}, false), TERM({3}, false) }, false), + false) }, strict)->asString()); } } @@ -600,7 +618,9 @@ TEST("require that ANDNOT can be partially termwise with first child being termw my_andnot.addChild(UP(new MyBlueprint({3}, true, 3))); for (bool strict: {true, false}) { EXPECT_EQUAL(my_andnot.createSearch(*md, strict)->asString(), - UP(ANDNOT({make_termwise(UP(ANDNOT({TERM({1}, strict), TERM({3}, false)}, strict)), strict).release(), TERM({2}, false)}, strict))->asString()); + ANDNOT({ make_termwise(ANDNOT({ TERM({1}, strict), TERM({3}, false) }, strict), + strict), + TERM({2}, false) }, strict)->asString()); } } @@ -613,13 +633,13 @@ TEST("require that termwise blueprint helper calculates unpack info correctly") my_or.addChild(UP(new MyBlueprint({3}, true, 3))); my_or.addChild(UP(new MyBlueprint({4}, true, 4))); // ranked my_or.addChild(UP(new MyBlueprint({5}, true, 5))); - MultiSearch::Children dummy_searches(5, nullptr); + MultiSearch::Children dummy_searches(5); UnpackInfo unpack; // non-termwise unpack info unpack.add(1); unpack.add(3); - TermwiseBlueprintHelper helper(my_or, dummy_searches, unpack); - EXPECT_EQUAL(helper.children.size(), 3u); - EXPECT_EQUAL(helper.termwise.size(), 2u); + TermwiseBlueprintHelper helper(my_or, std::move(dummy_searches), unpack); + EXPECT_EQUAL(helper.get_result().size(), 3u); + EXPECT_EQUAL(helper.get_termwise_children().size(), 2u); EXPECT_EQUAL(helper.first_termwise, 2u); EXPECT_TRUE(!helper.termwise_unpack.needUnpack(0)); EXPECT_TRUE(helper.termwise_unpack.needUnpack(1)); diff --git a/searchlib/src/tests/queryeval/weak_and/wand_bench_setup.hpp b/searchlib/src/tests/queryeval/weak_and/wand_bench_setup.hpp index 95e4cc8ea79..f5aafab87c3 100644 --- a/searchlib/src/tests/queryeval/weak_and/wand_bench_setup.hpp +++ b/searchlib/src/tests/queryeval/weak_and/wand_bench_setup.hpp @@ -179,9 +179,9 @@ struct FilterFactory : WandFactory { virtual std::string name() const override { return make_string("Filter (mod=%u) [%s]", n, factory.name().c_str()); } virtual SearchIterator::UP create(const wand::Terms &terms) override { AndNotSearch::Children children; - children.push_back(factory.create(terms).release()); - children.push_back(new ModSearch(stats, n, search::endDocId, n, NULL)); - return SearchIterator::UP(AndNotSearch::create(children, true)); + children.push_back(factory.create(terms)); + children.emplace_back(new ModSearch(stats, n, search::endDocId, n, NULL)); + return AndNotSearch::create(std::move(children), true); } }; diff --git a/searchlib/src/tests/queryeval/weighted_set_term/weighted_set_term_test.cpp b/searchlib/src/tests/queryeval/weighted_set_term/weighted_set_term_test.cpp index 85d2f3f4a37..8514a221230 100644 --- a/searchlib/src/tests/queryeval/weighted_set_term/weighted_set_term_test.cpp +++ b/searchlib/src/tests/queryeval/weighted_set_term/weighted_set_term_test.cpp @@ -123,7 +123,7 @@ struct MockFixture { mock = new MockSearch(initial); children.push_back(mock); weights.push_back(1); - search.reset(WeightedSetTermSearch::create(children, tfmd, weights, MatchData::UP(nullptr))); + search = WeightedSetTermSearch::create(children, tfmd, weights, MatchData::UP(nullptr)); } }; diff --git a/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp b/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp index a9a593b429d..74a23db8b95 100644 --- a/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp @@ -169,7 +169,8 @@ AttributeFieldBlueprint::visitMembers(vespalib::ObjectVisitor &visitor) const template <bool is_strict> struct LocationPreFilterIterator : public OrLikeSearch<is_strict, NoUnpack> { - LocationPreFilterIterator(const std::vector<SearchIterator *> &children) : OrLikeSearch<is_strict, NoUnpack>(children, NoUnpack()) {} + LocationPreFilterIterator(OrSearch::Children children) + : OrLikeSearch<is_strict, NoUnpack>(std::move(children), NoUnpack()) {} void doUnpack(uint32_t) override {} }; @@ -215,14 +216,14 @@ public: SearchIterator::UP createLeafSearch(const TermFieldMatchDataArray &tfmda, bool strict) const override { - std::vector<SearchIterator *> children; + OrSearch::Children children; for (auto it(_rangeSearches.begin()), mt(_rangeSearches.end()); it != mt; it++) { - children.push_back((*it)->createIterator(tfmda[0], strict).release()); + children.push_back((*it)->createIterator(tfmda[0], strict)); } if (strict) { - return std::make_unique<LocationPreFilterIterator<true>>(children); + return std::make_unique<LocationPreFilterIterator<true>>(std::move(children)); } else { - return std::make_unique<LocationPreFilterIterator<false>>(children); + return std::make_unique<LocationPreFilterIterator<false>>(std::move(children)); } } diff --git a/searchlib/src/vespa/searchlib/attribute/attribute_weighted_set_blueprint.cpp b/searchlib/src/vespa/searchlib/attribute/attribute_weighted_set_blueprint.cpp index af3fbca6943..121cb736471 100644 --- a/searchlib/src/vespa/searchlib/attribute/attribute_weighted_set_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attribute_weighted_set_blueprint.cpp @@ -162,6 +162,7 @@ AttributeWeightedSetBlueprint::createLeafSearch(const fef::TermFieldMatchDataArr auto child_tfmd = match_data->resolveTermField(handle); std::vector<queryeval::SearchIterator*> children(_contexts.size()); for (size_t i = 0; i < _contexts.size(); ++i) { + // TODO: pass ownership with unique_ptr children[i] = _contexts[i]->createIterator(child_tfmd, true).release(); } return queryeval::SearchIterator::UP(queryeval::WeightedSetTermSearch::create(children, tfmd, _weights, std::move(match_data))); diff --git a/searchlib/src/vespa/searchlib/queryeval/CMakeLists.txt b/searchlib/src/vespa/searchlib/queryeval/CMakeLists.txt index c6ca41a2a2a..a080fbcd387 100644 --- a/searchlib/src/vespa/searchlib/queryeval/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/queryeval/CMakeLists.txt @@ -5,6 +5,7 @@ vespa_add_library(searchlib_queryeval OBJECT andsearch.cpp blueprint.cpp booleanmatchiteratorwrapper.cpp + children_iterators.cpp create_blueprint_visitor_helper.cpp document_weight_search_iterator.cpp dot_product_blueprint.cpp diff --git a/searchlib/src/vespa/searchlib/queryeval/andnotsearch.cpp b/searchlib/src/vespa/searchlib/queryeval/andnotsearch.cpp index 5646af9e677..61d6d2d9259 100644 --- a/searchlib/src/vespa/searchlib/queryeval/andnotsearch.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/andnotsearch.cpp @@ -51,7 +51,7 @@ public: * * @param children the search objects we are andnot'ing **/ - AndNotSearchStrict(const Children & children) : AndNotSearchStrictBase(children) + AndNotSearchStrict(Children children) : AndNotSearchStrictBase(std::move(children)) { } @@ -105,8 +105,8 @@ AndNotSearchStrict::internalSeek(uint32_t docid) } // namespace -OptimizedAndNotForBlackListing::OptimizedAndNotForBlackListing(const MultiSearch::Children & children) : - AndNotSearchStrictBase(children) +OptimizedAndNotForBlackListing::OptimizedAndNotForBlackListing(MultiSearch::Children children) : + AndNotSearchStrictBase(std::move(children)) { } @@ -131,23 +131,24 @@ void OptimizedAndNotForBlackListing::doUnpack(uint32_t docid) positive()->doUnpack(docid); } -SearchIterator * -AndNotSearch::create(const AndNotSearch::Children &children, bool strict) { +std::unique_ptr<SearchIterator> +AndNotSearch::create(ChildrenIterators children_in, bool strict) { + MultiSearch::Children children = std::move(children_in); if (strict) { - if ((children.size() == 2) && OptimizedAndNotForBlackListing::isBlackListIterator(children[1])) { - return new OptimizedAndNotForBlackListing(children); + if ((children.size() == 2) && OptimizedAndNotForBlackListing::isBlackListIterator(children[1].get())) { + return std::make_unique<OptimizedAndNotForBlackListing>(std::move(children)); } else { - return new AndNotSearchStrict(children); + return std::make_unique<AndNotSearchStrict>(std::move(children)); } } else { - return new AndNotSearch(children); + return SearchIterator::UP(new AndNotSearch(std::move(children))); } } BitVector::UP AndNotSearch::get_hits(uint32_t begin_id) { const Children &children = getChildren(); - BitVector::UP result = children.front()->get_hits(begin_id); + BitVector::UP result = children[0]->get_hits(begin_id); result->notSelf(); result = TermwiseHelper::orChildren(std::move(result), children.begin()+1, children.end(), begin_id); result->notSelf(); diff --git a/searchlib/src/vespa/searchlib/queryeval/andnotsearch.h b/searchlib/src/vespa/searchlib/queryeval/andnotsearch.h index c29ef9407ad..53a9aad6cca 100644 --- a/searchlib/src/vespa/searchlib/queryeval/andnotsearch.h +++ b/searchlib/src/vespa/searchlib/queryeval/andnotsearch.h @@ -24,11 +24,10 @@ protected: * * @param children the search objects we are andnot'ing **/ - AndNotSearch(const Children & children) : MultiSearch(children) { } + AndNotSearch(MultiSearch::Children children) : MultiSearch(std::move(children)) { } public: - // Caller takes ownership of the returned SearchIterator. - static SearchIterator *create(const Children &children, bool strict); + static std::unique_ptr<SearchIterator> create(ChildrenIterators children, bool strict); std::unique_ptr<BitVector> get_hits(uint32_t begin_id) override; void or_hits_into(BitVector &result, uint32_t begin_id) override; @@ -43,7 +42,7 @@ private: class AndNotSearchStrictBase : public AndNotSearch { protected: - AndNotSearchStrictBase(const Children & children) : AndNotSearch(children) { } + AndNotSearchStrictBase(Children children) : AndNotSearch(std::move(children)) { } private: Trinary is_strict() const override { return Trinary::True; } UP andWith(UP filter, uint32_t estimate) override; @@ -61,7 +60,7 @@ private: //typedef FilterAttributeIteratorT<SingleValueSmallNumericAttribute::SingleSearchContext> BlackListIterator; typedef AttributeIteratorT<SingleValueSmallNumericAttribute::SingleSearchContext> BlackListIterator; public: - OptimizedAndNotForBlackListing(const MultiSearch::Children & children); + OptimizedAndNotForBlackListing(MultiSearch::Children children); static bool isBlackListIterator(const SearchIterator * iterator); uint32_t seekFast(uint32_t docid) { @@ -69,8 +68,8 @@ public: } void initRange(uint32_t beginid, uint32_t endid) override; private: - SearchIterator * positive() { return getChildren()[0]; } - BlackListIterator * blackList() { return static_cast<BlackListIterator *>(getChildren()[1]); } + SearchIterator * positive() { return getChildren()[0].get(); } + BlackListIterator * blackList() { return static_cast<BlackListIterator *>(getChildren()[1].get()); } template<bool doSeekOnly> uint32_t internalSeek(uint32_t docid) { uint32_t curr(docid); diff --git a/searchlib/src/vespa/searchlib/queryeval/andsearch.cpp b/searchlib/src/vespa/searchlib/queryeval/andsearch.cpp index 3cbb30e5f89..229cd27afed 100644 --- a/searchlib/src/vespa/searchlib/queryeval/andsearch.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/andsearch.cpp @@ -48,8 +48,8 @@ AndSearch::doUnpack(uint32_t docid) } } -AndSearch::AndSearch(const Children & children) : - MultiSearch(children), +AndSearch::AndSearch(Children children) : + MultiSearch(std::move(children)), _estimate(std::numeric_limits<uint32_t>::max()) { } @@ -99,31 +99,37 @@ private: } -AndSearch * -AndSearch::create(const MultiSearch::Children &children, bool strict) +std::unique_ptr<AndSearch> +AndSearch::create(ChildrenIterators children, bool strict) { UnpackInfo unpackInfo; unpackInfo.forceAll(); - return create(children, strict, unpackInfo); + return create(std::move(children), strict, unpackInfo); } -AndSearch * -AndSearch::create(const MultiSearch::Children &children, bool strict, const UnpackInfo & unpackInfo) { +std::unique_ptr<AndSearch> +AndSearch::create(ChildrenIterators children, bool strict, const UnpackInfo & unpackInfo) { if (strict) { if (unpackInfo.unpackAll()) { - return new AndSearchStrict<FullUnpack>(children, FullUnpack()); + using MyAnd = AndSearchStrict<FullUnpack>; + return std::make_unique<MyAnd>(std::move(children), FullUnpack()); } else if(unpackInfo.empty()) { - return new AndSearchStrict<NoUnpack>(children, NoUnpack()); + using MyAnd = AndSearchStrict<NoUnpack>; + return std::make_unique<MyAnd>(std::move(children), NoUnpack()); } else { - return new AndSearchStrict<SelectiveUnpack>(children, SelectiveUnpack(unpackInfo)); + using MyAnd = AndSearchStrict<SelectiveUnpack>; + return std::make_unique<MyAnd>(std::move(children), SelectiveUnpack(unpackInfo)); } } else { if (unpackInfo.unpackAll()) { - return new AndSearchNoStrict<FullUnpack>(children, FullUnpack()); + using MyAnd = AndSearchNoStrict<FullUnpack>; + return std::make_unique<MyAnd>(std::move(children), FullUnpack()); } else if (unpackInfo.empty()) { - return new AndSearchNoStrict<NoUnpack>(children, NoUnpack()); + using MyAnd = AndSearchNoStrict<NoUnpack>; + return std::make_unique<MyAnd>(std::move(children), NoUnpack()); } else { - return new AndSearchNoStrict<SelectiveUnpack>(children, SelectiveUnpack(unpackInfo)); + using MyAnd = AndSearchNoStrict<SelectiveUnpack>; + return std::make_unique<MyAnd>(std::move(children), SelectiveUnpack(unpackInfo)); } } } diff --git a/searchlib/src/vespa/searchlib/queryeval/andsearch.h b/searchlib/src/vespa/searchlib/queryeval/andsearch.h index 59c5769a1a2..b081951e826 100644 --- a/searchlib/src/vespa/searchlib/queryeval/andsearch.h +++ b/searchlib/src/vespa/searchlib/queryeval/andsearch.h @@ -13,9 +13,8 @@ namespace search::queryeval { class AndSearch : public MultiSearch { public: - // Caller takes ownership of the returned SearchIterator. - static AndSearch *create(const Children &children, bool strict, const UnpackInfo & unpackInfo); - static AndSearch *create(const Children &children, bool strict); + static std::unique_ptr<AndSearch> create(ChildrenIterators children, bool strict, const UnpackInfo & unpackInfo); + static std::unique_ptr<AndSearch> create(ChildrenIterators children, bool strict); std::unique_ptr<BitVector> get_hits(uint32_t begin_id) override; void or_hits_into(BitVector &result, uint32_t begin_id) override; @@ -24,7 +23,7 @@ public: AndSearch & estimate(uint32_t est) { _estimate = est; return *this; } uint32_t estimate() const { return _estimate; } protected: - AndSearch(const Children & children); + AndSearch(Children children); void doUnpack(uint32_t docid) override; UP andWith(UP filter, uint32_t estimate) override; UP offerFilterToChildren(UP filter, uint32_t estimate); diff --git a/searchlib/src/vespa/searchlib/queryeval/andsearchnostrict.h b/searchlib/src/vespa/searchlib/queryeval/andsearchnostrict.h index 1b1a5e77445..fac502f4c98 100644 --- a/searchlib/src/vespa/searchlib/queryeval/andsearchnostrict.h +++ b/searchlib/src/vespa/searchlib/queryeval/andsearchnostrict.h @@ -22,8 +22,8 @@ public: * @param children the search objects we are and'ing * ownership of the children is taken by the MultiSearch base class. **/ - AndSearchNoStrict(const Children & children, const Unpack & unpacker) : - AndSearch(children), + AndSearchNoStrict(Children children, const Unpack & unpacker) : + AndSearch(std::move(children)), _unpacker(unpacker) { } diff --git a/searchlib/src/vespa/searchlib/queryeval/andsearchstrict.h b/searchlib/src/vespa/searchlib/queryeval/andsearchstrict.h index 87be4248a0a..7ec179404b6 100644 --- a/searchlib/src/vespa/searchlib/queryeval/andsearchstrict.h +++ b/searchlib/src/vespa/searchlib/queryeval/andsearchstrict.h @@ -23,8 +23,8 @@ protected: Trinary is_strict() const override { return Trinary::True; } SearchIterator::UP andWith(SearchIterator::UP filter, uint32_t estimate) override; public: - AndSearchStrict(const MultiSearch::Children & children, const Unpack & unpacker) : - AndSearchNoStrict<Unpack>(children, unpacker) + AndSearchStrict(MultiSearch::Children children, const Unpack & unpacker) + : AndSearchNoStrict<Unpack>(std::move(children), unpacker) { } diff --git a/searchlib/src/vespa/searchlib/queryeval/blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/blueprint.cpp index 8881bc7da05..1a7a0c5eba2 100644 --- a/searchlib/src/vespa/searchlib/queryeval/blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/blueprint.cpp @@ -400,10 +400,9 @@ IntermediateBlueprint::createSearch(fef::MatchData &md, bool strict) const subSearches.reserve(_children.size()); for (size_t i = 0; i < _children.size(); ++i) { bool strictChild = (strict && inheritStrict(i)); - SearchIterator::UP search = _children[i]->createSearch(md, strictChild); - subSearches.push_back(search.release()); + subSearches.push_back(_children[i]->createSearch(md, strictChild)); } - return createIntermediateSearch(subSearches, strict, md); + return createIntermediateSearch(std::move(subSearches), strict, md); } IntermediateBlueprint::IntermediateBlueprint() = default; diff --git a/searchlib/src/vespa/searchlib/queryeval/blueprint.h b/searchlib/src/vespa/searchlib/queryeval/blueprint.h index 10a68e45d27..ef15736073e 100644 --- a/searchlib/src/vespa/searchlib/queryeval/blueprint.h +++ b/searchlib/src/vespa/searchlib/queryeval/blueprint.h @@ -6,6 +6,7 @@ #include "unpackinfo.h" #include "executeinfo.h" #include "global_filter.h" +#include "multisearch.h" #include <vespa/searchlib/common/bitvector.h> namespace vespalib { class ObjectVisitor; } @@ -297,7 +298,7 @@ public: virtual void sort(std::vector<Blueprint*> &children) const = 0; virtual bool inheritStrict(size_t i) const = 0; virtual SearchIteratorUP - createIntermediateSearch(const std::vector<SearchIterator *> &subSearches, + createIntermediateSearch(MultiSearch::Children subSearches, bool strict, fef::MatchData &md) const = 0; void visitMembers(vespalib::ObjectVisitor &visitor) const override; diff --git a/searchlib/src/vespa/searchlib/queryeval/children_iterators.cpp b/searchlib/src/vespa/searchlib/queryeval/children_iterators.cpp new file mode 100644 index 00000000000..3abbd6a1b81 --- /dev/null +++ b/searchlib/src/vespa/searchlib/queryeval/children_iterators.cpp @@ -0,0 +1,3 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "children_iterators.h" diff --git a/searchlib/src/vespa/searchlib/queryeval/children_iterators.h b/searchlib/src/vespa/searchlib/queryeval/children_iterators.h new file mode 100644 index 00000000000..aa147e0299b --- /dev/null +++ b/searchlib/src/vespa/searchlib/queryeval/children_iterators.h @@ -0,0 +1,39 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "searchiterator.h" + +namespace search::queryeval { + +/** + * Convenience for constructing MultiSearch::Children + * holding them or passing ownership around. + **/ +class ChildrenIterators { + private: + std::vector<SearchIterator::UP> _data; + public: + ChildrenIterators(std::vector<SearchIterator::UP> data) + : _data(std::move(data)) {} + ChildrenIterators(ChildrenIterators && other) = default; + + // convenience constructors for unit tests: + template <typename... Args> + ChildrenIterators(SearchIterator::UP a, Args&&... args) { + _data.reserve(1 + sizeof...(Args)); + _data.push_back(std::move(a)); + (_data.emplace_back(std::forward<Args>(args)), ...); + } + template <typename... Args> + ChildrenIterators(SearchIterator *a, Args&&... args) { + _data.reserve(1 + sizeof...(Args)); + _data.emplace_back(a); + (_data.emplace_back(std::forward<Args>(args)), ...); + } + operator std::vector<SearchIterator::UP> () && { + return std::move(_data); + } +}; + +} // namespace diff --git a/searchlib/src/vespa/searchlib/queryeval/dot_product_blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/dot_product_blueprint.cpp index 741aec98f4f..454db4e820a 100644 --- a/searchlib/src/vespa/searchlib/queryeval/dot_product_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/dot_product_blueprint.cpp @@ -59,6 +59,7 @@ DotProductBlueprint::createLeafSearch(const search::fef::TermFieldMatchDataArray const State &childState = _terms[i]->getState(); assert(childState.numFields() == 1); childMatch.push_back(childState.field(0).resolve(*md)); + // TODO: pass ownership with unique_ptr children[i] = _terms[i]->createSearch(*md, true).release(); } return DotProductSearch::create(children, *tfmda[0], childMatch, _weights, std::move(md)); diff --git a/searchlib/src/vespa/searchlib/queryeval/equiv_blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/equiv_blueprint.cpp index 8a798e6cce3..6d044ca337d 100644 --- a/searchlib/src/vespa/searchlib/queryeval/equiv_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/equiv_blueprint.cpp @@ -57,7 +57,8 @@ SearchIterator::UP EquivBlueprint::createLeafSearch(const fef::TermFieldMatchDataArray &outputs, bool strict) const { fef::MatchData::UP md = _layout.createMatchData(); - MultiSearch::Children children(_terms.size()); + MultiSearch::Children children; + children.reserve(_terms.size()); fef::TermMatchDataMerger::Inputs childMatch; vespalib::hash_map<uint16_t, UnpackNeed> unpack_needs(outputs.size()); for (size_t i = 0; i < outputs.size(); ++i) { @@ -70,20 +71,21 @@ EquivBlueprint::createLeafSearch(const fef::TermFieldMatchDataArray &outputs, bo unpack_needs[child_term_field_match_data->getFieldId()].notify(*child_term_field_match_data); childMatch.emplace_back(child_term_field_match_data, _exactness[i]); } - children[i] = _terms[i]->createSearch(*md, strict).release(); + children.push_back(_terms[i]->createSearch(*md, strict)); } - return SearchIterator::UP(EquivSearch::create(children, std::move(md), childMatch, outputs, strict)); + return EquivSearch::create(std::move(children), std::move(md), childMatch, outputs, strict); } SearchIterator::UP EquivBlueprint::createFilterSearch(bool strict, FilterConstraint constraint) const { - MultiSearch::Children children(_terms.size()); + MultiSearch::Children children; + children.reserve(_terms.size()); for (size_t i = 0; i < _terms.size(); ++i) { - children[i] = _terms[i]->createFilterSearch(strict, constraint).release(); + children.push_back(_terms[i]->createFilterSearch(strict, constraint)); } UnpackInfo unpack_info; - return SearchIterator::UP(OrSearch::create(children, strict, unpack_info)); + return OrSearch::create(std::move(children), strict, unpack_info); } void diff --git a/searchlib/src/vespa/searchlib/queryeval/equivsearch.cpp b/searchlib/src/vespa/searchlib/queryeval/equivsearch.cpp index 593701fd14f..95af4da01b0 100644 --- a/searchlib/src/vespa/searchlib/queryeval/equivsearch.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/equivsearch.cpp @@ -21,19 +21,19 @@ public: * * @param children the search objects that should be equivalent **/ - EquivImpl(const MultiSearch::Children &children, + EquivImpl(MultiSearch::Children children, fef::MatchData::UP inputMatchData, const fef::TermMatchDataMerger::Inputs &inputs, const fef::TermFieldMatchDataArray &outputs); }; template<bool strict> -EquivImpl<strict>::EquivImpl(const MultiSearch::Children &children, +EquivImpl<strict>::EquivImpl(MultiSearch::Children children, fef::MatchData::UP inputMatchData, const fef::TermMatchDataMerger::Inputs &inputs, const fef::TermFieldMatchDataArray &outputs) - : OrLikeSearch<strict, NoUnpack>(children, NoUnpack()), + : OrLikeSearch<strict, NoUnpack>(std::move(children), NoUnpack()), _inputMatchData(std::move(inputMatchData)), _merger(inputs, outputs), _valid(outputs.valid()) @@ -50,17 +50,17 @@ EquivImpl<strict>::doUnpack(uint32_t docid) } } -SearchIterator * -EquivSearch::create(const Children &children, +SearchIterator::UP +EquivSearch::create(Children children, fef::MatchData::UP inputMatchData, const fef::TermMatchDataMerger::Inputs &inputs, const fef::TermFieldMatchDataArray &outputs, bool strict) { if (strict) { - return new EquivImpl<true>(children, std::move(inputMatchData), inputs, outputs); + return std::make_unique<EquivImpl<true>>(std::move(children), std::move(inputMatchData), inputs, outputs); } else { - return new EquivImpl<false>(children, std::move(inputMatchData), inputs, outputs); + return std::make_unique<EquivImpl<false>>(std::move(children), std::move(inputMatchData), inputs, outputs); } } diff --git a/searchlib/src/vespa/searchlib/queryeval/equivsearch.h b/searchlib/src/vespa/searchlib/queryeval/equivsearch.h index 252e17e610a..7dc7f90ee23 100644 --- a/searchlib/src/vespa/searchlib/queryeval/equivsearch.h +++ b/searchlib/src/vespa/searchlib/queryeval/equivsearch.h @@ -18,12 +18,12 @@ class EquivSearch : public SearchIterator public: typedef MultiSearch::Children Children; - // Caller takes ownership of the returned SearchIterator. - static SearchIterator *create(const Children &children, - fef::MatchData::UP inputMD, - const fef::TermMatchDataMerger::Inputs &inputs, - const fef::TermFieldMatchDataArray &outputs, - bool strict); + static SearchIterator::UP + create(Children children, + fef::MatchData::UP inputMD, + const fef::TermMatchDataMerger::Inputs &inputs, + const fef::TermFieldMatchDataArray &outputs, + bool strict); }; } diff --git a/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.cpp b/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.cpp index 14e53d6d868..3d3a703cd7b 100644 --- a/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.cpp @@ -91,12 +91,12 @@ SearchIterator::UP createAndFilter(const IntermediateBlueprint &self, for (size_t i = 0; i < self.childCnt(); ++i) { bool child_strict = strict && (i == 0); auto search = self.getChild(i).createFilterSearch(child_strict, constraint); - sub_searches.push_back(search.release()); + sub_searches.push_back(std::move(search)); } UnpackInfo unpack_info; - AndSearch * search = AndSearch::create(sub_searches, strict, unpack_info); + auto search = AndSearch::create(std::move(sub_searches), strict, unpack_info); search->estimate(self.getState().estimate().estHits); - return SearchIterator::UP(search); + return search; } } // namespace search::queryeval::<unnamed> @@ -166,23 +166,24 @@ AndNotBlueprint::inheritStrict(size_t i) const } SearchIterator::UP -AndNotBlueprint::createIntermediateSearch(const MultiSearch::Children &sub_searches, +AndNotBlueprint::createIntermediateSearch(MultiSearch::Children sub_searches, bool strict, search::fef::MatchData &md) const { UnpackInfo unpack_info(calculateUnpackInfo(md)); if (should_do_termwise_eval(unpack_info, md.get_termwise_limit())) { - TermwiseBlueprintHelper helper(*this, sub_searches, unpack_info); + TermwiseBlueprintHelper helper(*this, std::move(sub_searches), unpack_info); bool termwise_strict = (strict && inheritStrict(helper.first_termwise)); auto termwise_search = (helper.first_termwise == 0) - ? SearchIterator::UP(AndNotSearch::create(helper.termwise, termwise_strict)) - : SearchIterator::UP(OrSearch::create(helper.termwise, termwise_strict)); + ? AndNotSearch::create(helper.get_termwise_children(), termwise_strict) + : OrSearch::create(helper.get_termwise_children(), termwise_strict); helper.insert_termwise(std::move(termwise_search), termwise_strict); - if (helper.children.size() == 1) { - return SearchIterator::UP(helper.children.front()); + auto rearranged = helper.get_result(); + if (rearranged.size() == 1) { + return std::move(rearranged[0]); } - return SearchIterator::UP(AndNotSearch::create(helper.children, strict)); + return AndNotSearch::create(std::move(rearranged), strict); } - return SearchIterator::UP(AndNotSearch::create(sub_searches, strict)); + return AndNotSearch::create(std::move(sub_searches), strict); } namespace { @@ -207,9 +208,9 @@ AndNotBlueprint::createFilterSearch(bool strict, FilterConstraint constraint) co auto search = (i == 0) ? getChild(i).createFilterSearch(child_strict, constraint) : getChild(i).createFilterSearch(child_strict, invert(constraint)); - sub_searches.push_back(search.release()); + sub_searches.push_back(std::move(search)); } - return SearchIterator::UP(AndNotSearch::create(sub_searches, strict)); + return AndNotSearch::create(std::move(sub_searches), strict); } //----------------------------------------------------------------------------- @@ -265,26 +266,27 @@ AndBlueprint::inheritStrict(size_t i) const } SearchIterator::UP -AndBlueprint::createIntermediateSearch(const MultiSearch::Children &sub_searches, +AndBlueprint::createIntermediateSearch(MultiSearch::Children sub_searches, bool strict, search::fef::MatchData & md) const { UnpackInfo unpack_info(calculateUnpackInfo(md)); - AndSearch * search = 0; + std::unique_ptr<AndSearch> search; if (should_do_termwise_eval(unpack_info, md.get_termwise_limit())) { - TermwiseBlueprintHelper helper(*this, sub_searches, unpack_info); + TermwiseBlueprintHelper helper(*this, std::move(sub_searches), unpack_info); bool termwise_strict = (strict && inheritStrict(helper.first_termwise)); - auto termwise_search = SearchIterator::UP(AndSearch::create(helper.termwise, termwise_strict)); + auto termwise_search = AndSearch::create(helper.get_termwise_children(), termwise_strict); helper.insert_termwise(std::move(termwise_search), termwise_strict); - if (helper.children.size() == 1) { - return SearchIterator::UP(helper.children.front()); + auto rearranged = helper.get_result(); + if (rearranged.size() == 1) { + return std::move(rearranged[0]); } else { - search = AndSearch::create(helper.children, strict, helper.termwise_unpack); + search = AndSearch::create(std::move(rearranged), strict, helper.termwise_unpack); } } else { - search = AndSearch::create(sub_searches, strict, unpack_info); + search = AndSearch::create(std::move(sub_searches), strict, unpack_info); } search->estimate(getState().estimate().estHits); - return SearchIterator::UP(search); + return search; } SearchIterator::UP @@ -353,21 +355,22 @@ OrBlueprint::inheritStrict(size_t) const } SearchIterator::UP -OrBlueprint::createIntermediateSearch(const MultiSearch::Children &sub_searches, +OrBlueprint::createIntermediateSearch(MultiSearch::Children sub_searches, bool strict, search::fef::MatchData & md) const { UnpackInfo unpack_info(calculateUnpackInfo(md)); if (should_do_termwise_eval(unpack_info, md.get_termwise_limit())) { - TermwiseBlueprintHelper helper(*this, sub_searches, unpack_info); + TermwiseBlueprintHelper helper(*this, std::move(sub_searches), unpack_info); bool termwise_strict = (strict && inheritStrict(helper.first_termwise)); - auto termwise_search = SearchIterator::UP(OrSearch::create(helper.termwise, termwise_strict)); + auto termwise_search = OrSearch::create(helper.get_termwise_children(), termwise_strict); helper.insert_termwise(std::move(termwise_search), termwise_strict); - if (helper.children.size() == 1) { - return SearchIterator::UP(helper.children.front()); + auto rearranged = helper.get_result(); + if (rearranged.size() == 1) { + return std::move(rearranged[0]); } - return SearchIterator::UP(OrSearch::create(helper.children, strict, helper.termwise_unpack)); + return OrSearch::create(std::move(rearranged), strict, helper.termwise_unpack); } - return SearchIterator::UP(OrSearch::create(sub_searches, strict, unpack_info)); + return OrSearch::create(std::move(sub_searches), strict, unpack_info); } SearchIterator::UP @@ -378,10 +381,10 @@ OrBlueprint::createFilterSearch(bool strict, FilterConstraint constraint) const for (size_t i = 0; i < childCnt(); ++i) { bool child_strict = strict && inheritStrict(i); auto search = getChild(i).createFilterSearch(child_strict, constraint); - sub_searches.push_back(search.release()); + sub_searches.push_back(std::move(search)); } UnpackInfo unpack_info; - return SearchIterator::UP(OrSearch::create(sub_searches, strict, unpack_info)); + return OrSearch::create(std::move(sub_searches), strict, unpack_info); } //----------------------------------------------------------------------------- @@ -423,18 +426,19 @@ WeakAndBlueprint::always_needs_unpack() const } SearchIterator::UP -WeakAndBlueprint::createIntermediateSearch(const MultiSearch::Children &sub_searches, +WeakAndBlueprint::createIntermediateSearch(MultiSearch::Children sub_searches, bool strict, search::fef::MatchData &) const { WeakAndSearch::Terms terms; assert(sub_searches.size() == childCnt()); assert(_weights.size() == childCnt()); for (size_t i = 0; i < sub_searches.size(); ++i) { - terms.push_back(wand::Term(sub_searches[i], + // TODO: pass ownership with unique_ptr + terms.push_back(wand::Term(sub_searches[i].release(), _weights[i], getChild(i).getState().estimate().estHits)); } - return SearchIterator::UP(WeakAndSearch::create(terms, _n, strict)); + return WeakAndSearch::create(terms, _n, strict); } //----------------------------------------------------------------------------- @@ -471,7 +475,7 @@ NearBlueprint::createSearch(fef::MatchData &md, bool strict) const } SearchIterator::UP -NearBlueprint::createIntermediateSearch(const MultiSearch::Children &sub_searches, +NearBlueprint::createIntermediateSearch(MultiSearch::Children sub_searches, bool strict, search::fef::MatchData &md) const { search::fef::TermFieldMatchDataArray tfmda; @@ -481,7 +485,7 @@ NearBlueprint::createIntermediateSearch(const MultiSearch::Children &sub_searche tfmda.add(cs.field(j).resolve(md)); } } - return SearchIterator::UP(new NearSearch(sub_searches, tfmda, _window, strict)); + return SearchIterator::UP(new NearSearch(std::move(sub_searches), tfmda, _window, strict)); } SearchIterator::UP @@ -529,7 +533,7 @@ ONearBlueprint::createSearch(fef::MatchData &md, bool strict) const } SearchIterator::UP -ONearBlueprint::createIntermediateSearch(const MultiSearch::Children &sub_searches, +ONearBlueprint::createIntermediateSearch(MultiSearch::Children sub_searches, bool strict, search::fef::MatchData &md) const { search::fef::TermFieldMatchDataArray tfmda; @@ -541,7 +545,7 @@ ONearBlueprint::createIntermediateSearch(const MultiSearch::Children &sub_search } // could sort sub_searches here // but then strictness inheritance would also need to be fixed - return SearchIterator::UP(new ONearSearch(sub_searches, tfmda, _window, strict)); + return SearchIterator::UP(new ONearSearch(std::move(sub_searches), tfmda, _window, strict)); } SearchIterator::UP @@ -604,27 +608,27 @@ RankBlueprint::inheritStrict(size_t i) const } SearchIterator::UP -RankBlueprint::createIntermediateSearch(const MultiSearch::Children &sub_searches, +RankBlueprint::createIntermediateSearch(MultiSearch::Children sub_searches, bool strict, search::fef::MatchData & md) const { UnpackInfo unpack_info(calculateUnpackInfo(md)); if (unpack_info.unpackAll()) { - return SearchIterator::UP(RankSearch::create(sub_searches, strict)); + return RankSearch::create(std::move(sub_searches), strict); } else { MultiSearch::Children require_unpack; require_unpack.reserve(sub_searches.size()); - require_unpack.push_back(sub_searches[0]); + require_unpack.push_back(std::move(sub_searches[0])); for (size_t i(1); i < sub_searches.size(); i++) { if (unpack_info.needUnpack(i)) { - require_unpack.push_back(sub_searches[i]); + require_unpack.push_back(std::move(sub_searches[i])); } else { - delete sub_searches[i]; + sub_searches[i].reset(); } } if (require_unpack.size() == 1) { - return SearchIterator::UP(require_unpack[0]); + return SearchIterator::UP(std::move(require_unpack[0])); } else { - return SearchIterator::UP(RankSearch::create(require_unpack, strict)); + return RankSearch::create(std::move(require_unpack), strict); } } } @@ -688,18 +692,19 @@ SourceBlenderBlueprint::findSource(uint32_t sourceId) const } SearchIterator::UP -SourceBlenderBlueprint::createIntermediateSearch(const MultiSearch::Children &sub_searches, +SourceBlenderBlueprint::createIntermediateSearch(MultiSearch::Children sub_searches, bool strict, search::fef::MatchData &) const { SourceBlenderSearch::Children children; assert(sub_searches.size() == childCnt()); for (size_t i = 0; i < sub_searches.size(); ++i) { - children.push_back(SourceBlenderSearch::Child(sub_searches[i], + // TODO: pass ownership with unique_ptr + children.push_back(SourceBlenderSearch::Child(sub_searches[i].release(), getChild(i).getSourceId())); assert(children.back().sourceId != 0xffffffff); } - return SearchIterator::UP(SourceBlenderSearch::create(_selector.createIterator(), - children, strict)); + return SourceBlenderSearch::create(_selector.createIterator(), + children, strict); } SearchIterator::UP @@ -711,10 +716,10 @@ SourceBlenderBlueprint::createFilterSearch(bool strict, FilterConstraint constra for (size_t i = 0; i < childCnt(); ++i) { bool child_strict = strict && inheritStrict(i); auto search = getChild(i).createFilterSearch(child_strict, constraint); - sub_searches.push_back(search.release()); + sub_searches.push_back(std::move(search)); } UnpackInfo unpack_info; - return SearchIterator::UP(OrSearch::create(sub_searches, strict, unpack_info)); + return OrSearch::create(std::move(sub_searches), strict, unpack_info); } else { return std::make_unique<EmptySearch>(); } diff --git a/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.h b/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.h index 65dced2ea6b..6bbe4562641 100644 --- a/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.h +++ b/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.h @@ -22,7 +22,7 @@ public: void sort(std::vector<Blueprint*> &children) const override; bool inheritStrict(size_t i) const override; SearchIterator::UP - createIntermediateSearch(const MultiSearch::Children &subSearches, + createIntermediateSearch(MultiSearch::Children subSearches, bool strict, fef::MatchData &md) const override; SearchIterator::UP createFilterSearch(bool strict, FilterConstraint constraint) const override; @@ -49,7 +49,7 @@ public: void sort(std::vector<Blueprint*> &children) const override; bool inheritStrict(size_t i) const override; SearchIterator::UP - createIntermediateSearch(const MultiSearch::Children &subSearches, + createIntermediateSearch(MultiSearch::Children subSearches, bool strict, fef::MatchData &md) const override; SearchIterator::UP createFilterSearch(bool strict, FilterConstraint constraint) const override; @@ -69,7 +69,7 @@ public: void sort(std::vector<Blueprint*> &children) const override; bool inheritStrict(size_t i) const override; SearchIterator::UP - createIntermediateSearch(const MultiSearch::Children &subSearches, + createIntermediateSearch(MultiSearch::Children subSearches, bool strict, fef::MatchData &md) const override; SearchIterator::UP createFilterSearch(bool strict, FilterConstraint constraint) const override; @@ -90,7 +90,7 @@ public: bool inheritStrict(size_t i) const override; bool always_needs_unpack() const override; SearchIterator::UP - createIntermediateSearch(const MultiSearch::Children &subSearches, + createIntermediateSearch(MultiSearch::Children subSearches, bool strict, fef::MatchData &md) const override; WeakAndBlueprint(uint32_t n) : _n(n) {} @@ -118,7 +118,7 @@ public: bool inheritStrict(size_t i) const override; SearchIteratorUP createSearch(fef::MatchData &md, bool strict) const override; SearchIterator::UP - createIntermediateSearch(const MultiSearch::Children &subSearches, + createIntermediateSearch(MultiSearch::Children subSearches, bool strict, fef::MatchData &md) const override; SearchIterator::UP createFilterSearch(bool strict, FilterConstraint constraint) const override; @@ -140,7 +140,7 @@ public: bool inheritStrict(size_t i) const override; SearchIteratorUP createSearch(fef::MatchData &md, bool strict) const override; SearchIterator::UP - createIntermediateSearch(const MultiSearch::Children &subSearches, + createIntermediateSearch(MultiSearch::Children subSearches, bool strict, fef::MatchData &md) const override; SearchIterator::UP createFilterSearch(bool strict, FilterConstraint constraint) const override; @@ -159,7 +159,7 @@ public: void sort(std::vector<Blueprint*> &children) const override; bool inheritStrict(size_t i) const override; SearchIterator::UP - createIntermediateSearch(const MultiSearch::Children &subSearches, + createIntermediateSearch(MultiSearch::Children subSearches, bool strict, fef::MatchData &md) const override; SearchIterator::UP createFilterSearch(bool strict, FilterConstraint constraint) const override; @@ -185,7 +185,7 @@ public: */ ssize_t findSource(uint32_t sourceId) const; SearchIterator::UP - createIntermediateSearch(const MultiSearch::Children &subSearches, + createIntermediateSearch(MultiSearch::Children subSearches, bool strict, fef::MatchData &md) const override; SearchIterator::UP createFilterSearch(bool strict, FilterConstraint constraint) const override; diff --git a/searchlib/src/vespa/searchlib/queryeval/multibitvectoriterator.cpp b/searchlib/src/vespa/searchlib/queryeval/multibitvectoriterator.cpp index b1134c3dc6e..d36d16a679a 100644 --- a/searchlib/src/vespa/searchlib/queryeval/multibitvectoriterator.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/multibitvectoriterator.cpp @@ -21,8 +21,8 @@ template<typename Update> class MultiBitVectorIterator : public MultiBitVectorIteratorBase { public: - explicit MultiBitVectorIterator(const Children & children) - : MultiBitVectorIteratorBase(children), + explicit MultiBitVectorIterator(Children children) + : MultiBitVectorIteratorBase(std::move(children)), _update(), _accel(IAccelrated::getAccelerator()), _lastWords() @@ -46,8 +46,8 @@ template<typename Update> class MultiBitVectorIteratorStrict : public MultiBitVectorIterator<Update> { public: - explicit MultiBitVectorIteratorStrict(const MultiSearch::Children & children) - : MultiBitVectorIterator<Update>(children) + explicit MultiBitVectorIteratorStrict(MultiSearch::Children children) + : MultiBitVectorIterator<Update>(std::move(children)) { } private: void doSeek(uint32_t docId) override { this->strictSeek(docId); } @@ -127,7 +127,7 @@ typedef MultiBitVectorIteratorStrict<Or> OrBVIteratorStrict; bool hasAtLeast2Bitvectors(const MultiSearch::Children & children) { size_t count(0); - for (const SearchIterator * search : children) { + for (const auto & search : children) { if (search->isBitVector()) { count++; } @@ -148,17 +148,17 @@ bool canOptimize(const MultiSearch & s) { } -MultiBitVectorIteratorBase::MultiBitVectorIteratorBase(const Children & children) : - MultiSearch(children), +MultiBitVectorIteratorBase::MultiBitVectorIteratorBase(Children children) : + MultiSearch(std::move(children)), _numDocs(std::numeric_limits<unsigned int>::max()), _lastMaxDocIdLimit(0), _lastMaxDocIdLimitRequireFetch(0), _lastValue(0), _bvs() { - _bvs.reserve(children.size()); - for (const auto & child : children) { - const auto * bv = static_cast<const BitVectorIterator *>(child); + _bvs.reserve(getChildren().size()); + for (const auto & child : getChildren()) { + const auto * bv = static_cast<const BitVectorIterator *>(child.get()); _bvs.emplace_back(bv->getBitValues(), bv->isInverted()); _numDocs = std::min(_numDocs, bv->getDocIdLimit()); } @@ -196,8 +196,8 @@ MultiBitVectorIteratorBase::doUnpack(uint32_t docid) } else { auto &children = getChildren(); _unpackInfo.each([&children,docid](size_t i) { - static_cast<BitVectorIterator *>(children[i])->unpack(docid); - }, children.size()); + static_cast<BitVectorIterator *>(children[i].get())->unpack(docid); + }, children.size()); } } @@ -236,7 +236,7 @@ MultiBitVectorIteratorBase::optimizeMultiSearch(SearchIterator::UP parentIt) if ( ! strict && (bit->is_strict() == Trinary::True)) { strict = true; } - stolen.push_back(bit.release()); + stolen.push_back(std::move(bit)); } else { it++; } @@ -244,21 +244,21 @@ MultiBitVectorIteratorBase::optimizeMultiSearch(SearchIterator::UP parentIt) SearchIterator::UP next; if (parent.isAnd()) { if (strict) { - next = std::make_unique<AndBVIteratorStrict>(stolen); + next = std::make_unique<AndBVIteratorStrict>(std::move(stolen)); } else { - next = std::make_unique<AndBVIterator>(stolen); + next = std::make_unique<AndBVIterator>(std::move(stolen)); } } else if (parent.isOr()) { if (strict) { - next = std::make_unique<OrBVIteratorStrict>(stolen); + next = std::make_unique<OrBVIteratorStrict>(std::move(stolen)); } else { - next = std::make_unique<OrBVIterator>(stolen); + next = std::make_unique<OrBVIterator>(std::move(stolen)); } } else if (parent.isAndNot()) { if (strict) { - next = std::make_unique<OrBVIteratorStrict>(stolen); + next = std::make_unique<OrBVIteratorStrict>(std::move(stolen)); } else { - next = std::make_unique<OrBVIterator>(stolen); + next = std::make_unique<OrBVIterator>(std::move(stolen)); } } auto & nextM(static_cast<MultiBitVectorIteratorBase &>(*next)); @@ -272,8 +272,8 @@ MultiBitVectorIteratorBase::optimizeMultiSearch(SearchIterator::UP parentIt) } } auto & toOptimize(const_cast<MultiSearch::Children &>(parent.getChildren())); - for (SearchIterator * & search : toOptimize) { - search = optimize(MultiSearch::UP(search)).release(); + for (auto & search : toOptimize) { + search = optimize(std::move(search)); } return parentIt; diff --git a/searchlib/src/vespa/searchlib/queryeval/multibitvectoriterator.h b/searchlib/src/vespa/searchlib/queryeval/multibitvectoriterator.h index dbe2d6f8965..29e92584ffe 100644 --- a/searchlib/src/vespa/searchlib/queryeval/multibitvectoriterator.h +++ b/searchlib/src/vespa/searchlib/queryeval/multibitvectoriterator.h @@ -20,7 +20,7 @@ public: */ static SearchIterator::UP optimize(SearchIterator::UP parent); protected: - MultiBitVectorIteratorBase(const Children & children); + MultiBitVectorIteratorBase(Children hildren); using MetaWord = std::pair<const void *, bool>; uint32_t _numDocs; diff --git a/searchlib/src/vespa/searchlib/queryeval/multisearch.cpp b/searchlib/src/vespa/searchlib/queryeval/multisearch.cpp index b63a54785a4..51098f50b37 100644 --- a/searchlib/src/vespa/searchlib/queryeval/multisearch.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/multisearch.cpp @@ -10,7 +10,7 @@ void MultiSearch::insert(size_t index, SearchIterator::UP search) { assert(index <= _children.size()); - _children.insert(_children.begin()+index, search.release()); + _children.insert(_children.begin()+index, std::move(search)); onInsert(index); } @@ -18,7 +18,7 @@ SearchIterator::UP MultiSearch::remove(size_t index) { assert(index < _children.size()); - SearchIterator::UP search(_children[index]); + SearchIterator::UP search = std::move(_children[index]); _children.erase(_children.begin() + index); onRemove(index); return search; @@ -27,7 +27,7 @@ MultiSearch::remove(size_t index) void MultiSearch::doUnpack(uint32_t docid) { - for (SearchIterator *child: _children) { + for (auto &child: _children) { if (__builtin_expect(child->getDocId() < docid, false)) { child->doSeek(docid); } @@ -37,23 +37,20 @@ MultiSearch::doUnpack(uint32_t docid) } } -MultiSearch::MultiSearch(const Children & children) - : _children(children) +MultiSearch::MultiSearch(Children children) + : _children(std::move(children)) { } MultiSearch::~MultiSearch() { - for (SearchIterator * child : _children) { - delete child; - } } void MultiSearch::initRange(uint32_t beginid, uint32_t endid) { SearchIterator::initRange(beginid, endid); - for (SearchIterator * child : _children) { + for (auto & child : _children) { child->initRange(beginid, endid); } } diff --git a/searchlib/src/vespa/searchlib/queryeval/multisearch.h b/searchlib/src/vespa/searchlib/queryeval/multisearch.h index af96734b26a..bd916f7953b 100644 --- a/searchlib/src/vespa/searchlib/queryeval/multisearch.h +++ b/searchlib/src/vespa/searchlib/queryeval/multisearch.h @@ -3,6 +3,7 @@ #pragma once #include "searchiterator.h" +#include "children_iterators.h" struct MultiSearchRemoveTest; @@ -12,25 +13,26 @@ class MultiBitVectorIteratorBase; /** * A virtual intermediate class that serves as the basis for combining searches - * like and, or any or others that take a list of children. + * like AND, OR, RANK or others that take a list of children. **/ class MultiSearch : public SearchIterator { friend struct ::MultiSearchRemoveTest; friend class ::search::queryeval::MultiBitVectorIteratorBase; + friend class MySearch; public: /** - * Defines how to represent the children iterators. vespalib::Array usage - * generates faster and more compact code then using std::vector. + * Defines how to represent the children iterators. */ - typedef std::vector<SearchIterator *> Children; + using Children = std::vector<SearchIterator::UP>; + /** * Create a new Multi Search with the given children. * * @param children the search objects we are and'ing * this object takes ownership of the children. **/ - MultiSearch(const Children & children); + MultiSearch(Children children); virtual ~MultiSearch() override; const Children & getChildren() const { return _children; } virtual bool isAnd() const { return false; } @@ -40,6 +42,7 @@ public: virtual bool needUnpack(size_t index) const { (void) index; return true; } void initRange(uint32_t beginId, uint32_t endId) override; protected: + MultiSearch() {} void doUnpack(uint32_t docid) override; void visitMembers(vespalib::ObjectVisitor &visitor) const override; private: diff --git a/searchlib/src/vespa/searchlib/queryeval/nearsearch.cpp b/searchlib/src/vespa/searchlib/queryeval/nearsearch.cpp index 4ecd84fb7c3..f7b83de5f4b 100644 --- a/searchlib/src/vespa/searchlib/queryeval/nearsearch.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/nearsearch.cpp @@ -31,11 +31,11 @@ void setup_fields(uint32_t window, std::vector<T> &matchers, const TermFieldMatc } // namespace search::queryeval::<unnamed> -NearSearchBase::NearSearchBase(const Children & terms, +NearSearchBase::NearSearchBase(Children terms, const TermFieldMatchDataArray &data, uint32_t window, bool strict) - : AndSearch(terms), + : AndSearch(std::move(terms)), _data_size(data.size()), _window(window), _strict(strict) @@ -107,8 +107,7 @@ NearSearchBase::doSeek(uint32_t docId) const Children & terms(getChildren()); bool foundHit = true; for (uint32_t i = 0, len = terms.size(); i < len; ++i) { - SearchIterator *term = terms[i]; - if (!term->seek(docId)) { + if (! terms[i]->seek(docId)) { LOG(debug, "Term %d does not occur in document %d.", i, docId); foundHit = false; break; @@ -123,11 +122,11 @@ NearSearchBase::doSeek(uint32_t docId) } } -NearSearch::NearSearch(const Children & terms, +NearSearch::NearSearch(Children terms, const TermFieldMatchDataArray &data, uint32_t window, bool strict) - : NearSearchBase(terms, data, window, strict), + : NearSearchBase(std::move(terms), data, window, strict), _matchers() { setup_fields(window, _matchers, data); @@ -224,11 +223,11 @@ NearSearch::match(uint32_t docId) return false; } -ONearSearch::ONearSearch(const Children & terms, +ONearSearch::ONearSearch(Children terms, const TermFieldMatchDataArray &data, uint32_t window, bool strict) - : NearSearchBase(terms, data, window, strict), + : NearSearchBase(std::move(terms), data, window, strict), _matchers() { setup_fields(window, _matchers, data); diff --git a/searchlib/src/vespa/searchlib/queryeval/nearsearch.h b/searchlib/src/vespa/searchlib/queryeval/nearsearch.h index 27c862735b1..13bd1c53383 100644 --- a/searchlib/src/vespa/searchlib/queryeval/nearsearch.h +++ b/searchlib/src/vespa/searchlib/queryeval/nearsearch.h @@ -72,7 +72,7 @@ public: * @param window The size of the window in which all terms must occur. * @param strict Whether or not to skip to next matching document if seek fails. */ - NearSearchBase(const Children & terms, + NearSearchBase(Children terms, const TermFieldMatchDataArray &data, uint32_t window, bool strict); @@ -106,7 +106,7 @@ public: * @param window The size of the window in which all terms must occur. * @param strict Whether or not to skip to next matching document if seek fails. */ - NearSearch(const Children & terms, + NearSearch(Children terms, const TermFieldMatchDataArray &data, uint32_t window, bool strict = true); @@ -138,7 +138,7 @@ public: * @param window The size of the window in which all terms must occur. * @param strict Whether or not to skip to next matching document if seek fails. */ - ONearSearch(const Children & terms, + ONearSearch(Children terms, const TermFieldMatchDataArray &data, uint32_t window, bool strict = true); diff --git a/searchlib/src/vespa/searchlib/queryeval/orlikesearch.h b/searchlib/src/vespa/searchlib/queryeval/orlikesearch.h index 1dec50ffc97..57a2ff11997 100644 --- a/searchlib/src/vespa/searchlib/queryeval/orlikesearch.h +++ b/searchlib/src/vespa/searchlib/queryeval/orlikesearch.h @@ -46,8 +46,8 @@ public: * * @param children the search objects we are or'ing **/ - OrLikeSearch(const Children &children, const Unpack & unpacker) : - OrSearch(children), + OrLikeSearch(Children children, const Unpack & unpacker) + : OrSearch(std::move(children)), _unpacker(unpacker) { } private: diff --git a/searchlib/src/vespa/searchlib/queryeval/orsearch.cpp b/searchlib/src/vespa/searchlib/queryeval/orsearch.cpp index b5e0ff9c423..9d1dd252796 100644 --- a/searchlib/src/vespa/searchlib/queryeval/orsearch.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/orsearch.cpp @@ -78,30 +78,36 @@ OrSearch::or_hits_into(BitVector &result, uint32_t begin_id) TermwiseHelper::orChildren(result, getChildren().begin(), getChildren().end(), begin_id); } -SearchIterator * -OrSearch::create(const MultiSearch::Children &children, bool strict) { +SearchIterator::UP +OrSearch::create(ChildrenIterators children, bool strict) { UnpackInfo unpackInfo; unpackInfo.forceAll(); - return create(children, strict, unpackInfo); + return create(std::move(children), strict, unpackInfo); } -SearchIterator * -OrSearch::create(const MultiSearch::Children &children, bool strict, const UnpackInfo & unpackInfo) { +SearchIterator::UP +OrSearch::create(ChildrenIterators children, bool strict, const UnpackInfo & unpackInfo) { if (strict) { if (unpackInfo.unpackAll()) { - return new OrLikeSearch<true, FullUnpack>(children, FullUnpack()); + using MyOr = OrLikeSearch<true, FullUnpack>; + return std::make_unique<MyOr>(std::move(children), FullUnpack()); } else if(unpackInfo.empty()) { - return new OrLikeSearch<true, NoUnpack>(children, NoUnpack()); + using MyOr = OrLikeSearch<true, NoUnpack>; + return std::make_unique<MyOr>(std::move(children), NoUnpack()); } else { - return new OrLikeSearch<true, SelectiveUnpack>(children, SelectiveUnpack(unpackInfo)); + using MyOr = OrLikeSearch<true, SelectiveUnpack>; + return std::make_unique<MyOr>(std::move(children), SelectiveUnpack(unpackInfo)); } } else { if (unpackInfo.unpackAll()) { - return new OrLikeSearch<false, FullUnpack>(children, FullUnpack()); + using MyOr = OrLikeSearch<false, FullUnpack>; + return std::make_unique<MyOr>(std::move(children), FullUnpack()); } else if(unpackInfo.empty()) { - return new OrLikeSearch<false, NoUnpack>(children, NoUnpack()); + using MyOr = OrLikeSearch<false, NoUnpack>; + return std::make_unique<MyOr>(std::move(children), NoUnpack()); } else { - return new OrLikeSearch<false, SelectiveUnpack>(children, SelectiveUnpack(unpackInfo)); + using MyOr = OrLikeSearch<false, SelectiveUnpack>; + return std::make_unique<MyOr>(std::move(children), SelectiveUnpack(unpackInfo)); } } } diff --git a/searchlib/src/vespa/searchlib/queryeval/orsearch.h b/searchlib/src/vespa/searchlib/queryeval/orsearch.h index e3d74573db8..79da1b85b6f 100644 --- a/searchlib/src/vespa/searchlib/queryeval/orsearch.h +++ b/searchlib/src/vespa/searchlib/queryeval/orsearch.h @@ -15,16 +15,15 @@ class OrSearch : public MultiSearch public: typedef MultiSearch::Children Children; - // Caller takes ownership of the returned SearchIterator. - static SearchIterator *create(const Children &children, bool strict); - static SearchIterator *create(const Children &children, bool strict, const UnpackInfo & unpackInfo); + static SearchIterator::UP create(ChildrenIterators children, bool strict); + static SearchIterator::UP create(ChildrenIterators children, bool strict, const UnpackInfo & unpackInfo); std::unique_ptr<BitVector> get_hits(uint32_t begin_id) override; void or_hits_into(BitVector &result, uint32_t begin_id) override; void and_hits_into(BitVector &result, uint32_t begin_id) override; protected: - OrSearch(const Children & children) : MultiSearch(children) { } + OrSearch(Children children) : MultiSearch(std::move(children)) { } private: bool isOr() const override { return true; } diff --git a/searchlib/src/vespa/searchlib/queryeval/ranksearch.cpp b/searchlib/src/vespa/searchlib/queryeval/ranksearch.cpp index 2bcf2267b1d..a915442bf0c 100644 --- a/searchlib/src/vespa/searchlib/queryeval/ranksearch.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/ranksearch.cpp @@ -32,7 +32,7 @@ public: * * @param children the search objects we are rank'ing **/ - RankSearchStrict(const Children & children) : RankSearch(children) { } + RankSearchStrict(Children children) : RankSearch(std::move(children)) { } }; SearchIterator::UP @@ -49,12 +49,12 @@ RankSearchStrict::doSeek(uint32_t docid) } } // namespace -SearchIterator * -RankSearch::create(const RankSearch::Children &children, bool strict) { +SearchIterator::UP +RankSearch::create(ChildrenIterators children, bool strict) { if (strict) { - return new RankSearchStrict(children); + return UP(new RankSearchStrict(std::move(children))); } else { - return new RankSearch(children); + return UP(new RankSearch(std::move(children))); } } diff --git a/searchlib/src/vespa/searchlib/queryeval/ranksearch.h b/searchlib/src/vespa/searchlib/queryeval/ranksearch.h index 60efed3c694..443202f3e59 100644 --- a/searchlib/src/vespa/searchlib/queryeval/ranksearch.h +++ b/searchlib/src/vespa/searchlib/queryeval/ranksearch.h @@ -20,11 +20,10 @@ protected: * * @param children the search objects we are rank'ing **/ - RankSearch(const Children & children) : MultiSearch(children) { } + RankSearch(Children children) : MultiSearch(std::move(children)) { } public: - // Caller takes ownership of the returned SearchIterator. - static SearchIterator *create(const Children &children, bool strict); + static SearchIterator::UP create(ChildrenIterators children, bool strict); }; } diff --git a/searchlib/src/vespa/searchlib/queryeval/simple_phrase_blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/simple_phrase_blueprint.cpp index fc29c7ef45d..83d5a5e1739 100644 --- a/searchlib/src/vespa/searchlib/queryeval/simple_phrase_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/simple_phrase_blueprint.cpp @@ -61,7 +61,8 @@ SimplePhraseBlueprint::createLeafSearch(const fef::TermFieldMatchDataArray &tfmd assert(tfmda.size() == 1); fef::MatchData::UP md = _layout.createMatchData(); fef::TermFieldMatchDataArray childMatch; - SimplePhraseSearch::Children children(_terms.size()); + SimplePhraseSearch::Children children; + children.reserve(_terms.size()); std::multimap<uint32_t, uint32_t> order_map; for (size_t i = 0; i < _terms.size(); ++i) { const State &childState = _terms[i]->getState(); @@ -70,7 +71,7 @@ SimplePhraseBlueprint::createLeafSearch(const fef::TermFieldMatchDataArray &tfmd child_term_field_match_data->setNeedInterleavedFeatures(tfmda[0]->needs_interleaved_features()); child_term_field_match_data->setNeedNormalFeatures(true); childMatch.add(child_term_field_match_data); - children[i] = _terms[i]->createSearch(*md, strict).release(); + children.push_back(_terms[i]->createSearch(*md, strict)); order_map.insert(std::make_pair(childState.estimate().estHits, i)); } std::vector<uint32_t> eval_order; @@ -78,7 +79,8 @@ SimplePhraseBlueprint::createLeafSearch(const fef::TermFieldMatchDataArray &tfmd eval_order.push_back(child.second); } - auto phrase = std::make_unique<SimplePhraseSearch>(children, std::move(md), childMatch, + auto phrase = std::make_unique<SimplePhraseSearch>(std::move(children), + std::move(md), childMatch, eval_order, *tfmda[0], strict); phrase->setDoom(& _doom); return phrase; @@ -88,13 +90,14 @@ SearchIterator::UP SimplePhraseBlueprint::createFilterSearch(bool strict, FilterConstraint constraint) const { if (constraint == FilterConstraint::UPPER_BOUND) { - MultiSearch::Children children(_terms.size()); + MultiSearch::Children children; + children.reserve(_terms.size()); for (size_t i = 0; i < _terms.size(); ++i) { bool child_strict = strict && (i == 0); - children[i] = _terms[i]->createFilterSearch(child_strict, constraint).release(); + children.push_back(_terms[i]->createFilterSearch(child_strict, constraint)); } UnpackInfo unpack_info; - return SearchIterator::UP(AndSearch::create(children, strict, unpack_info)); + return AndSearch::create(std::move(children), strict, unpack_info); } else { return std::make_unique<EmptySearch>(); } diff --git a/searchlib/src/vespa/searchlib/queryeval/simple_phrase_search.cpp b/searchlib/src/vespa/searchlib/queryeval/simple_phrase_search.cpp index df0dff06582..43a9ee4ab91 100644 --- a/searchlib/src/vespa/searchlib/queryeval/simple_phrase_search.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/simple_phrase_search.cpp @@ -157,23 +157,23 @@ SimplePhraseSearch::phraseSeek(uint32_t doc_id) { } -SimplePhraseSearch::SimplePhraseSearch(const Children &children, +SimplePhraseSearch::SimplePhraseSearch(Children children, fef::MatchData::UP md, const fef::TermFieldMatchDataArray &childMatch, vector<uint32_t> eval_order, TermFieldMatchData &tmd, bool strict) - : AndSearch(children), + : AndSearch(std::move(children)), _md(std::move(md)), _childMatch(childMatch), _eval_order(std::move(eval_order)), _tmd(tmd), _doom(nullptr), _strict(strict), - _iterators(children.size()) + _iterators(getChildren().size()) { - assert(!children.empty()); - assert(children.size() == _childMatch.size()); - assert(children.size() == _eval_order.size()); + assert(getChildren().size() > 0); + assert(getChildren().size() == _childMatch.size()); + assert(getChildren().size() == _eval_order.size()); } void diff --git a/searchlib/src/vespa/searchlib/queryeval/simple_phrase_search.h b/searchlib/src/vespa/searchlib/queryeval/simple_phrase_search.h index d45e67ed4cb..5cec039e733 100644 --- a/searchlib/src/vespa/searchlib/queryeval/simple_phrase_search.h +++ b/searchlib/src/vespa/searchlib/queryeval/simple_phrase_search.h @@ -43,7 +43,7 @@ public: * terms. The term with fewest hits should be * evaluated first. **/ - SimplePhraseSearch(const Children &children, + SimplePhraseSearch(Children children, fef::MatchData::UP md, const fef::TermFieldMatchDataArray &childMatch, std::vector<uint32_t> eval_order, diff --git a/searchlib/src/vespa/searchlib/queryeval/sourceblendersearch.cpp b/searchlib/src/vespa/searchlib/queryeval/sourceblendersearch.cpp index 3280e0ac2cc..b50e71d5d53 100644 --- a/searchlib/src/vespa/searchlib/queryeval/sourceblendersearch.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/sourceblendersearch.cpp @@ -10,6 +10,14 @@ namespace search::queryeval { EmptySearch SourceBlenderSearch::_emptySearch; +class SourceBlenderSearchNonStrict : public SourceBlenderSearch +{ +public: + SourceBlenderSearchNonStrict(std::unique_ptr<Iterator> sourceSelector, const Children &children) + : SourceBlenderSearch(std::move(sourceSelector), children) + {} +}; + class SourceBlenderSearchStrict : public SourceBlenderSearch { public: @@ -158,14 +166,14 @@ SourceBlenderSearch::setChild(size_t index, SearchIterator::UP child) { _sources[_children[index]] = child.release(); } -SourceBlenderSearch * +SearchIterator::UP SourceBlenderSearch::create(std::unique_ptr<sourceselector::Iterator> sourceSelector, const Children &children, bool strict) { if (strict) { - return new SourceBlenderSearchStrict(std::move(sourceSelector), children); + return std::make_unique<SourceBlenderSearchStrict>(std::move(sourceSelector), children); } else { - return new SourceBlenderSearch(std::move(sourceSelector), children); + return std::make_unique<SourceBlenderSearchNonStrict>(std::move(sourceSelector), children); } } diff --git a/searchlib/src/vespa/searchlib/queryeval/sourceblendersearch.h b/searchlib/src/vespa/searchlib/queryeval/sourceblendersearch.h index 9b4d8f58034..f209e0f7fd8 100644 --- a/searchlib/src/vespa/searchlib/queryeval/sourceblendersearch.h +++ b/searchlib/src/vespa/searchlib/queryeval/sourceblendersearch.h @@ -68,8 +68,9 @@ public: * @param strict whether this search is strict * (a strict search will locate its next hit when seeking fails) **/ - static SourceBlenderSearch * create(std::unique_ptr<Iterator> sourceSelector, - const Children &children, bool strict); + static SearchIterator::UP create(std::unique_ptr<Iterator> sourceSelector, + const Children &children, + bool strict); ~SourceBlenderSearch() override; size_t getNumChildren() const { return _children.size(); } SearchIterator::UP steal(size_t index) { diff --git a/searchlib/src/vespa/searchlib/queryeval/termwise_blueprint_helper.cpp b/searchlib/src/vespa/searchlib/queryeval/termwise_blueprint_helper.cpp index ae21fd93ba3..e26a1652441 100644 --- a/searchlib/src/vespa/searchlib/queryeval/termwise_blueprint_helper.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/termwise_blueprint_helper.cpp @@ -6,27 +6,27 @@ namespace search::queryeval { TermwiseBlueprintHelper::TermwiseBlueprintHelper(const IntermediateBlueprint &self, - const MultiSearch::Children &subSearches, + MultiSearch::Children subSearches, UnpackInfo &unpackInfo) - : children(), - termwise(), + : termwise_ch(), + other_ch(), first_termwise(subSearches.size()), termwise_unpack() { - children.reserve(subSearches.size()); - termwise.reserve(subSearches.size()); + other_ch.reserve(subSearches.size()); + termwise_ch.reserve(subSearches.size()); for (size_t i = 0; i < subSearches.size(); ++i) { bool need_unpack = unpackInfo.needUnpack(i); bool allow_termwise = self.getChild(i).getState().allow_termwise_eval(); if (need_unpack || !allow_termwise) { if (need_unpack) { - size_t index = (i < first_termwise) ? children.size() : (children.size() + 1); + size_t index = (i < first_termwise) ? other_ch.size() : (other_ch.size() + 1); termwise_unpack.add(index); } - children.push_back(subSearches[i]); + other_ch.push_back(std::move(subSearches[i])); } else { first_termwise = std::min(i, first_termwise); - termwise.push_back(subSearches[i]); + termwise_ch.push_back(std::move(subSearches[i])); } } } @@ -37,7 +37,7 @@ void TermwiseBlueprintHelper::insert_termwise(SearchIterator::UP search, bool strict) { auto termwise_search = make_termwise(std::move(search), strict); - children.insert(children.begin() + first_termwise, termwise_search.release()); + other_ch.insert(other_ch.begin() + first_termwise, std::move(termwise_search)); } } diff --git a/searchlib/src/vespa/searchlib/queryeval/termwise_blueprint_helper.h b/searchlib/src/vespa/searchlib/queryeval/termwise_blueprint_helper.h index e6b46cfb7d2..2917ce4d8c9 100644 --- a/searchlib/src/vespa/searchlib/queryeval/termwise_blueprint_helper.h +++ b/searchlib/src/vespa/searchlib/queryeval/termwise_blueprint_helper.h @@ -15,13 +15,18 @@ namespace search::queryeval { * termwise and non-termwise parts with each other. **/ struct TermwiseBlueprintHelper { - MultiSearch::Children children; - MultiSearch::Children termwise; +private: + MultiSearch::Children termwise_ch; + MultiSearch::Children other_ch; +public: size_t first_termwise; UnpackInfo termwise_unpack; + MultiSearch::Children get_termwise_children() { return std::move(termwise_ch); } + MultiSearch::Children get_result() { return std::move(other_ch); } + TermwiseBlueprintHelper(const IntermediateBlueprint &self, - const MultiSearch::Children &subSearches, UnpackInfo &unpackInfo); + MultiSearch::Children subSearches, UnpackInfo &unpackInfo); ~TermwiseBlueprintHelper(); void insert_termwise(SearchIterator::UP search, bool strict); diff --git a/searchlib/src/vespa/searchlib/queryeval/test/leafspec.h b/searchlib/src/vespa/searchlib/queryeval/test/leafspec.h index 47b5ed26b60..fa14941844e 100644 --- a/searchlib/src/vespa/searchlib/queryeval/test/leafspec.h +++ b/searchlib/src/vespa/searchlib/queryeval/test/leafspec.h @@ -18,7 +18,7 @@ struct LeafSpec int32_t weight; int32_t maxWeight; FakeResult result; - SearchIterator *search; + SearchIterator::UP search; LeafSpec(const std::string &n, int32_t w = 100) : name(n), weight(w), @@ -26,31 +26,36 @@ struct LeafSpec result(), search() {} + LeafSpec(LeafSpec && other) = default; ~LeafSpec() {} - LeafSpec &doc(uint32_t docid) { + LeafSpec && doc(uint32_t docid) && { result.doc(docid); - return *this; + return std::move(*this); } - LeafSpec &doc(uint32_t docid, int32_t w) { + LeafSpec && doc(uint32_t docid, int32_t w) && { result.doc(docid); result.weight(w); result.pos(0); maxWeight = std::max(maxWeight, w); - return *this; + return std::move(*this); } - LeafSpec &itr(SearchIterator *si) { - search = si; - return *this; + LeafSpec && itr(SearchIterator::UP si) && { + search = std::move(si); + return std::move(*this); } - SearchIterator *create(SearchHistory &hist, fef::TermFieldMatchData *tfmd) const { - if (search != nullptr) { - return new TrackedSearch(name, hist, search); + LeafSpec && itr(SearchIterator *si) && { + search.reset(si); + return std::move(*this); + } + SearchIterator::UP create(SearchHistory &hist, fef::TermFieldMatchData *tfmd) { + if (search) { + return SearchIterator::UP(new TrackedSearch(name, hist, std::move(search))); } else if (tfmd != nullptr) { - return new TrackedSearch(name, hist, result, *tfmd, - MinMaxPostingInfo(0, maxWeight)); + return SearchIterator::UP(new TrackedSearch(name, hist, result, *tfmd, + MinMaxPostingInfo(0, maxWeight))); } - return new TrackedSearch(name, hist, result, - MinMaxPostingInfo(0, maxWeight)); + return SearchIterator::UP(new TrackedSearch(name, hist, result, + MinMaxPostingInfo(0, maxWeight))); } }; diff --git a/searchlib/src/vespa/searchlib/queryeval/test/trackedsearch.h b/searchlib/src/vespa/searchlib/queryeval/test/trackedsearch.h index 6cb4c1a9dda..ae04d35e658 100644 --- a/searchlib/src/vespa/searchlib/queryeval/test/trackedsearch.h +++ b/searchlib/src/vespa/searchlib/queryeval/test/trackedsearch.h @@ -56,6 +56,12 @@ public: _search(new FakeSearch("<tag>", "<field>", "<term>", result, makeArray(tfmd))), _minMaxPostingInfo(new MinMaxPostingInfo(minMaxPostingInfo)) { setDocId(_search->getDocId()); } + + // wraps a generic search (typically wand) + TrackedSearch(const std::string &name, SearchHistory &hist, SearchIterator::UP search) + : _name(name), _history(hist), _matchData(), _search(std::move(search)), _minMaxPostingInfo() + { setDocId(_search->getDocId()); } + // wraps a generic search (typically wand) TrackedSearch(const std::string &name, SearchHistory &hist, SearchIterator *search) : _name(name), _history(hist), _matchData(), _search(search), _minMaxPostingInfo() diff --git a/searchlib/src/vespa/searchlib/queryeval/test/wandspec.h b/searchlib/src/vespa/searchlib/queryeval/test/wandspec.h index bf456c287d6..2fb2b3bc9e2 100644 --- a/searchlib/src/vespa/searchlib/queryeval/test/wandspec.h +++ b/searchlib/src/vespa/searchlib/queryeval/test/wandspec.h @@ -26,8 +26,8 @@ private: public: WandSpec() : _leafs(), _layout(), _handles(), _history() {} ~WandSpec() {} - WandSpec &leaf(const LeafSpec &l) { - _leafs.push_back(l); + WandSpec &leaf(LeafSpec && l) { + _leafs.emplace_back(std::move(l)); _handles.push_back(_layout.allocTermField(0)); return *this; } @@ -35,7 +35,7 @@ public: wand::Terms terms; for (size_t i = 0; i < _leafs.size(); ++i) { fef::TermFieldMatchData *tfmd = (matchData != NULL ? matchData->resolveTermField(_handles[i]) : NULL); - terms.push_back(wand::Term(_leafs[i].create(_history, tfmd), + terms.push_back(wand::Term(_leafs[i].create(_history, tfmd).release(), _leafs[i].weight, _leafs[i].result.inspect().size(), tfmd)); diff --git a/searchlib/src/vespa/searchlib/queryeval/wand/parallel_weak_and_blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/wand/parallel_weak_and_blueprint.cpp index 7ad4a36f871..d4f16e2f91c 100644 --- a/searchlib/src/vespa/searchlib/queryeval/wand/parallel_weak_and_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/wand/parallel_weak_and_blueprint.cpp @@ -86,6 +86,7 @@ ParallelWeakAndBlueprint::createLeafSearch(const search::fef::TermFieldMatchData for (size_t i = 0; i < _terms.size(); ++i) { const State &childState = _terms[i]->getState(); assert(childState.numFields() == 1); + // TODO: pass ownership with unique_ptr terms.push_back(wand::Term(_terms[i]->createSearch(*childrenMatchData, true).release(), _weights[i], childState.estimate().estHits, diff --git a/searchlib/src/vespa/searchlib/queryeval/wand/wand_parts.h b/searchlib/src/vespa/searchlib/queryeval/wand/wand_parts.h index bd60473e05d..071d6d99470 100644 --- a/searchlib/src/vespa/searchlib/queryeval/wand/wand_parts.h +++ b/searchlib/src/vespa/searchlib/queryeval/wand/wand_parts.h @@ -35,6 +35,7 @@ typedef std::vector<AttrDictEntry> AttrDictEntries; * Wrapper used to specify underlying terms during setup **/ struct Term { + // TODO: use unique_ptr for ownership SearchIterator *search; int32_t weight; uint32_t estHits; @@ -44,6 +45,7 @@ struct Term { : search(s), weight(w), estHits(e), matchData(tfmd) {} Term() : Term(nullptr, 0, 0, nullptr){} Term(SearchIterator *s, int32_t w, uint32_t e) : Term(s, w, e, nullptr) {} + Term(SearchIterator::UP s, int32_t w, uint32_t e) : Term(s.release(), w, e, nullptr) {} }; //----------------------------------------------------------------------------- diff --git a/searchlib/src/vespa/searchlib/queryeval/wand/weak_and_search.cpp b/searchlib/src/vespa/searchlib/queryeval/wand/weak_and_search.cpp index f290d3e19ea..d94dc6d8ae8 100644 --- a/searchlib/src/vespa/searchlib/queryeval/wand/weak_and_search.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/wand/weak_and_search.cpp @@ -106,27 +106,27 @@ WeakAndSearch::visitMembers(vespalib::ObjectVisitor &visitor) const //----------------------------------------------------------------------------- -SearchIterator * +SearchIterator::UP WeakAndSearch::createArrayWand(const Terms &terms, uint32_t n, bool strict) { if (strict) { - return new wand::WeakAndSearchLR<vespalib::LeftArrayHeap, vespalib::RightArrayHeap, true>(terms, n); + return SearchIterator::UP(new wand::WeakAndSearchLR<vespalib::LeftArrayHeap, vespalib::RightArrayHeap, true>(terms, n)); } else { - return new wand::WeakAndSearchLR<vespalib::LeftArrayHeap, vespalib::RightArrayHeap, false>(terms, n); + return SearchIterator::UP(new wand::WeakAndSearchLR<vespalib::LeftArrayHeap, vespalib::RightArrayHeap, false>(terms, n)); } } -SearchIterator * +SearchIterator::UP WeakAndSearch::createHeapWand(const Terms &terms, uint32_t n, bool strict) { if (strict) { - return new wand::WeakAndSearchLR<vespalib::LeftHeap, vespalib::RightHeap, true>(terms, n); + return SearchIterator::UP(new wand::WeakAndSearchLR<vespalib::LeftHeap, vespalib::RightHeap, true>(terms, n)); } else { - return new wand::WeakAndSearchLR<vespalib::LeftHeap, vespalib::RightHeap, false>(terms, n); + return SearchIterator::UP(new wand::WeakAndSearchLR<vespalib::LeftHeap, vespalib::RightHeap, false>(terms, n)); } } -SearchIterator * +SearchIterator::UP WeakAndSearch::create(const Terms &terms, uint32_t n, bool strict) { if (terms.size() < 128) { diff --git a/searchlib/src/vespa/searchlib/queryeval/wand/weak_and_search.h b/searchlib/src/vespa/searchlib/queryeval/wand/weak_and_search.h index 5b09d087873..e51b5ca102d 100644 --- a/searchlib/src/vespa/searchlib/queryeval/wand/weak_and_search.h +++ b/searchlib/src/vespa/searchlib/queryeval/wand/weak_and_search.h @@ -16,9 +16,9 @@ struct WeakAndSearch : SearchIterator { virtual const Terms &getTerms() const = 0; virtual uint32_t getN() const = 0; void visitMembers(vespalib::ObjectVisitor &visitor) const override; - static SearchIterator *createArrayWand(const Terms &terms, uint32_t n, bool strict); - static SearchIterator *createHeapWand(const Terms &terms, uint32_t n, bool strict); - static SearchIterator *create(const Terms &terms, uint32_t n, bool strict); + static SearchIterator::UP createArrayWand(const Terms &terms, uint32_t n, bool strict); + static SearchIterator::UP createHeapWand(const Terms &terms, uint32_t n, bool strict); + static SearchIterator::UP create(const Terms &terms, uint32_t n, bool strict); }; } // namespace queryeval diff --git a/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_blueprint.cpp index 36378439c01..cea35d976f0 100644 --- a/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_blueprint.cpp @@ -50,6 +50,7 @@ WeightedSetTermBlueprint::createLeafSearch(const fef::TermFieldMatchDataArray &t fef::MatchData::UP md = _layout.createMatchData(); std::vector<SearchIterator*> children(_terms.size()); for (size_t i = 0; i < _terms.size(); ++i) { + // TODO: pass ownership with unique_ptr children[i] = _terms[i]->createSearch(*md, true).release(); } return SearchIterator::UP(WeightedSetTermSearch::create(children, *tfmda[0], _weights, std::move(md))); diff --git a/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_search.cpp b/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_search.cpp index 2801f1c5e0c..71270c84c63 100644 --- a/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_search.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_search.cpp @@ -131,8 +131,8 @@ public: //----------------------------------------------------------------------------- -SearchIterator * -WeightedSetTermSearch::create(const std::vector<SearchIterator*> &children, +SearchIterator::UP +WeightedSetTermSearch::create(const std::vector<SearchIterator *> &children, TermFieldMatchData &tmd, const std::vector<int32_t> &weights, fef::MatchData::UP match_data) @@ -141,9 +141,9 @@ WeightedSetTermSearch::create(const std::vector<SearchIterator*> &children, typedef WeightedSetTermSearchImpl<vespalib::LeftHeap, SearchIteratorPack> HeapImpl; if (children.size() < 128) { - return new ArrayHeapImpl(tmd, weights, SearchIteratorPack(children, std::move(match_data))); + return SearchIterator::UP(new ArrayHeapImpl(tmd, weights, SearchIteratorPack(children, std::move(match_data)))); } - return new HeapImpl(tmd, weights, SearchIteratorPack(children, std::move(match_data))); + return SearchIterator::UP(new HeapImpl(tmd, weights, SearchIteratorPack(children, std::move(match_data)))); } //----------------------------------------------------------------------------- diff --git a/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_search.h b/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_search.h index 397ac0caf2e..ecc620a3adb 100644 --- a/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_search.h +++ b/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_search.h @@ -27,10 +27,11 @@ protected: WeightedSetTermSearch() {} public: - static SearchIterator* create(const std::vector<SearchIterator*> &children, - search::fef::TermFieldMatchData &tmd, - const std::vector<int32_t> &weights, - fef::MatchData::UP match_data); + // TODO: pass ownership with unique_ptr + static SearchIterator::UP create(const std::vector<SearchIterator *> &children, + search::fef::TermFieldMatchData &tmd, + const std::vector<int32_t> &weights, + fef::MatchData::UP match_data); static SearchIterator::UP create(search::fef::TermFieldMatchData &tmd, const std::vector<int32_t> &weights, diff --git a/searchlib/src/vespa/searchlib/test/initrange.h b/searchlib/src/vespa/searchlib/test/initrange.h index 9431740ac08..a143dfdb119 100644 --- a/searchlib/src/vespa/searchlib/test/initrange.h +++ b/searchlib/src/vespa/searchlib/test/initrange.h @@ -25,6 +25,7 @@ public: void verify(SearchIterator & iterator) const; /// Convenience that takes ownership of the pointer. void verify(SearchIterator * iterator) const; + void verify(SearchIterator::UP iterator) const { verify(*iterator); } private: void verify(SearchIterator & iterator, bool strict) const; void verify(SearchIterator & iterator, const Ranges & ranges, bool strict) const; diff --git a/searchlib/src/vespa/searchlib/test/searchiteratorverifier.cpp b/searchlib/src/vespa/searchlib/test/searchiteratorverifier.cpp index fce768ab0d2..ec53d6d9d00 100644 --- a/searchlib/src/vespa/searchlib/test/searchiteratorverifier.cpp +++ b/searchlib/src/vespa/searchlib/test/searchiteratorverifier.cpp @@ -170,9 +170,9 @@ void SearchIteratorVerifier::verifyAnd(bool strict) const { fef::TermFieldMatchData tfmd; MultiSearch::Children children; - children.emplace_back(create(strict).release()); - children.emplace_back(BitVectorIterator::create(_everyOddBitSet.get(), getDocIdLimit(), tfmd, false).release()); - SearchIterator::UP search(AndSearch::create(children, strict, UnpackInfo())); + children.push_back(create(strict)); + children.push_back(BitVectorIterator::create(_everyOddBitSet.get(), getDocIdLimit(), tfmd, false)); + auto search = AndSearch::create(std::move(children), strict, UnpackInfo()); TEST_DO(verify(*search, strict, _expectedAnd)); TEST_DO(verifyTermwise(std::move(search), strict, _expectedAnd)); } @@ -183,18 +183,18 @@ SearchIteratorVerifier::verifyAndNot(bool strict) const { { for (bool notStrictness : {false, true}) { MultiSearch::Children children; - children.emplace_back(create(strict).release()); - children.emplace_back(BitVectorIterator::create(_everyOddBitSet.get(), getDocIdLimit(), tfmd, notStrictness).release()); - SearchIterator::UP search(AndNotSearch::create(children, strict)); + children.push_back(create(strict)); + children.push_back(BitVectorIterator::create(_everyOddBitSet.get(), getDocIdLimit(), tfmd, notStrictness)); + auto search = AndNotSearch::create(std::move(children), strict); TEST_DO(verify(*search, strict, _expectedAndNotPositive)); TEST_DO(verifyTermwise(std::move(search), strict, _expectedAndNotPositive)); } } { MultiSearch::Children children; - children.emplace_back(BitVectorIterator::create(_everyOddBitSet.get(), getDocIdLimit(), tfmd, true).release()); - children.emplace_back(create(strict).release()); - SearchIterator::UP search(AndNotSearch::create(children, strict)); + children.push_back(BitVectorIterator::create(_everyOddBitSet.get(), getDocIdLimit(), tfmd, true)); + children.push_back(create(strict)); + auto search = AndNotSearch::create(std::move(children), strict); TEST_DO(verify(*search, strict, _expectedAndNotNegative)); TEST_DO(verifyTermwise(std::move(search), strict, _expectedAndNotNegative)); } @@ -205,9 +205,9 @@ void SearchIteratorVerifier::verifyOr(bool strict) const { fef::TermFieldMatchData tfmd; MultiSearch::Children children; - children.emplace_back(create(strict).release()); - children.emplace_back(BitVectorIterator::create(_everyOddBitSet.get(), getDocIdLimit(), tfmd, strict).release()); - SearchIterator::UP search(OrSearch::create(children, strict, UnpackInfo())); + children.push_back(create(strict)); + children.push_back(BitVectorIterator::create(_everyOddBitSet.get(), getDocIdLimit(), tfmd, strict)); + SearchIterator::UP search(OrSearch::create(std::move(children), strict, UnpackInfo())); TEST_DO(verify(*search, strict, _expectedOr)); TEST_DO(verifyTermwise(std::move(search), strict, _expectedOr)); } diff --git a/searchsummary/CMakeLists.txt b/searchsummary/CMakeLists.txt index 2a23dd4c495..3792f2b6218 100644 --- a/searchsummary/CMakeLists.txt +++ b/searchsummary/CMakeLists.txt @@ -20,11 +20,13 @@ vespa_define_module( src/vespa/searchsummary src/vespa/searchsummary/config src/vespa/searchsummary/docsummary + src/vespa/searchsummary/test TESTS src/tests/docsumformat src/tests/docsummary src/tests/docsummary/attribute_combiner + src/tests/docsummary/attributedfw src/tests/docsummary/matched_elements_filter src/tests/docsummary/slime_summary src/tests/extractkeywords diff --git a/searchsummary/src/tests/docsummary/attribute_combiner/CMakeLists.txt b/searchsummary/src/tests/docsummary/attribute_combiner/CMakeLists.txt index cffdef25e5b..3ac95211aec 100644 --- a/searchsummary/src/tests/docsummary/attribute_combiner/CMakeLists.txt +++ b/searchsummary/src/tests/docsummary/attribute_combiner/CMakeLists.txt @@ -5,6 +5,7 @@ vespa_add_executable(searchsummary_attribute_combiner_test_app TEST attribute_combiner_test.cpp DEPENDS searchsummary + searchsummary_test GTest::GTest ) vespa_add_test(NAME searchsummary_attribute_combiner_test_app COMMAND searchsummary_attribute_combiner_test_app) diff --git a/searchsummary/src/tests/docsummary/attribute_combiner/attribute_combiner_test.cpp b/searchsummary/src/tests/docsummary/attribute_combiner/attribute_combiner_test.cpp index ae8e5f88c28..eaeaa27f053 100644 --- a/searchsummary/src/tests/docsummary/attribute_combiner/attribute_combiner_test.cpp +++ b/searchsummary/src/tests/docsummary/attribute_combiner/attribute_combiner_test.cpp @@ -1,200 +1,41 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/searchcommon/common/undefinedvalues.h> -#include <vespa/searchlib/attribute/attributefactory.h> -#include <vespa/searchlib/attribute/attributemanager.h> #include <vespa/searchlib/attribute/attributevector.h> -#include <vespa/searchlib/attribute/attributevector.hpp> -#include <vespa/searchlib/attribute/floatbase.h> -#include <vespa/searchlib/attribute/integerbase.h> -#include <vespa/searchlib/attribute/stringbase.h> #include <vespa/searchlib/common/matching_elements.h> #include <vespa/searchlib/common/matching_elements_fields.h> #include <vespa/searchlib/util/slime_output_raw_buf_adapter.h> +#include <vespa/searchsummary/docsummary/docsumfieldwriter.h> #include <vespa/searchsummary/docsummary/docsumstate.h> #include <vespa/searchsummary/docsummary/docsum_field_writer_state.h> #include <vespa/searchsummary/docsummary/attribute_combiner_dfw.h> +#include <vespa/searchsummary/test/mock_attribute_manager.h> +#include <vespa/searchsummary/test/mock_state_callback.h> +#include <vespa/searchsummary/test/slime_value.h> #include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/gtest/gtest.h> #include <vespa/log/log.h> LOG_SETUP("attribute_combiner_test"); -using search::AttributeFactory; -using search::AttributeManager; -using search::AttributeVector; -using search::IntegerAttribute; -using search::FloatingPointAttribute; -using search::MatchingElements; -using search::StringAttribute; using search::attribute::BasicType; -using search::attribute::CollectionType; -using search::attribute::Config; -using search::attribute::IAttributeVector; using search::attribute::getUndefined; using search::docsummary::AttributeCombinerDFW; using search::docsummary::GetDocsumsState; using search::docsummary::GetDocsumsStateCallback; using search::docsummary::IDocsumEnvironment; using search::docsummary::IDocsumFieldWriter; +using search::docsummary::test::MockAttributeManager; +using search::docsummary::test::MockStateCallback; +using search::docsummary::test::SlimeValue; namespace { -vespalib::string -toCompactJsonString(const vespalib::Slime &slime) -{ - vespalib::SimpleBuffer buf; - vespalib::slime::JsonFormat::encode(slime, buf, true); - return buf.get().make_string(); -} - -struct FieldBlock { - vespalib::string input; - vespalib::Slime slime; - search::RawBuf binary; - vespalib::string json; - - explicit FieldBlock(const vespalib::string &jsonInput); - ~FieldBlock(); - const char *data() const { return binary.GetDrainPos(); } - size_t dataLen() const { return binary.GetUsedLen(); } -}; - -FieldBlock::FieldBlock(const vespalib::string &jsonInput) - : input(jsonInput), slime(), binary(1024), json() -{ - size_t used = vespalib::slime::JsonFormat::decode(jsonInput, slime); - EXPECT_TRUE(used > 0); - json = toCompactJsonString(slime); - search::SlimeOutputRawBufAdapter adapter(binary); - vespalib::slime::BinaryFormat::encode(slime, adapter); -} - -FieldBlock::~FieldBlock() = default; - -struct AttributeManagerFixture -{ - AttributeManager mgr; - - AttributeManagerFixture(); - - ~AttributeManagerFixture(); - - template <typename AttributeType, typename ValueType> - void - buildAttribute(const vespalib::string &name, - BasicType type, - std::vector<std::vector<ValueType>> values); - - void - buildStringAttribute(const vespalib::string &name, - std::vector<std::vector<vespalib::string>> values); - void - buildFloatAttribute(const vespalib::string &name, - std::vector<std::vector<double>> values); - - void - buildIntegerAttribute(const vespalib::string &name, - BasicType type, - std::vector<std::vector<IAttributeVector::largeint_t>> values); -}; - -AttributeManagerFixture::AttributeManagerFixture() - : mgr() -{ - buildStringAttribute("array.name", {{"n1.1", "n1.2"}, {"n2"}, {"n3.1", "n3.2"}, {"", "n4.2"}, {}}); - buildIntegerAttribute("array.val", BasicType::Type::INT8, {{ 10, 11}, {20, 21 }, {30}, { getUndefined<int8_t>(), 41}, {}}); - buildFloatAttribute("array.fval", {{ 110.0}, { 120.0, 121.0 }, { 130.0, 131.0}, { getUndefined<double>(), 141.0 }, {}}); - buildStringAttribute("smap.key", {{"k1.1", "k1.2"}, {"k2"}, {"k3.1", "k3.2"}, {"", "k4.2"}, {}}); - buildStringAttribute("smap.value.name", {{"n1.1", "n1.2"}, {"n2"}, {"n3.1", "n3.2"}, {"", "n4.2"}, {}}); - buildIntegerAttribute("smap.value.val", BasicType::Type::INT8, {{ 10, 11}, {20, 21 }, {30}, { getUndefined<int8_t>(), 41}, {}}); - buildFloatAttribute("smap.value.fval", {{ 110.0}, { 120.0, 121.0 }, { 130.0, 131.0}, { getUndefined<double>(), 141.0 }, {}}); - buildStringAttribute("map.key", {{"k1.1", "k1.2"}, {"k2"}, {"k3.1"}, {"", "k4.2"}, {}}); - buildStringAttribute("map.value", {{"n1.1", "n1.2"}, {}, {"n3.1", "n3.2"}, {"", "n4.2"}, {}}); -} - -AttributeManagerFixture::~AttributeManagerFixture() = default; - -template <typename AttributeType, typename ValueType> -void -AttributeManagerFixture::buildAttribute(const vespalib::string &name, - BasicType type, - std::vector<std::vector<ValueType>> values) -{ - Config cfg(type, CollectionType::Type::ARRAY); - auto attrBase = AttributeFactory::createAttribute(name, cfg); - EXPECT_TRUE(attrBase); - auto attr = std::dynamic_pointer_cast<AttributeType>(attrBase); - EXPECT_TRUE(attr); - attr->addReservedDoc(); - for (const auto &docValues : values) { - uint32_t docId = 0; - EXPECT_TRUE(attr->addDoc(docId)); - EXPECT_NE(0u, docId); - for (const auto &value : docValues) { - attr->append(docId, value, 1); - } - attr->commit(); - } - EXPECT_TRUE(mgr.add(attr)); -} - -void -AttributeManagerFixture::buildStringAttribute(const vespalib::string &name, - std::vector<std::vector<vespalib::string>> values) -{ - buildAttribute<StringAttribute, vespalib::string>(name, BasicType::Type::STRING, std::move(values)); -} - -void -AttributeManagerFixture::buildFloatAttribute(const vespalib::string &name, - std::vector<std::vector<double>> values) -{ - buildAttribute<FloatingPointAttribute, double>(name, BasicType::Type::DOUBLE, std::move(values)); -} - -void -AttributeManagerFixture::buildIntegerAttribute(const vespalib::string &name, - BasicType type, - std::vector<std::vector<IAttributeVector::largeint_t>> values) -{ - buildAttribute<IntegerAttribute, IAttributeVector::largeint_t>(name, type, std::move(values)); -} - - -class DummyStateCallback : public GetDocsumsStateCallback -{ -public: - MatchingElements _matching_elements; - - DummyStateCallback(); - void FillSummaryFeatures(GetDocsumsState *, IDocsumEnvironment *) override { } - void FillRankFeatures(GetDocsumsState *, IDocsumEnvironment *) override { } - void ParseLocation(GetDocsumsState *) override { } - std::unique_ptr<MatchingElements> fill_matching_elements(const search::MatchingElementsFields &) override { return std::make_unique<MatchingElements>(_matching_elements); } - ~DummyStateCallback() override { } -}; - -DummyStateCallback::DummyStateCallback() - : GetDocsumsStateCallback(), - _matching_elements() -{ - _matching_elements.add_matching_elements(1, "array", {1}); - _matching_elements.add_matching_elements(3, "array", {0}); - _matching_elements.add_matching_elements(4, "array", {1}); - _matching_elements.add_matching_elements(1, "smap", {1}); - _matching_elements.add_matching_elements(3, "smap", {0}); - _matching_elements.add_matching_elements(4, "smap", {1}); - _matching_elements.add_matching_elements(1, "map", {1}); - _matching_elements.add_matching_elements(3, "map", {0}); - _matching_elements.add_matching_elements(4, "map", {1}); -} - struct AttributeCombinerTest : public ::testing::Test { - AttributeManagerFixture attrs; + MockAttributeManager attrs; std::unique_ptr<IDocsumFieldWriter> writer; - DummyStateCallback stateCallback; + MockStateCallback callback; GetDocsumsState state; std::shared_ptr<search::MatchingElementsFields> _matching_elems_fields; @@ -207,11 +48,31 @@ struct AttributeCombinerTest : public ::testing::Test AttributeCombinerTest::AttributeCombinerTest() : attrs(), writer(), - stateCallback(), - state(stateCallback), + callback(), + state(callback), _matching_elems_fields() { - state._attrCtx = attrs.mgr.createContext(); + attrs.build_string_attribute("array.name", {{"n1.1", "n1.2"}, {"n2"}, {"n3.1", "n3.2"}, {"", "n4.2"}, {}}); + attrs.build_int_attribute("array.val", BasicType::Type::INT8, {{ 10, 11}, {20, 21 }, {30}, { getUndefined<int8_t>(), 41}, {}}); + attrs.build_float_attribute("array.fval", {{ 110.0}, { 120.0, 121.0 }, { 130.0, 131.0}, { getUndefined<double>(), 141.0 }, {}}); + attrs.build_string_attribute("smap.key", {{"k1.1", "k1.2"}, {"k2"}, {"k3.1", "k3.2"}, {"", "k4.2"}, {}}); + attrs.build_string_attribute("smap.value.name", {{"n1.1", "n1.2"}, {"n2"}, {"n3.1", "n3.2"}, {"", "n4.2"}, {}}); + attrs.build_int_attribute("smap.value.val", BasicType::Type::INT8, {{ 10, 11}, {20, 21 }, {30}, { getUndefined<int8_t>(), 41}, {}}); + attrs.build_float_attribute("smap.value.fval", {{ 110.0}, { 120.0, 121.0 }, { 130.0, 131.0}, { getUndefined<double>(), 141.0 }, {}}); + attrs.build_string_attribute("map.key", {{"k1.1", "k1.2"}, {"k2"}, {"k3.1"}, {"", "k4.2"}, {}}); + attrs.build_string_attribute("map.value", {{"n1.1", "n1.2"}, {}, {"n3.1", "n3.2"}, {"", "n4.2"}, {}}); + + callback.add_matching_elements(1, "array", {1}); + callback.add_matching_elements(3, "array", {0}); + callback.add_matching_elements(4, "array", {1}); + callback.add_matching_elements(1, "smap", {1}); + callback.add_matching_elements(3, "smap", {0}); + callback.add_matching_elements(4, "smap", {1}); + callback.add_matching_elements(1, "map", {1}); + callback.add_matching_elements(3, "map", {0}); + callback.add_matching_elements(4, "map", {1}); + + state._attrCtx = attrs.mgr().createContext(); } AttributeCombinerTest::~AttributeCombinerTest() = default; @@ -228,83 +89,73 @@ AttributeCombinerTest::set_field(const vespalib::string &field_name, bool filter } void -AttributeCombinerTest::assertWritten(const vespalib::string &expectedJson, uint32_t docId) +AttributeCombinerTest::assertWritten(const vespalib::string &exp_slime_as_json, uint32_t docId) { - vespalib::Slime target; - vespalib::slime::SlimeInserter inserter(target); + vespalib::Slime act; + vespalib::slime::SlimeInserter inserter(act); writer->insertField(docId, nullptr, &state, search::docsummary::RES_JSONSTRING, inserter); - search::RawBuf binary(1024); - vespalib::string json = toCompactJsonString(target); - search::SlimeOutputRawBufAdapter adapter(binary); - vespalib::slime::BinaryFormat::encode(target, adapter); - FieldBlock block(expectedJson); - EXPECT_EQ(block.dataLen(), binary.GetUsedLen()); - EXPECT_EQ(0, memcmp(block.data(), binary.GetDrainPos(), block.dataLen())); - if (block.dataLen() != binary.GetUsedLen() || - memcmp(block.data(), binary.GetDrainPos(), block.dataLen()) != 0) { - LOG(error, "Expected '%s'", expectedJson.c_str()); - LOG(error, "Expected normalized '%s'", block.json.c_str()); - LOG(error, "Got '%s'", json.c_str()); - } + + SlimeValue exp(exp_slime_as_json); + EXPECT_EQ(exp.slime, act); } TEST_F(AttributeCombinerTest, require_that_attribute_combiner_dfw_generates_correct_slime_output_for_array_of_struct) { set_field("array", false); - assertWritten("[ { fval: 110.0, name: \"n1.1\", val: 10}, { name: \"n1.2\", val: 11}]", 1); - assertWritten("[ { fval: 120.0, name: \"n2\", val: 20}, { fval: 121.0, val: 21 }]", 2); - assertWritten("[ { fval: 130.0, name: \"n3.1\", val: 30}, { fval: 131.0, name: \"n3.2\"} ]", 3); - assertWritten("[ { }, { fval: 141.0, name: \"n4.2\", val: 41} ]", 4); + assertWritten("[ { fval: 110.0, name: 'n1.1', val: 10}, { name: 'n1.2', val: 11}]", 1); + assertWritten("[ { fval: 120.0, name: 'n2', val: 20}, { fval: 121.0, val: 21 }]", 2); + assertWritten("[ { fval: 130.0, name: 'n3.1', val: 30}, { fval: 131.0, name: 'n3.2'} ]", 3); + assertWritten("[ { }, { fval: 141.0, name: 'n4.2', val: 41} ]", 4); assertWritten("null", 5); } TEST_F(AttributeCombinerTest, require_that_attribute_combiner_dfw_generates_correct_slime_output_for_map_of_struct) { set_field("smap", false); - assertWritten("[ { key: \"k1.1\", value: { fval: 110.0, name: \"n1.1\", val: 10} }, { key: \"k1.2\", value: { name: \"n1.2\", val: 11} }]", 1); - assertWritten("[ { key: \"k2\", value: { fval: 120.0, name: \"n2\", val: 20} }, { key: \"\", value: { fval: 121.0, val: 21 } }]", 2); - assertWritten("[ { key: \"k3.1\", value: { fval: 130.0, name: \"n3.1\", val: 30} }, { key: \"k3.2\", value: { fval: 131.0, name: \"n3.2\"} } ]", 3); - assertWritten("[ { key: \"\", value: { } }, { key: \"k4.2\", value: { fval: 141.0, name: \"n4.2\", val: 41} } ]", 4); + assertWritten("[ { key: 'k1.1', value: { fval: 110.0, name: 'n1.1', val: 10} }, { key: 'k1.2', value: { name: 'n1.2', val: 11} }]", 1); + assertWritten("[ { key: 'k2', value: { fval: 120.0, name: 'n2', val: 20} }, { key: '', value: { fval: 121.0, val: 21 } }]", 2); + assertWritten("[ { key: 'k3.1', value: { fval: 130.0, name: 'n3.1', val: 30} }, { key: 'k3.2', value: { fval: 131.0, name: 'n3.2'} } ]", 3); + assertWritten("[ { key: '', value: { } }, { key: 'k4.2', value: { fval: 141.0, name: 'n4.2', val: 41} } ]", 4); assertWritten("null", 5); } TEST_F(AttributeCombinerTest, require_that_attribute_combiner_dfw_generates_correct_slime_output_for_map_of_string) { set_field("map", false); - assertWritten("[ { key: \"k1.1\", value: \"n1.1\" }, { key: \"k1.2\", value: \"n1.2\"}]", 1); - assertWritten("[ { key: \"k2\", value: \"\" }]", 2); - assertWritten("[ { key: \"k3.1\", value: \"n3.1\" }, { key: \"\", value: \"n3.2\"} ]", 3); - assertWritten("[ { key: \"\", value: \"\" }, { key: \"k4.2\", value: \"n4.2\" } ]", 4); + assertWritten("[ { key: 'k1.1', value: 'n1.1' }, { key: 'k1.2', value: 'n1.2'}]", 1); + assertWritten("[ { key: 'k2', value: '' }]", 2); + assertWritten("[ { key: 'k3.1', value: 'n3.1' }, { key: '', value: 'n3.2'} ]", 3); + assertWritten("[ { key: '', value: '' }, { key: 'k4.2', value: 'n4.2' } ]", 4); assertWritten("null", 5); } TEST_F(AttributeCombinerTest, require_that_attribute_combiner_dfw_generates_correct_slime_output_for_filtered_array_of_struct) { set_field("array", true); - assertWritten("[ { name: \"n1.2\", val: 11}]", 1); + assertWritten("[ { name: 'n1.2', val: 11}]", 1); assertWritten("[ ]", 2); - assertWritten("[ { fval: 130.0, name: \"n3.1\", val: 30} ]", 3); - assertWritten("[ { fval: 141.0, name: \"n4.2\", val: 41} ]", 4); + assertWritten("[ { fval: 130.0, name: 'n3.1', val: 30} ]", 3); + assertWritten("[ { fval: 141.0, name: 'n4.2', val: 41} ]", 4); assertWritten("null", 5); } TEST_F(AttributeCombinerTest, require_that_attribute_combiner_dfw_generates_correct_slime_output_for_filtered_map_of_struct) { set_field("smap", true); - assertWritten("[ { key: \"k1.2\", value: { name: \"n1.2\", val: 11} }]", 1); + assertWritten("[ { key: 'k1.2', value: { name: 'n1.2', val: 11} }]", 1); assertWritten("[ ]", 2); - assertWritten("[ { key: \"k3.1\", value: { fval: 130.0, name: \"n3.1\", val: 30} } ]", 3); - assertWritten("[ { key: \"k4.2\", value: { fval: 141.0, name: \"n4.2\", val: 41} } ]", 4); + assertWritten("[ { key: 'k3.1', value: { fval: 130.0, name: 'n3.1', val: 30} } ]", 3); + assertWritten("[ { key: 'k4.2', value: { fval: 141.0, name: 'n4.2', val: 41} } ]", 4); assertWritten("null", 5); } TEST_F(AttributeCombinerTest, require_that_attribute_combiner_dfw_generates_correct_slime_output_for_filtered_map_of_string) { set_field("map", true); - assertWritten("[ { key: \"k1.2\", value: \"n1.2\"}]", 1); + assertWritten("[ { key: 'k1.2', value: 'n1.2'}]", 1); assertWritten("[ ]", 2); - assertWritten("[ { key: \"k3.1\", value: \"n3.1\" } ]", 3); - assertWritten("[ { key: \"k4.2\", value: \"n4.2\" } ]", 4); + assertWritten("[ { key: 'k3.1', value: 'n3.1' } ]", 3); + assertWritten("[ { key: 'k4.2', value: 'n4.2' } ]", 4); assertWritten("null", 5); } diff --git a/searchsummary/src/tests/docsummary/attributedfw/CMakeLists.txt b/searchsummary/src/tests/docsummary/attributedfw/CMakeLists.txt new file mode 100644 index 00000000000..8bdf30273f2 --- /dev/null +++ b/searchsummary/src/tests/docsummary/attributedfw/CMakeLists.txt @@ -0,0 +1,11 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) +vespa_add_executable(searchsummary_attributedfw_test_app TEST + SOURCES + attributedfw_test.cpp + DEPENDS + searchsummary + searchsummary_test + GTest::GTest +) +vespa_add_test(NAME searchsummary_attributedfw_test_app COMMAND searchsummary_attributedfw_test_app) diff --git a/searchsummary/src/tests/docsummary/attributedfw/attributedfw_test.cpp b/searchsummary/src/tests/docsummary/attributedfw/attributedfw_test.cpp new file mode 100644 index 00000000000..7bea92ec8f3 --- /dev/null +++ b/searchsummary/src/tests/docsummary/attributedfw/attributedfw_test.cpp @@ -0,0 +1,150 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/searchlib/common/matching_elements_fields.h> +#include <vespa/searchsummary/docsummary/attributedfw.h> +#include <vespa/searchsummary/test/mock_attribute_manager.h> +#include <vespa/searchsummary/test/mock_state_callback.h> +#include <vespa/searchsummary/test/slime_value.h> +#include <vespa/vespalib/gtest/gtest.h> + +#include <vespa/log/log.h> +LOG_SETUP("attributedfw_test"); + +using search::MatchingElements; +using search::MatchingElementsFields; +using search::attribute::BasicType; +using search::attribute::CollectionType; +using search::docsummary::AttributeDFWFactory; +using search::docsummary::GetDocsumsState; +using search::docsummary::IDocsumFieldWriter; +using search::docsummary::test::MockAttributeManager; +using search::docsummary::test::MockStateCallback; +using search::docsummary::test::SlimeValue; + +using ElementVector = std::vector<uint32_t>; + +class AttributeDFWTest : public ::testing::Test { +protected: + MockAttributeManager _attrs; + std::unique_ptr<IDocsumFieldWriter> _writer; + MockStateCallback _callback; + GetDocsumsState _state; + std::shared_ptr<search::MatchingElementsFields> _matching_elems_fields; + vespalib::string _field_name; + +public: + AttributeDFWTest() + : _attrs(), + _writer(), + _callback(), + _state(_callback), + _matching_elems_fields(), + _field_name() + { + _attrs.build_string_attribute("array_str", { {"a", "b", "c"}, {} }); + _attrs.build_int_attribute("array_int", BasicType::INT32, { {10, 20, 30}, {} }); + _attrs.build_float_attribute("array_float", { {10.5, 20.5, 30.5}, {} }); + + _attrs.build_string_attribute("wset_str", { {"a", "b", "c"}, {} }, CollectionType::WSET); + _attrs.build_int_attribute("wset_int", BasicType::INT32, { {10, 20, 30}, {} }, CollectionType::WSET); + _attrs.build_float_attribute("wset_float", { {10.5, 20.5, 30.5}, {} }, CollectionType::WSET); + + _state._attrCtx = _attrs.mgr().createContext(); + } + ~AttributeDFWTest() {} + + void setup(const vespalib::string& field_name, bool filter_elements) { + if (filter_elements) { + _matching_elems_fields = std::make_shared<MatchingElementsFields>(); + } + _writer = AttributeDFWFactory::create(_attrs.mgr(), field_name, filter_elements, _matching_elems_fields); + _writer->setIndex(0); + _field_name = field_name; + _state._attributes.resize(1); + _state._attributes[0] = _state._attrCtx->getAttribute(field_name); + } + + void expect_field(const vespalib::string& exp_slime_as_json, uint32_t docid) { + vespalib::Slime act; + vespalib::slime::SlimeInserter inserter(act); + _writer->insertField(docid, nullptr, &_state, search::docsummary::RES_JSONSTRING, inserter); + + SlimeValue exp(exp_slime_as_json); + EXPECT_EQ(exp.slime, act); + } + + void expect_filtered(const ElementVector& matching_elems, const std::string& exp_slime_as_json, uint32_t docid = 1) { + _callback.clear(); + _callback.add_matching_elements(docid, _field_name, matching_elems); + _state._matching_elements = std::unique_ptr<MatchingElements>(); + expect_field(exp_slime_as_json, docid); + } +}; + +TEST_F(AttributeDFWTest, outputs_slime_for_array_of_string) +{ + setup("array_str", false); + expect_field("[ 'a', 'b', 'c' ]", 1); + expect_field("null", 2); +} + +TEST_F(AttributeDFWTest, outputs_slime_for_array_of_int) +{ + setup("array_int", false); + expect_field("[ 10, 20, 30 ]", 1); + expect_field("null", 2); +} + +TEST_F(AttributeDFWTest, outputs_slime_for_array_of_float) +{ + setup("array_float", false); + expect_field("[ 10.5, 20.5, 30.5 ]", 1); + expect_field("null", 2); +} + +TEST_F(AttributeDFWTest, outputs_slime_for_wset_of_string) +{ + setup("wset_str", false); + expect_field("[ {'item':'a', 'weight':1}, {'item':'b', 'weight':1}, {'item':'c', 'weight':1} ]", 1); + expect_field("null", 2); +} + +TEST_F(AttributeDFWTest, outputs_slime_for_wset_of_int) +{ + setup("wset_int", false); + expect_field("[ {'item':10, 'weight':1}, {'item':20, 'weight':1}, {'item':30, 'weight':1} ]", 1); + expect_field("null", 2); +} + +TEST_F(AttributeDFWTest, outputs_slime_for_wset_of_float) +{ + setup("wset_float", false); + expect_field("[ {'item':10.5, 'weight':1}, {'item':20.5, 'weight':1}, {'item':30.5, 'weight':1} ]", 1); + expect_field("null", 2); +} + +TEST_F(AttributeDFWTest, matched_elements_fields_is_populated) +{ + setup("array_str", true); + EXPECT_TRUE(_matching_elems_fields->has_field("array_str")); +} + +TEST_F(AttributeDFWTest, filteres_matched_elements_in_array_attribute) +{ + setup("array_str", true); + expect_filtered({}, "[]"); + expect_filtered({0}, "[ 'a' ]"); + expect_filtered({1, 2}, "[ 'b', 'c' ]"); + expect_filtered({3}, "[]"); +} + +TEST_F(AttributeDFWTest, filteres_matched_elements_in_wset_attribute) +{ + setup("wset_str", true); + expect_filtered({}, "[]"); + expect_filtered({0}, "[ {'item':'a', 'weight':1} ]"); + expect_filtered({1, 2}, "[ {'item':'b', 'weight':1}, {'item':'c', 'weight':1} ]"); + expect_filtered({3}, "[]"); +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp b/searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp index ed91d4a88eb..0ac2f09e1b0 100644 --- a/searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp +++ b/searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp @@ -15,7 +15,7 @@ #include <vespa/searchsummary/docsummary/resultconfig.h> #include <vespa/searchsummary/docsummary/resultpacker.h> #include <vespa/searchsummary/docsummary/summaryfieldconverter.h> -#include <vespa/vespalib/data/slime/json_format.h> +#include <vespa/searchsummary/test/slime_value.h> #include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/gtest/gtest.h> #include <iostream> @@ -32,6 +32,7 @@ using search::attribute::CollectionType; using search::attribute::Config; using search::attribute::IAttributeContext; using search::attribute::IAttributeVector; +using search::docsummary::test::SlimeValue; using vespalib::Slime; using namespace document; @@ -40,17 +41,6 @@ using namespace vespalib::slime; using ElementVector = std::vector<uint32_t>; -struct SlimeValue { - Slime slime; - - SlimeValue(const std::string& json_input) - : slime() - { - size_t used = JsonFormat::decode(json_input, slime); - EXPECT_GT(used, 0); - } -}; - StructDataType::UP make_struct_elem_type() { diff --git a/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.cpp index f1b12d8a227..05ba12ddff9 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.cpp @@ -1,25 +1,29 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "docsumwriter.h" #include "attributedfw.h" #include "docsumstate.h" -#include <vespa/searchlib/attribute/stringbase.h> -#include <vespa/searchlib/attribute/integerbase.h> +#include "docsumwriter.h" +#include <vespa/eval/tensor/serialization/typed_binary_format.h> +#include <vespa/eval/tensor/tensor.h> +#include <vespa/searchcommon/attribute/iattributecontext.h> #include <vespa/searchlib/attribute/iattributemanager.h> +#include <vespa/searchlib/attribute/integerbase.h> +#include <vespa/searchlib/attribute/stringbase.h> +#include <vespa/searchlib/common/matching_elements.h> +#include <vespa/searchlib/common/matching_elements_fields.h> #include <vespa/searchlib/tensor/i_tensor_attribute.h> -#include <vespa/searchcommon/attribute/iattributecontext.h> -#include <vespa/eval/tensor/tensor.h> -#include <vespa/eval/tensor/serialization/typed_binary_format.h> -#include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/data/slime/slime.h> +#include <vespa/vespalib/objects/nbostream.h> #include <vespa/log/log.h> LOG_SETUP(".searchlib.docsummary.attributedfw"); using namespace search; +using search::attribute::BasicType; using search::attribute::IAttributeContext; using search::attribute::IAttributeVector; -using search::attribute::BasicType; +using vespalib::Memory; +using vespalib::slime::Cursor; using vespalib::slime::Inserter; namespace search::docsummary { @@ -30,7 +34,7 @@ AttrDFW::AttrDFW(const vespalib::string & attrName) : } const attribute::IAttributeVector & -AttrDFW::vec(const GetDocsumsState & s) const { +AttrDFW::get_attribute(const GetDocsumsState& s) const { return *s.getAttribute(getIndex()); } @@ -48,13 +52,13 @@ public: bool SingleAttrDFW::isDefaultValue(uint32_t docid, const GetDocsumsState * state) const { - return vec(*state).isUndefined(docid); + return get_attribute(*state).isUndefined(docid); } void SingleAttrDFW::insertField(uint32_t docid, GetDocsumsState * state, ResType type, Inserter &target) { - const IAttributeVector & v = vec(*state); + const auto& v = get_attribute(*state); switch (type) { case RES_INT: { uint32_t val = v.getInt(docid); @@ -133,46 +137,115 @@ SingleAttrDFW::insertField(uint32_t docid, GetDocsumsState * state, ResType type //----------------------------------------------------------------------------- -class MultiAttrDFW : public AttrDFW -{ -public: - explicit MultiAttrDFW(const vespalib::string & attrName) : AttrDFW(attrName) {} - void insertField(uint32_t docid, GetDocsumsState *state, ResType type, Inserter &target) override; +template <typename DataType> +class MultiAttrDFW : public AttrDFW { +private: + bool _is_weighted_set; + bool _filter_elements; + std::shared_ptr<MatchingElementsFields> _matching_elems_fields; +public: + explicit MultiAttrDFW(const vespalib::string& attr_name, bool is_weighted_set, + bool filter_elements, std::shared_ptr<MatchingElementsFields> matching_elems_fields) + : AttrDFW(attr_name), + _is_weighted_set(is_weighted_set), + _filter_elements(filter_elements), + _matching_elems_fields(std::move(matching_elems_fields)) + { + if (filter_elements && _matching_elems_fields) { + _matching_elems_fields->add_field(attr_name); + } + } + void insertField(uint32_t docid, GetDocsumsState* state, ResType type, Inserter& target) override; }; void -MultiAttrDFW::insertField(uint32_t docid, GetDocsumsState *state, ResType, Inserter &target) +insert_element(const std::vector<IAttributeVector::WeightedString>& elements, + size_t idx, bool is_weighted_set, Cursor& arr) +{ + const vespalib::string& sv = elements[idx].getValue(); + Memory value(sv.c_str(), sv.size()); + if (is_weighted_set) { + Cursor& elem = arr.addObject(); + elem.setString("item", value); + elem.setLong("weight", elements[idx].getWeight()); + } else { + arr.addString(value); + } +} + +void +insert_element(const std::vector<IAttributeVector::WeightedInt>& elements, + size_t idx, bool is_weighted_set, Cursor& arr) +{ + if (is_weighted_set) { + Cursor& elem = arr.addObject(); + elem.setLong("item", elements[idx].getValue()); + elem.setLong("weight", elements[idx].getWeight()); + } else { + arr.addLong(elements[idx].getValue()); + } +} + +void +insert_element(const std::vector<IAttributeVector::WeightedFloat>& elements, + size_t idx, bool is_weighted_set, Cursor& arr) { - using vespalib::slime::Cursor; - using vespalib::Memory; - const IAttributeVector & v = vec(*state); - bool isWeightedSet = v.hasWeightedSetType(); + if (is_weighted_set) { + Cursor& elem = arr.addObject(); + elem.setDouble("item", elements[idx].getValue()); + elem.setLong("weight", elements[idx].getWeight()); + } else { + arr.addDouble(elements[idx].getValue()); + } +} - uint32_t entries = v.getValueCount(docid); +template <typename DataType> +void +MultiAttrDFW<DataType>::insertField(uint32_t docid, GetDocsumsState* state, ResType, Inserter& target) +{ + const auto& attr = get_attribute(*state); + uint32_t entries = attr.getValueCount(docid); if (entries == 0) { return; // Don't insert empty fields } Cursor &arr = target.insertArray(); - BasicType::Type t = v.getBasicType(); - switch (t) { - case BasicType::NONE: - case BasicType::STRING: { - std::vector<IAttributeVector::WeightedString> elements(entries); - entries = std::min(entries, v.get(docid, &elements[0], entries)); - for (uint32_t i = 0; i < entries; ++i) { - const vespalib::string &sv = elements[i].getValue(); - Memory value(sv.c_str(), sv.size()); - if (isWeightedSet) { - Cursor &elem = arr.addObject(); - elem.setString("item", value); - elem.setLong("weight", elements[i].getWeight()); - } else { - arr.addString(value); + std::vector<DataType> elements(entries); + entries = std::min(entries, attr.get(docid, elements.data(), entries)); + + if (_filter_elements) { + const auto& matching_elems = state->get_matching_elements(*_matching_elems_fields) + .get_matching_elements(docid, getAttributeName()); + if (!matching_elems.empty() && matching_elems.back() < entries) { + for (uint32_t id_to_keep : matching_elems) { + insert_element(elements, id_to_keep, _is_weighted_set, arr); } } - return; } + } else { + for (uint32_t i = 0; i < entries; ++i) { + insert_element(elements, i, _is_weighted_set, arr); + } + } +} + +//----------------------------------------------------------------------------- + +namespace { + +std::unique_ptr<IDocsumFieldWriter> +create_multi_writer(const IAttributeVector& attr, + bool filter_elements, + std::shared_ptr<MatchingElementsFields> matching_elems_fields) +{ + auto type = attr.getBasicType(); + bool is_weighted_set = attr.hasWeightedSetType(); + switch (type) { + case BasicType::NONE: + case BasicType::STRING: { + return std::make_unique<MultiAttrDFW<IAttributeVector::WeightedString>>(attr.getName(), is_weighted_set, + filter_elements, std::move(matching_elems_fields)); + } case BasicType::BOOL: case BasicType::UINT2: case BasicType::UINT4: @@ -180,54 +253,39 @@ MultiAttrDFW::insertField(uint32_t docid, GetDocsumsState *state, ResType, Inser case BasicType::INT16: case BasicType::INT32: case BasicType::INT64: { - std::vector<IAttributeVector::WeightedInt> elements(entries); - entries = std::min(entries, v.get(docid, &elements[0], entries)); - for (uint32_t i = 0; i < entries; ++i) { - if (isWeightedSet) { - Cursor &elem = arr.addObject(); - elem.setLong("item", elements[i].getValue()); - elem.setLong("weight", elements[i].getWeight()); - } else { - arr.addLong(elements[i].getValue()); - } - } - return; } + return std::make_unique<MultiAttrDFW<IAttributeVector::WeightedInt>>(attr.getName(), is_weighted_set, + filter_elements, std::move(matching_elems_fields)); + } case BasicType::FLOAT: case BasicType::DOUBLE: { - std::vector<IAttributeVector::WeightedFloat> elements(entries); - entries = std::min(entries, v.get(docid, &elements[0], entries)); - for (uint32_t i = 0; i < entries; ++i) { - if (isWeightedSet) { - Cursor &elem = arr.addObject(); - elem.setDouble("item", elements[i].getValue()); - elem.setLong("weight", elements[i].getWeight()); - } else { - arr.addDouble(elements[i].getValue()); - } - } - return; } + return std::make_unique<MultiAttrDFW<IAttributeVector::WeightedFloat>>(attr.getName(), is_weighted_set, + filter_elements, std::move(matching_elems_fields)); + } default: // should not happen - LOG(error, "bad value for type: %u\n", t); + LOG(error, "Bad value for attribute type: %u", type); LOG_ASSERT(false); } } -//----------------------------------------------------------------------------- +} -IDocsumFieldWriter * -AttributeDFWFactory::create(IAttributeManager & vecMan, const char *vecName) +std::unique_ptr<IDocsumFieldWriter> +AttributeDFWFactory::create(IAttributeManager& attr_mgr, + const vespalib::string& attr_name, + bool filter_elements, + std::shared_ptr<MatchingElementsFields> matching_elems_fields) { - IAttributeContext::UP ctx = vecMan.createContext(); - const IAttributeVector * vec = ctx->getAttribute(vecName); - if (vec == nullptr) { - LOG(warning, "No valid attribute vector found: %s", vecName); - return nullptr; - } - if (vec->hasMultiValue()) { - return new MultiAttrDFW(vec->getName()); + auto ctx = attr_mgr.createContext(); + const auto* attr = ctx->getAttribute(attr_name); + if (attr == nullptr) { + LOG(warning, "No valid attribute vector found: '%s'", attr_name.c_str()); + return std::unique_ptr<IDocsumFieldWriter>(); + } + if (attr->hasMultiValue()) { + return create_multi_writer(*attr, filter_elements, std::move(matching_elems_fields)); } else { - return new SingleAttrDFW(vec->getName()); + return std::make_unique<SingleAttrDFW>(attr->getName()); } } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.h b/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.h index 211becea16c..55a30f0bb7b 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.h @@ -4,16 +4,30 @@ #include "docsumfieldwriter.h" + +namespace search { class MatchingElementsFields; } namespace search::attribute { class IAttributeVector; } namespace search::docsummary { +/** + * Factory to create an IDocsumFieldWriter to write an attribute vector to slime. + */ +class AttributeDFWFactory { +public: + static std::unique_ptr<IDocsumFieldWriter> create(IAttributeManager& attr_mgr, + const vespalib::string& attr_name, + bool filter_elements = false, + std::shared_ptr<MatchingElementsFields> matching_elems_fields + = std::shared_ptr<MatchingElementsFields>()); +}; + class AttrDFW : public ISimpleDFW { private: vespalib::string _attrName; protected: - const attribute::IAttributeVector & vec(const GetDocsumsState & s) const; + const attribute::IAttributeVector& get_attribute(const GetDocsumsState& s) const; const vespalib::string & getAttributeName() const override { return _attrName; } public: AttrDFW(const vespalib::string & attrName); diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumconfig.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsumconfig.cpp index 2c3366cb94d..f41ada8b2e8 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumconfig.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumconfig.cpp @@ -88,11 +88,9 @@ DynamicDocsumConfig::createFieldWriter(const string & fieldName, const string & rc = fieldWriter.get(); } } else if (overrideName == "attribute") { - const char *vectorName = argument.c_str(); if (getEnvironment() && getEnvironment()->getAttributeManager()) { - IDocsumFieldWriter *fw = AttributeDFWFactory::create(*getEnvironment()->getAttributeManager(), vectorName); - fieldWriter.reset(fw); - rc = fw != NULL; + fieldWriter = AttributeDFWFactory::create(*getEnvironment()->getAttributeManager(), argument); + rc = static_cast<bool>(fieldWriter); } } else if (overrideName == "attributecombiner") { if (getEnvironment() && getEnvironment()->getAttributeManager()) { diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.h b/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.h index 43375fb47f3..a40f105a1cb 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.h @@ -84,14 +84,5 @@ public: vespalib::slime::Inserter &target) override; }; -//-------------------------------------------------------------------------- - -class AttributeDFWFactory -{ -private: - AttributeDFWFactory(); -public: - static IDocsumFieldWriter *create(IAttributeManager & vecMan, const char *vecName); -}; } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.cpp index ae3d6acde43..df510d5bcbc 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.cpp @@ -47,7 +47,7 @@ GeoPositionDFW::insertField(uint32_t docid, GetDocsumsState * dsState, ResType, using vespalib::slime::Symbol; using vespalib::slime::ArrayInserter; - const IAttributeVector & attribute = vec(*dsState); + const auto& attribute = get_attribute(*dsState); if (attribute.hasMultiValue()) { uint32_t entries = attribute.getValueCount(docid); Cursor &arr = target.insertArray(); diff --git a/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.cpp index 6b003553f49..ecdde13b919 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.cpp @@ -30,7 +30,7 @@ uint64_t AbsDistanceDFW::findMinDistance(uint32_t docid, GetDocsumsState *state) { search::common::Location &location = *state->_parsedLocation; - const IAttributeVector & attribute(vec(*state)); + const auto& attribute = get_attribute(*state); uint64_t absdist = std::numeric_limits<int64_t>::max(); int32_t docx = 0; @@ -221,10 +221,10 @@ void PositionsDFW::insertField(uint32_t docid, GetDocsumsState * dsState, ResType type, vespalib::slime::Inserter &target) { if (type == RES_XMLSTRING) { - insertFromAttr(vec(*dsState), docid, target); + insertFromAttr(get_attribute(*dsState), docid, target); return; } - vespalib::asciistream val(formatField(vec(*dsState), docid, type)); + vespalib::asciistream val(formatField(get_attribute(*dsState), docid, type)); target.insertString(vespalib::Memory(val.c_str(), val.size())); } diff --git a/searchsummary/src/vespa/searchsummary/test/CMakeLists.txt b/searchsummary/src/vespa/searchsummary/test/CMakeLists.txt new file mode 100644 index 00000000000..ae4414bb078 --- /dev/null +++ b/searchsummary/src/vespa/searchsummary/test/CMakeLists.txt @@ -0,0 +1,7 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_library(searchsummary_test OBJECT + SOURCES + mock_attribute_manager.cpp + AFTER + searchsummary_config +) diff --git a/searchsummary/src/vespa/searchsummary/test/mock_attribute_manager.cpp b/searchsummary/src/vespa/searchsummary/test/mock_attribute_manager.cpp new file mode 100644 index 00000000000..bd7307d1624 --- /dev/null +++ b/searchsummary/src/vespa/searchsummary/test/mock_attribute_manager.cpp @@ -0,0 +1,72 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "mock_attribute_manager.h" +#include <vespa/searchlib/attribute/attributefactory.h> +#include <vespa/searchlib/attribute/attributevector.h> +#include <vespa/searchlib/attribute/attributevector.hpp> +#include <vespa/searchlib/attribute/floatbase.h> +#include <vespa/searchlib/attribute/integerbase.h> +#include <vespa/searchlib/attribute/stringbase.h> +#include <cassert> + +using search::attribute::BasicType; +using search::attribute::CollectionType; +using search::attribute::Config; + +namespace search::docsummary::test { + +template <typename AttributeType, typename ValueType> +void +MockAttributeManager::build_attribute(const vespalib::string& name, BasicType type, + search::attribute::CollectionType col_type, + const std::vector<std::vector<ValueType>>& values) +{ + Config cfg(type, col_type); + auto attr_base = AttributeFactory::createAttribute(name, cfg); + assert(attr_base); + auto attr = std::dynamic_pointer_cast<AttributeType>(attr_base); + assert(attr); + attr->addReservedDoc(); + for (const auto& docValues : values) { + uint32_t docId = 0; + attr->addDoc(docId); + for (const auto& value : docValues) { + attr->append(docId, value, 1); + } + attr->commit(); + } + _mgr.add(attr); +} + +MockAttributeManager::MockAttributeManager() + : _mgr() +{ +} + +MockAttributeManager::~MockAttributeManager() = default; + +void +MockAttributeManager::build_string_attribute(const vespalib::string& name, + const std::vector<std::vector<vespalib::string>>& values, + CollectionType col_type) +{ + build_attribute<StringAttribute, vespalib::string>(name, BasicType::Type::STRING, col_type, values); +} + +void +MockAttributeManager::build_float_attribute(const vespalib::string& name, + const std::vector<std::vector<double>>& values, + CollectionType col_type) +{ + build_attribute<FloatingPointAttribute, double>(name, BasicType::Type::DOUBLE, col_type, values); +} + +void +MockAttributeManager::build_int_attribute(const vespalib::string& name, BasicType type, + const std::vector<std::vector<int64_t>>& values, + CollectionType col_type) +{ + build_attribute<IntegerAttribute, int64_t>(name, type, col_type, values); +} + +} diff --git a/searchsummary/src/vespa/searchsummary/test/mock_attribute_manager.h b/searchsummary/src/vespa/searchsummary/test/mock_attribute_manager.h new file mode 100644 index 00000000000..a7e425e50b6 --- /dev/null +++ b/searchsummary/src/vespa/searchsummary/test/mock_attribute_manager.h @@ -0,0 +1,37 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/searchcommon/attribute/basictype.h> +#include <vespa/searchlib/attribute/attributemanager.h> + +namespace search::docsummary::test { + +/** + * Class used to build attributes and populate a manager for testing. + */ +class MockAttributeManager { +private: + AttributeManager _mgr; + + template <typename AttributeType, typename ValueType> + void build_attribute(const vespalib::string& name, search::attribute::BasicType type, + search::attribute::CollectionType col_type, + const std::vector<std::vector<ValueType>>& values); + +public: + MockAttributeManager(); + ~MockAttributeManager(); + AttributeManager& mgr() { return _mgr; } + + void build_string_attribute(const vespalib::string& name, + const std::vector<std::vector<vespalib::string>>& values, + search::attribute::CollectionType col_type = search::attribute::CollectionType::ARRAY); + void build_float_attribute(const vespalib::string& name, + const std::vector<std::vector<double>>& values, + search::attribute::CollectionType col_type = search::attribute::CollectionType::ARRAY); + void build_int_attribute(const vespalib::string& name, search::attribute::BasicType type, + const std::vector<std::vector<int64_t>>& values, + search::attribute::CollectionType col_type = search::attribute::CollectionType::ARRAY); + +}; + +} diff --git a/searchsummary/src/vespa/searchsummary/test/mock_state_callback.h b/searchsummary/src/vespa/searchsummary/test/mock_state_callback.h new file mode 100644 index 00000000000..b3ee405c856 --- /dev/null +++ b/searchsummary/src/vespa/searchsummary/test/mock_state_callback.h @@ -0,0 +1,35 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/searchlib/common/matching_elements.h> +#include <vespa/searchsummary/docsummary/docsumstate.h> + +namespace search::docsummary::test { + +class MockStateCallback : public GetDocsumsStateCallback { +private: + MatchingElements _matching_elems; + +public: + MockStateCallback() + : GetDocsumsStateCallback(), + _matching_elems() + { + } + ~MockStateCallback() override { } + void FillSummaryFeatures(GetDocsumsState*, IDocsumEnvironment*) override { } + void FillRankFeatures(GetDocsumsState*, IDocsumEnvironment*) override { } + void ParseLocation(GetDocsumsState*) override { } + std::unique_ptr<MatchingElements> fill_matching_elements(const search::MatchingElementsFields&) override { + return std::make_unique<MatchingElements>(_matching_elems); + } + + void add_matching_elements(uint32_t docid, const vespalib::string& field_name, + const std::vector<uint32_t>& elements) { + _matching_elems.add_matching_elements(docid, field_name, elements); + } + void clear() { + _matching_elems = MatchingElements(); + } +}; + +} diff --git a/searchsummary/src/vespa/searchsummary/test/slime_value.h b/searchsummary/src/vespa/searchsummary/test/slime_value.h new file mode 100644 index 00000000000..3cc461d04ca --- /dev/null +++ b/searchsummary/src/vespa/searchsummary/test/slime_value.h @@ -0,0 +1,24 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/data/slime/slime.h> +#include <cassert> + +namespace search::docsummary::test { + +/** + * Utility class that wraps a slime object generated from json. + */ +struct SlimeValue { + vespalib::Slime slime; + + SlimeValue(const vespalib::string& json_input) + : slime() + { + size_t used = vespalib::slime::JsonFormat::decode(json_input, slime); + assert(used > 0); + } +}; + +} diff --git a/storage/src/tests/bucketdb/lockablemaptest.cpp b/storage/src/tests/bucketdb/lockablemaptest.cpp index 3dd4dfccf77..101b9d014fa 100644 --- a/storage/src/tests/bucketdb/lockablemaptest.cpp +++ b/storage/src/tests/bucketdb/lockablemaptest.cpp @@ -641,155 +641,6 @@ TEST(LockableMapTest, find_all_inconsistent_below_16_bits) { EXPECT_EQ(A(3,4,5), *results[id3.stripUnused()]); // sub bucket } -TEST(LockableMapTest, create) { - Map map; - { - document::BucketId id1(58, 0x43d6c878000004d2ull); - - auto entries = map.getContained(id1, "foo"); - - EXPECT_EQ(0, entries.size()); - - Map::WrappedEntry entry = map.createAppropriateBucket(36, "", id1); - EXPECT_EQ(document::BucketId(36,0x8000004d2ull), entry.getBucketId()); - } - { - document::BucketId id1(58, 0x423bf1e0000004d2ull); - - auto entries = map.getContained(id1, "foo"); - EXPECT_EQ(0, entries.size()); - - Map::WrappedEntry entry = map.createAppropriateBucket(36, "", id1); - EXPECT_EQ(document::BucketId(36,0x0000004d2ull), entry.getBucketId()); - } - - EXPECT_EQ(2, map.size()); -} - -TEST(LockableMapTest, create_2) { - Map map; - { - document::BucketId id1(58, 0xeaf77782000004d2); - Map::WrappedEntry entry( - map.get(id1.stripUnused().toKey(), "foo", true)); - } - { - document::BucketId id1(58, 0x00000000000004d2); - auto entries = map.getContained(id1, "foo"); - - EXPECT_EQ(0, entries.size()); - - Map::WrappedEntry entry = map.createAppropriateBucket(16, "", id1); - - EXPECT_EQ(document::BucketId(34, 0x0000004d2ull), entry.getBucketId()); - } - - EXPECT_EQ(2, map.size()); -} - -TEST(LockableMapTest, create_3) { - Map map; - { - document::BucketId id1(58, 0xeaf77780000004d2); - Map::WrappedEntry entry( - map.get(id1.stripUnused().toKey(), "foo", true)); - } - { - document::BucketId id1(58, 0xeaf77782000004d2); - Map::WrappedEntry entry( - map.get(id1.stripUnused().toKey(), "foo", true)); - } - { - document::BucketId id1(58, 0x00000000000004d2); - auto entries = map.getContained(id1, "foo"); - - EXPECT_EQ(0, entries.size()); - - Map::WrappedEntry entry = map.createAppropriateBucket(16, "", id1); - EXPECT_EQ(document::BucketId(40, 0x0000004d2ull), entry.getBucketId()); - } -} - -TEST(LockableMapTest, create_4) { - Map map; - { - document::BucketId id1(16, 0x00000000000004d1); - Map::WrappedEntry entry( - map.get(id1.stripUnused().toKey(), "foo", true)); - } - { - document::BucketId id1(40, 0x00000000000004d2); - Map::WrappedEntry entry( - map.get(id1.stripUnused().toKey(), "foo", true)); - } - { - document::BucketId id1(58, 0x00000000010004d2); - Map::WrappedEntry entry = map.createAppropriateBucket(16, "", id1); - - EXPECT_EQ(document::BucketId(25, 0x0010004d2ull), entry.getBucketId()); - } -} - -TEST(LockableMapTest, create_5) { - Map map; - { - document::BucketId id1(0x8c000000000004d2); - Map::WrappedEntry entry( - map.get(id1.stripUnused().toKey(), "foo", true)); - } - - { - document::BucketId id1(0xeb54b3ac000004d2); - Map::WrappedEntry entry( - map.get(id1.stripUnused().toKey(), "foo", true)); - } - - { - document::BucketId id1(0x88000002000004d2); - Map::WrappedEntry entry( - map.get(id1.stripUnused().toKey(), "foo", true)); - } - { - document::BucketId id1(0x84000001000004d2); - Map::WrappedEntry entry( - map.get(id1.stripUnused().toKey(), "foo", true)); - } - { - document::BucketId id1(0xe9944a44000004d2); - Map::WrappedEntry entry = map.createAppropriateBucket(16, "", id1); - EXPECT_EQ(document::BucketId(0x90000004000004d2), entry.getBucketId()); - } -} - -TEST(LockableMapTest, create_6) { - Map map; - { - document::BucketId id1(58, 0xeaf77780000004d2); - Map::WrappedEntry entry( - map.get(id1.stripUnused().toKey(), "foo", true)); - } - { - document::BucketId id1(40, 0x00000000000004d1); - - Map::WrappedEntry entry( - map.get(id1.stripUnused().toKey(), "foo", true)); - } - { - document::BucketId id1(58, 0x00000000010004d2); - Map::WrappedEntry entry = map.createAppropriateBucket(16, "", id1); - EXPECT_EQ(document::BucketId(25, 0x0010004d2ull), entry.getBucketId()); - } -} - -TEST(LockableMapTest, create_empty) { - Map map; - { - document::BucketId id1(58, 0x00000000010004d2); - Map::WrappedEntry entry = map.createAppropriateBucket(16, "", id1); - EXPECT_EQ(document::BucketId(16, 0x0000004d2ull), entry.getBucketId()); - } -} - TEST(LockableMapTest, is_consistent) { Map map; document::BucketId id1(16, 0x00001); // contains id2-id3 diff --git a/storage/src/vespa/storage/bucketdb/lockablemap.h b/storage/src/vespa/storage/bucketdb/lockablemap.h index f6357e8851e..8b4e403b899 100644 --- a/storage/src/vespa/storage/bucketdb/lockablemap.h +++ b/storage/src/vespa/storage/bucketdb/lockablemap.h @@ -187,11 +187,6 @@ public: std::map<BucketId, WrappedEntry> getContained(const BucketId& bucketId, const char* clientId); - WrappedEntry - createAppropriateBucket(uint16_t newBucketBits, - const char* clientId, - const BucketId& bucket); - typedef std::map<BucketId, WrappedEntry> EntryMap; /** diff --git a/storage/src/vespa/storage/bucketdb/lockablemap.hpp b/storage/src/vespa/storage/bucketdb/lockablemap.hpp index a0d7e63c1fd..2ca2183ae26 100644 --- a/storage/src/vespa/storage/bucketdb/lockablemap.hpp +++ b/storage/src/vespa/storage/bucketdb/lockablemap.hpp @@ -591,40 +591,6 @@ LockableMap<Map>::addAndLockResults( uint8_t getMinDiffBits(uint16_t minBits, const document::BucketId& a, const document::BucketId& b); template<typename Map> -typename LockableMap<Map>::WrappedEntry -LockableMap<Map>::createAppropriateBucket( - uint16_t newBucketBits, - const char* clientId, - const BucketId& bucket) -{ - std::unique_lock<std::mutex> guard(_lock); - typename Map::const_iterator iter = _map.lower_bound(bucket.toKey()); - - // Find the two buckets around the possible new bucket. The new - // bucket's used bits should be the highest used bits it can be while - // still being different from both of these. - if (iter != _map.end()) { - newBucketBits = getMinDiffBits(newBucketBits, BucketId(BucketId::keyToBucketId(iter->first)), bucket); - } - - if (iter != _map.begin()) { - --iter; - newBucketBits = getMinDiffBits(newBucketBits, BucketId(BucketId::keyToBucketId(iter->first)), bucket); - } - - BucketId newBucket(newBucketBits, bucket.getRawId()); - newBucket.setUsedBits(newBucketBits); - BucketId::Type key = newBucket.stripUnused().toKey(); - - LockId lid(key, clientId); - acquireKey(lid, guard); - bool preExisted; - typename Map::iterator it = _map.find(key, true, preExisted); - _lockedKeys.insert(LockId(key, clientId)); - return WrappedEntry(*this, key, it->second, clientId, preExisted); -} - -template<typename Map> std::map<document::BucketId, typename LockableMap<Map>::WrappedEntry> LockableMap<Map>::getContained(const BucketId& bucket, const char* clientId) diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/Deployment.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Deployment.java index 8327916b41d..7d7b2f74981 100644 --- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/Deployment.java +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Deployment.java @@ -2,7 +2,7 @@ package ai.vespa.hosted.cd; /** - * A deployment of a Vespa application, which contains endpoints for document and metrics retrieval. + * A deployment of a Vespa application, which contains endpoints for document retrieval. * * @author jonmv */ diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/Endpoint.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Endpoint.java index 46f5f8ef5fd..bd6f30767f2 100644 --- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/Endpoint.java +++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/Endpoint.java @@ -2,7 +2,6 @@ package ai.vespa.hosted.cd; import ai.vespa.hosted.api.EndpointAuthenticator; -import ai.vespa.hosted.cd.metric.Metrics; import java.net.URI; import java.net.http.HttpClient; @@ -17,7 +16,7 @@ import static java.net.URLEncoder.encode; import static java.nio.charset.StandardCharsets.UTF_8; /** - * An endpoint in a Vespa application {@link Deployment}, which allows document and metrics retrieval. + * An endpoint in a Vespa application {@link Deployment}, which allows document retrieval. * * @author jonmv */ diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Metric.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Metric.java deleted file mode 100644 index 39e9cd5bc75..00000000000 --- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Metric.java +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package ai.vespa.hosted.cd.metric; - -import java.util.HashMap; -import java.util.Map; -import java.util.NoSuchElementException; -import java.util.Set; -import java.util.StringJoiner; - -import static java.util.Map.copyOf; -import static java.util.stream.Collectors.reducing; -import static java.util.stream.Collectors.toUnmodifiableMap; - -/** - * A set of statistics for a metric, for points over a space with named dimensions of arbitrary type. - * - * @author jonmv - */ -public class Metric { - - private final Map<Map<String, ?>, Statistic> statistics; - - private Metric(Map<Map<String, ?>, Statistic> statistics) { - this.statistics = statistics; - } - - /** Creates a new Metric with a copy of the given data. */ - public static Metric of(Map<Map<String, ?>, Statistic> data) { - if (data.isEmpty()) - throw new IllegalArgumentException("No data given."); - - Map<Map<String, ?>, Statistic> copies = new HashMap<>(); - Set<String> dimensions = data.keySet().iterator().next().keySet(); - data.forEach((point, statistic) -> { - if ( ! point.keySet().equals(dimensions)) - throw new IllegalArgumentException("Given data has inconsistent dimensions: '" + dimensions + "' vs '" + point.keySet() + "'."); - - copies.put(copyOf(point), statistic); - }); - - return new Metric(copyOf(copies)); - } - - /** Returns a Metric view of the subset of points in the given hyperplane; its dimensions must be a subset of those of this Metric. */ - public Metric at(Map<String, ?> hyperplane) { - return new Metric(statistics.keySet().stream() - .filter(point -> point.entrySet().containsAll(hyperplane.entrySet())) - .collect(toUnmodifiableMap(point -> point, statistics::get))); - } - - /** Returns a version of this where statistics along the given hyperspace are aggregated. This does not preserve last, 95 and 99 percentile values. */ - public Metric collapse(Set<String> hyperspace) { - return new Metric(statistics.keySet().stream() - .collect(toUnmodifiableMap(point -> point.keySet().stream() - .filter(dimension -> ! hyperspace.contains(dimension)) - .collect(toUnmodifiableMap(dimension -> dimension, point::get)), - statistics::get, - Statistic::mergedWith))); - } - - /** Returns a collapsed version of this, with all statistics aggregated. This does not preserve last, 95 and 99 percentile values. */ - public Metric collapse() { - Map<String, ?> firstStatistic = statistics.keySet().iterator().next(); - return firstStatistic == null ? this : collapse(firstStatistic.keySet()); - } - - /** If this Metric contains a single point, returns the Statistic of that point; otherwise, throws an exception. */ - public Statistic statistic() { - if (statistics.size() == 1) - return statistics.values().iterator().next(); - - if (statistics.isEmpty()) - throw new NoSuchElementException("This Metric has no data."); - - throw new IllegalStateException("This Metric has more than one point of data."); - } - - /** Returns the underlying, unmodifiable Map. */ - public Map<Map<String, ?>, Statistic> asMap() { - return statistics; - } - - @Override - public String toString() { - return new StringJoiner(", ", Metric.class.getSimpleName() + "[", "]") - .add("statistics=" + statistics) - .toString(); - } - -} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Metrics.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Metrics.java deleted file mode 100644 index bdcfac2529e..00000000000 --- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Metrics.java +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package ai.vespa.hosted.cd.metric; - -import ai.vespa.hosted.cd.Endpoint; - -import java.time.Instant; -import java.util.Map; -import java.util.NoSuchElementException; -import java.util.StringJoiner; - -import static java.util.Map.copyOf; - -/** - * Metrics from a Vespa application {@link Endpoint}, indexed by their names, and optionally by a set of custom dimensions. - * - * Metrics are collected from the <a href="https://docs.vespa.ai/documentation/reference/metrics.html">metrics</a> - * API of a Vespa endpoint, and contain the current health status of the endpoint, values for all configured metrics in - * that endpoint, and the time interval from which these metrics were sampled. - * - * Each metric is indexed by a name, and, optionally, along a custom set of dimensions, given by a {@code Map<String, String>}. - * - * @author jonmv - */ -public class Metrics { - - private final Instant start, end; - private final Map<String, Metric> metrics; - - private Metrics(Instant start, Instant end, Map<String, Metric> metrics) { - this.start = start; - this.end = end; - this.metrics = metrics; - } - - public static Metrics of(Instant start, Instant end, Map<String, Metric> metrics) { - if ( ! start.isBefore(end)) - throw new IllegalArgumentException("Given time interval must be positive: '" + start + "' to '" + end + "'."); - - return new Metrics(start, end, copyOf(metrics)); - } - - /** Returns the start of the time window from which these metrics were sampled, or throws if the status is {@code Status.down}. */ - public Instant start() { - return start; - } - - /** Returns the end of the time window from which these metrics were sampled, or throws if the status is {@code Status.down}. */ - public Instant end() { - return end; - } - - /** Returns the metric with the given name, or throws a NoSuchElementException if no such Metric is known. */ - public Metric get(String name) { - if ( ! metrics.containsKey(name)) - throw new NoSuchElementException("No metric with name '" + name + "'."); - - return metrics.get(name); - } - - /** Returns the underlying, unmodifiable Map. */ - public Map<String, Metric> asMap() { - return metrics; - } - - @Override - public String toString() { - return new StringJoiner(", ", Metrics.class.getSimpleName() + "[", "]") - .add("start=" + start) - .add("end=" + end) - .add("metrics=" + metrics) - .toString(); - } - -} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Space.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Space.java deleted file mode 100644 index 561c0f9dee3..00000000000 --- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Space.java +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package ai.vespa.hosted.cd.metric; - -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.IntStream; - -import static java.util.stream.Collectors.toUnmodifiableMap; - -/** - * Used to easily generate points (Map<String, ?>) for a space defined here by its dimension names. - * - * @author jonmv - */ -public class Space { - - private final List<String> dimensions; - - private Space(List<String> dimensions) { - this.dimensions = dimensions; - } - - /** Creates a new space with the given named dimensions, in order. */ - public static Space of(List<String> dimensions) { - if (Set.copyOf(dimensions).size() != dimensions.size()) - throw new IllegalArgumentException("Duplicated dimension names in '" + dimensions + "'."); - - return new Space(List.copyOf(dimensions)); - } - - /** Returns a point in this space, with the given values along each dimensions, in order. */ - public Map<String, ?> at(List<?> values) { - if (dimensions.size() != values.size()) - throw new IllegalArgumentException("This space has " + dimensions.size() + " dimensions, but " + values.size() + " were given."); - - return IntStream.range(0, dimensions.size()).boxed().collect(toUnmodifiableMap(dimensions::get, values::get)); - } - - /** Returns a point in this space, with the given values along each dimensions, in order. */ - public Map<String, ?> at(Object... values) { - return at(List.of(values)); - } - -} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Statistic.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Statistic.java deleted file mode 100644 index 62b2528e0a4..00000000000 --- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Statistic.java +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package ai.vespa.hosted.cd.metric; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.NoSuchElementException; -import java.util.StringJoiner; - -import static java.util.Map.copyOf; - -/** - * Known statistic about a metric, at a certain point. - * - * @author jonmv - */ -public class Statistic { - - private final Map<Type, Double> data; - - /** Creates a new Statistic with a copy of the given data. */ - private Statistic(Map<Type, Double> data) { - this.data = data; - } - - public static Statistic of(Map<Type, Double> data) { - for (Type type : List.of(Type.count, Type.rate, Type.average)) - if ( ! data.containsKey(type)) - throw new IllegalArgumentException("Required data type '" + type + "' not present in '" + data + "'"); - - return new Statistic(copyOf(data)); - } - - /** Returns the value of the given type, or throws a NoSuchElementException if this isn't known. */ - public double get(Type key) { - if ( ! data.containsKey(key)) - throw new NoSuchElementException("No value with key '" + key + "' is known."); - - return data.get(key); - } - - /** Returns the underlying, unmodifiable Map. */ - public Map<Type, Double> asMap() { - return data; - } - - Statistic mergedWith(Statistic other) { - if (data.keySet().equals(other.data.keySet())) - throw new IllegalArgumentException("Unequal key sets '" + data.keySet() + "' and '" + other.data.keySet() + "'."); - - Map<Type, Double> merged = new HashMap<>(); - double n1 = get(Type.count), n2 = other.get(Type.count); - for (Type type : data.keySet()) switch (type) { - case count: merged.put(type, n1 + n2); break; - case rate: merged.put(type, get(Type.rate) + other.get(Type.rate)); break; - case max: merged.put(type, Math.max(get(Type.max), other.get(Type.max))); break; - case min: merged.put(type, Math.min(get(Type.min), other.get(Type.min))); break; - case average: merged.put(type, (n1 * get(Type.average) + n2 * other.get(Type.average)) / (n1 + n2)); break; - case last: - case percentile95: - case percentile99: break; - default: throw new IllegalArgumentException("Unexpected type '" + type + "'."); - } - return of(merged); - } - - @Override - public String toString() { - return new StringJoiner(", ", Statistic.class.getSimpleName() + "[", "]") - .add("data=" + data) - .toString(); - } - -} diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Type.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Type.java deleted file mode 100644 index d02593e5eb3..00000000000 --- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/metric/Type.java +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package ai.vespa.hosted.cd.metric; - -/** - * Known statistic types. - */ -public enum Type { - - /** 95th percentile measurement. */ - percentile95, - - /** 99th percentile measurement. */ - percentile99, - - /** Average over all measurements. */ - average, - - /** Number of measurements. */ - count, - - /** Last measurement. */ - last, - - /** Maximum measurement. */ - max, - - /** Minimum measurement. */ - min, - - /** Number of measurements per second. */ - rate; - -} |