diff options
| author | Harald Musum <musum@verizonmedia.com> | 2021-04-08 09:41:33 +0200 |
|---|---|---|
| committer | Harald Musum <musum@verizonmedia.com> | 2021-04-08 09:41:33 +0200 |
| commit | 52a5aa107a526326022b0a5779a54f6b48f585e5 (patch) | |
| tree | 70f96190f0de4e6fd699fa3dfb9d3ad566cc1b65 | |
| parent | 710092a282bd71f363901831d3feeeca43c64d08 (diff) | |
| parent | 0f8387d7f21d392f9d7ab5ee60b10acce1dff4d8 (diff) | |
Merge branch 'master' into hmusum/cleanup-7
440 files changed, 5229 insertions, 3176 deletions
diff --git a/Code-map.md b/Code-map.md index 0ed326a2005..a02b52c76a8 100644 --- a/Code-map.md +++ b/Code-map.md @@ -56,7 +56,7 @@ Document operation modules: - [document](https://github.com/vespa-engine/vespa/tree/master/document) - the document model - documents, fields and document types, and operations on documents. Implemented in both Java and C++. - [messagebus](https://github.com/vespa-engine/vespa/tree/master/messagebus) - generic async, multi-hop message passing implemented in both Java and C++. -- [jdisc_messagebus_service](https://github.com/vespa-engine/vespa/tree/master/jdisc_messagebus_service) - MessageBus connector for jDisc. +- [container-messagebus](https://github.com/vespa-engine/vespa/tree/master/container-messagebus) - MessageBus connector for jDisc. - [documentapi](https://github.com/vespa-engine/vespa/tree/master/documentapi) - API for issuing document operations to Vespa over messagebus. - [docproc](https://github.com/vespa-engine/vespa/tree/master/docproc) - chainable document (operation) processors: Document operations issued over messagebus to Vespa will usually be routed through a container running a document processor chain. - [indexinglanguage](https://github.com/vespa-engine/vespa/tree/master/indexinglanguage) - implementation of the "indexing" language which is used to express the statements prefixed by "indexing:" in the search definition. diff --git a/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ServiceCluster.java b/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ServiceCluster.java index 417b5792586..baede298896 100644 --- a/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ServiceCluster.java +++ b/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ServiceCluster.java @@ -52,10 +52,12 @@ public class ServiceCluster { return applicationInstance.get(); } + @JsonIgnore public boolean isConfigServerLike() { return isConfigServer() || isController(); } + @JsonIgnore public boolean isController() { return isHostedVespaApplicationWithId(ApplicationInstanceId.CONTROLLER) && Objects.equals(clusterId, ClusterId.CONTROLLER) && @@ -63,28 +65,33 @@ public class ServiceCluster { } /** Is a config server (and not controller!) */ + @JsonIgnore public boolean isConfigServer() { return isHostedVespaApplicationWithId(ApplicationInstanceId.CONFIG_SERVER) && Objects.equals(clusterId, ClusterId.CONFIG_SERVER) && Objects.equals(serviceType, ServiceType.CONFIG_SERVER); } + @JsonIgnore public boolean isConfigServerHost() { return isHostedVespaApplicationWithPredicate(ApplicationInstanceId::isConfigServerHost) && Objects.equals(clusterId, ClusterId.CONFIG_SERVER_HOST) && Objects.equals(serviceType, ServiceType.HOST_ADMIN); } + @JsonIgnore public boolean isControllerHost() { return isHostedVespaApplicationWithId(ApplicationInstanceId.CONTROLLER_HOST) && Objects.equals(clusterId, ClusterId.CONTROLLER_HOST) && Objects.equals(serviceType, ServiceType.HOST_ADMIN); } + @JsonIgnore public boolean isConfigServerHostLike() { return isConfigServerHost() || isControllerHost(); } + @JsonIgnore public boolean isTenantHost() { return isHostedVespaApplicationWithPredicate(ApplicationInstanceId::isTenantHost) && Objects.equals(clusterId, ClusterId.TENANT_HOST) && diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/IdentityProviderRequestHandler.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/IdentityProviderRequestHandler.java index 8593401b887..fe6fa0baf2d 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/IdentityProviderRequestHandler.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/IdentityProviderRequestHandler.java @@ -42,9 +42,12 @@ public class IdentityProviderRequestHandler extends RestApiRequestHandler<Identi .addRoute(RestApi.route("/athenz/v1/provider/identity-document/tenant/{host}") .get(self::getTenantIdentityDocument)) .addRoute(RestApi.route("/athenz/v1/provider/instance") - .post(self::confirmInstance)) + .post(InstanceConfirmation.class, self::confirmInstance)) .addRoute(RestApi.route("/athenz/v1/provider/refresh") - .post(self::confirmInstanceRefresh)) + .post(InstanceConfirmation.class, self::confirmInstanceRefresh)) + .registerJacksonRequestEntity(InstanceConfirmation.class) + .registerJacksonResponseEntity(InstanceConfirmation.class) + .registerJacksonResponseEntity(SignedIdentityDocumentEntity.class) // Overriding object mapper to change serialization of timestamps .setObjectMapper(new ObjectMapper() .registerModule(new JavaTimeModule()) @@ -63,8 +66,7 @@ public class IdentityProviderRequestHandler extends RestApiRequestHandler<Identi return getIdentityDocument(host, IdentityType.TENANT); } - private InstanceConfirmation confirmInstance(RestApi.RequestContext context) { - InstanceConfirmation instanceConfirmation = context.requestContentOrThrow().consumeJacksonEntity(InstanceConfirmation.class); + private InstanceConfirmation confirmInstance(RestApi.RequestContext context, InstanceConfirmation instanceConfirmation) { log.log(Level.FINE, instanceConfirmation.toString()); if (!instanceValidator.isValidInstance(instanceConfirmation)) { log.log(Level.SEVERE, "Invalid instance: " + instanceConfirmation); @@ -73,8 +75,7 @@ public class IdentityProviderRequestHandler extends RestApiRequestHandler<Identi return instanceConfirmation; } - private InstanceConfirmation confirmInstanceRefresh(RestApi.RequestContext context) { - InstanceConfirmation instanceConfirmation = context.requestContentOrThrow().consumeJacksonEntity(InstanceConfirmation.class); + private InstanceConfirmation confirmInstanceRefresh(RestApi.RequestContext context, InstanceConfirmation instanceConfirmation) { log.log(Level.FINE, instanceConfirmation.toString()); if (!instanceValidator.isValidRefresh(instanceConfirmation)) { log.log(Level.SEVERE, "Invalid instance refresh: " + instanceConfirmation); diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/InstanceConfirmation.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/InstanceConfirmation.java index 27507c425cd..5641eb51668 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/InstanceConfirmation.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/InstanceConfirmation.java @@ -13,7 +13,6 @@ import com.fasterxml.jackson.databind.JsonSerializer; import com.fasterxml.jackson.databind.SerializerProvider; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonSerialize; -import com.yahoo.restapi.RestApi; import com.yahoo.vespa.athenz.identityprovider.api.bindings.SignedIdentityDocumentEntity; import java.io.IOException; @@ -26,7 +25,7 @@ import java.util.Objects; * * @author bjorncs */ -public class InstanceConfirmation implements RestApi.JacksonRequestEntity, RestApi.JacksonResponseEntity { +public class InstanceConfirmation { @JsonProperty("provider") public final String provider; @JsonProperty("domain") public final String domain; diff --git a/cloud-tenant-base-dependencies-enforcer/pom.xml b/cloud-tenant-base-dependencies-enforcer/pom.xml index d2188e37123..8a504635f25 100644 --- a/cloud-tenant-base-dependencies-enforcer/pom.xml +++ b/cloud-tenant-base-dependencies-enforcer/pom.xml @@ -167,10 +167,8 @@ <include>com.yahoo.vespa:hosted-zone-api:*:jar:provided</include> <include>com.yahoo.vespa:http-utils:*:jar:provided</include> <include>com.yahoo.vespa:jdisc_core:*:jar:provided</include> - <include>com.yahoo.vespa:jdisc_messagebus_service:*:jar:provided</include> <include>com.yahoo.vespa:jrt:*:jar:provided</include> <include>com.yahoo.vespa:linguistics:*:jar:provided</include> - <include>com.yahoo.vespa:messagebus-disc:*:jar:provided</include> <include>com.yahoo.vespa:messagebus:*:jar:provided</include> <include>com.yahoo.vespa:model-evaluation:*:jar:provided</include> <include>com.yahoo.vespa:predicate-search-core:*:jar:provided</include> diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java index 2f79393a020..75eea998346 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java @@ -170,7 +170,7 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd StatusPageServerInterface statusPageServer, MetricReporter metricReporter) throws Exception { Timer timer = new RealTimer(); - MetricUpdater metricUpdater = new MetricUpdater(metricReporter, options.fleetControllerIndex); + MetricUpdater metricUpdater = new MetricUpdater(metricReporter, options.fleetControllerIndex, options.clusterName); EventLog log = new EventLog(timer, metricUpdater); ContentCluster cluster = new ContentCluster( options.clusterName, @@ -757,21 +757,25 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd } private boolean processNextQueuedRemoteTask() { - if ( ! remoteTasks.isEmpty()) { - final RemoteClusterControllerTask.Context context = createRemoteTaskProcessingContext(); - final RemoteClusterControllerTask task = remoteTasks.poll(); - log.finest(() -> String.format("Processing remote task of type '%s'", task.getClass().getName())); - task.doRemoteFleetControllerTask(context); - if (taskMayBeCompletedImmediately(task)) { - log.finest(() -> String.format("Done processing remote task of type '%s'", task.getClass().getName())); - task.notifyCompleted(); - } else { - log.finest(() -> String.format("Remote task of type '%s' queued until state recomputation", task.getClass().getName())); - tasksPendingStateRecompute.add(task); - } - return true; + metricUpdater.updateRemoteTaskQueueSize(remoteTasks.size()); + + RemoteClusterControllerTask task = remoteTasks.poll(); + if (task == null) { + return false; } - return false; + + final RemoteClusterControllerTask.Context context = createRemoteTaskProcessingContext(); + log.finest(() -> String.format("Processing remote task of type '%s'", task.getClass().getName())); + task.doRemoteFleetControllerTask(context); + if (taskMayBeCompletedImmediately(task)) { + log.finest(() -> String.format("Done processing remote task of type '%s'", task.getClass().getName())); + task.notifyCompleted(); + } else { + log.finest(() -> String.format("Remote task of type '%s' queued until state recomputation", task.getClass().getName())); + tasksPendingStateRecompute.add(task); + } + + return true; } private boolean taskMayBeCompletedImmediately(RemoteClusterControllerTask task) { diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MetricUpdater.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MetricUpdater.java index 6a80772904f..94b6f412ce6 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MetricUpdater.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MetricUpdater.java @@ -14,9 +14,10 @@ public class MetricUpdater { private final ComponentMetricReporter metricReporter; - public MetricUpdater(MetricReporter metricReporter, int controllerIndex) { + public MetricUpdater(MetricReporter metricReporter, int controllerIndex, String clusterName) { this.metricReporter = new ComponentMetricReporter(metricReporter, "cluster-controller."); this.metricReporter.addDimension("controller-index", String.valueOf(controllerIndex)); + this.metricReporter.addDimension("clusterid", clusterName); } public MetricReporter.Context createContext(Map<String, String> dimensions) { @@ -99,4 +100,7 @@ public class MetricUpdater { metricReporter.add("node-event", 1); } + public void updateRemoteTaskQueueSize(int size) { + metricReporter.set("remote-task-queue.size", size); + } } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterFeedBlockTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterFeedBlockTest.java index 56d567db035..c4db6b9525b 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterFeedBlockTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterFeedBlockTest.java @@ -49,7 +49,7 @@ public class ClusterFeedBlockTest extends FleetControllerTest { } communicator = new DummyCommunicator(nodes, timer); - MetricUpdater metricUpdater = new MetricUpdater(new NoMetricReporter(), options.fleetControllerIndex); + MetricUpdater metricUpdater = new MetricUpdater(new NoMetricReporter(), options.fleetControllerIndex, options.clusterName); EventLog eventLog = new EventLog(timer, metricUpdater); ContentCluster cluster = new ContentCluster(options.clusterName, options.nodes, options.storageDistribution); NodeStateGatherer stateGatherer = new NodeStateGatherer(timer, timer, eventLog); diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java index b238a9eadcf..1d7b6886222 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java @@ -153,7 +153,7 @@ public abstract class FleetControllerTest implements Waiter { FleetController createFleetController(boolean useFakeTimer, FleetControllerOptions options, boolean startThread, StatusPageServerInterface status) throws Exception { Objects.requireNonNull(status, "status server cannot be null"); Timer timer = useFakeTimer ? this.timer : new RealTimer(); - MetricUpdater metricUpdater = new MetricUpdater(new NoMetricReporter(), options.fleetControllerIndex); + MetricUpdater metricUpdater = new MetricUpdater(new NoMetricReporter(), options.fleetControllerIndex, options.clusterName); EventLog log = new EventLog(timer, metricUpdater); ContentCluster cluster = new ContentCluster( options.clusterName, diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MetricReporterTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MetricReporterTest.java index 3e44995dc78..aef223d7aee 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MetricReporterTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MetricReporterTest.java @@ -21,9 +21,11 @@ import static org.mockito.hamcrest.MockitoHamcrest.doubleThat; public class MetricReporterTest { + private static final String CLUSTER_NAME = "foo"; + private static class Fixture { final MetricReporter mockReporter = mock(MetricReporter.class); - final MetricUpdater metricUpdater = new MetricUpdater(mockReporter, 0); + final MetricUpdater metricUpdater = new MetricUpdater(mockReporter, 0, CLUSTER_NAME); final ClusterFixture clusterFixture; Fixture() { @@ -42,8 +44,8 @@ public class MetricReporterTest { private static HasMetricContext.Dimension[] withClusterDimension() { // Dimensions that are always present HasMetricContext.Dimension controllerDim = withDimension("controller-index", "0"); - HasMetricContext.Dimension clusterDim = withDimension("cluster", "foo"); - HasMetricContext.Dimension clusteridDim = withDimension("clusterid", "foo"); + HasMetricContext.Dimension clusterDim = withDimension("cluster", CLUSTER_NAME); + HasMetricContext.Dimension clusteridDim = withDimension("clusterid", CLUSTER_NAME); return new HasMetricContext.Dimension[] { controllerDim, clusterDim, clusteridDim }; } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateChangeTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateChangeTest.java index af39e43bb36..2b71f909115 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateChangeTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateChangeTest.java @@ -50,7 +50,7 @@ public class StateChangeTest extends FleetControllerTest { } communicator = new DummyCommunicator(nodes, timer); - MetricUpdater metricUpdater = new MetricUpdater(new NoMetricReporter(), options.fleetControllerIndex); + MetricUpdater metricUpdater = new MetricUpdater(new NoMetricReporter(), options.fleetControllerIndex, options.clusterName); eventLog = new EventLog(timer, metricUpdater); ContentCluster cluster = new ContentCluster(options.clusterName, options.nodes, options.storageDistribution); NodeStateGatherer stateGatherer = new NodeStateGatherer(timer, timer, eventLog); diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 0a96b3c1030..f15d700a398 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -77,7 +77,7 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"baldersheim"}) default boolean skipCommunicationManagerThread() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default boolean skipMbusRequestThread() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default boolean skipMbusReplyThread() { throw new UnsupportedOperationException("TODO specify default value"); } - @ModelFeatureFlag(owners = {"tokle"}) default boolean useAccessControlTlsHandshakeClientAuth() { return false; } + @ModelFeatureFlag(owners = {"tokle"}) default boolean useAccessControlTlsHandshakeClientAuth() { return true; } @ModelFeatureFlag(owners = {"baldersheim"}) default boolean useAsyncMessageHandlingOnSchedule() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default double feedConcurrency() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default boolean useBucketExecutorForLidSpaceCompact() { throw new UnsupportedOperationException("TODO specify default value"); } diff --git a/config-model/pom.xml b/config-model/pom.xml index daedc7b4a96..0059d8067e4 100644 --- a/config-model/pom.xml +++ b/config-model/pom.xml @@ -207,12 +207,6 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> - <artifactId>messagebus-disc</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> <artifactId>container-messagebus</artifactId> <version>${project.version}</version> <scope>provided</scope> diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java index da25680ca47..ae06d34dfb8 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java @@ -161,7 +161,7 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer { addUriIndexCommands(field); } - if (field.getDataType() instanceof NumericDataType) { + if (field.getDataType().getPrimitiveType() instanceof NumericDataType) { addIndexCommand(field, CMD_NUMERICAL); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValues.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValues.java index 72777b7dfb4..7bc12dec331 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValues.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValues.java @@ -51,9 +51,9 @@ public class IndexingValues extends Processor { protected boolean shouldConvert(Expression exp) { if (exp instanceof OutputExpression && mutatedBy != null) { throw newProcessException(search, field, - "Indexing expression '" + mutatedBy + "' modifies the value of the " + - "document field '" + field.getName() + "'. This is no longer supported -- " + - "declare such fields outside the document."); + "Indexing expression '" + mutatedBy + "' attempts to modify the value of the " + + "document field '" + field.getName() + "'. Use a field outside the document " + + "block instead."); } if (exp instanceof InputExpression && ((InputExpression)exp).getFieldName().equals(field.getName())) { mutatedBy = null; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java index 29ed27675ca..986fd0b77ba 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java @@ -139,6 +139,7 @@ public class MetricsProxyContainer extends Container implements getHostResource().spec().membership().map(ClusterMembership::cluster).ifPresent(cluster -> { dimensions.put(PublicDimensions.INTERNAL_CLUSTER_TYPE, cluster.type().name()); dimensions.put(PublicDimensions.INTERNAL_CLUSTER_ID, cluster.id().value()); + cluster.group().ifPresent(group -> dimensions.put(PublicDimensions.GROUP_ID, group.toString())); }); builder.dimensions(dimensions); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index f3519f3ba0b..7929dc1e93f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -54,6 +54,7 @@ public class VespaMetricSet { metrics.add(new Metric("slobrok.heartbeats.failed.count")); metrics.add(new Metric("logd.processed.lines.count")); + metrics.add(new Metric("worker.connections.max")); // Java (JRT) TLS metrics metrics.add(new Metric("jrt.transport.tls-certificate-verification-failures")); @@ -240,6 +241,7 @@ public class VespaMetricSet { metrics.add(new Metric("cluster-controller.cluster-state-change.count")); metrics.add(new Metric("cluster-controller.is-master.last")); + metrics.add(new Metric("cluster-controller.remote-task-queue.size.last")); // TODO(hakonhall): Update this name once persistent "count" metrics has been implemented. // DO NOT RELY ON THIS METRIC YET. metrics.add(new Metric("cluster-controller.node-event.count")); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java index 06e02821544..30ebb843aa7 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java @@ -28,7 +28,6 @@ public class HostedSslConnectorFactory extends ConnectorFactory { /** * Create connector factory that uses a certificate provided by the config-model / configserver and default hosted Vespa truststore. */ - // TODO Enforce client authentication public static HostedSslConnectorFactory withProvidedCertificate( String serverName, EndpointCertificateSecrets endpointCertificateSecrets, boolean enforceHandshakeClientAuth) { return new HostedSslConnectorFactory(createConfiguredDirectSslProvider(serverName, endpointCertificateSecrets, DEFAULT_HOSTED_TRUSTSTORE, /*tlsCaCertificates*/null, enforceHandshakeClientAuth), false, enforceHandshakeClientAuth); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index d03246b4e8c..d713ae2e016 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -435,8 +435,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { } EndpointCertificateSecrets endpointCertificateSecrets = deployState.endpointCertificateSecrets().get(); - boolean enforceHandshakeClientAuth = context.properties().featureFlags().useAccessControlTlsHandshakeClientAuth() && - cluster.getHttp().getAccessControl() + boolean enforceHandshakeClientAuth = cluster.getHttp().getAccessControl() .map(accessControl -> accessControl.clientAuthentication) .map(clientAuth -> clientAuth.equals(AccessControl.ClientAuthentication.need)) .orElse(false); diff --git a/config-model/src/test/derived/advanced/attributes.cfg b/config-model/src/test/derived/advanced/attributes.cfg index 641e2d8fde5..760d39b8f55 100644 --- a/config-model/src/test/derived/advanced/attributes.cfg +++ b/config-model/src/test/derived/advanced/attributes.cfg @@ -3,6 +3,7 @@ attribute[].datatype INT64 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch true diff --git a/config-model/src/test/derived/array_of_struct_attribute/attributes.cfg b/config-model/src/test/derived/array_of_struct_attribute/attributes.cfg index bb4ba665406..951cd8e4157 100644 --- a/config-model/src/test/derived/array_of_struct_attribute/attributes.cfg +++ b/config-model/src/test/derived/array_of_struct_attribute/attributes.cfg @@ -3,6 +3,7 @@ attribute[].datatype STRING attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch true @@ -32,6 +33,7 @@ attribute[].datatype INT32 attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-model/src/test/derived/arrays/index-info.cfg b/config-model/src/test/derived/arrays/index-info.cfg index c51c927071d..64bcfcc59c0 100644 --- a/config-model/src/test/derived/arrays/index-info.cfg +++ b/config-model/src/test/derived/arrays/index-info.cfg @@ -24,6 +24,8 @@ indexinfo[].command[].command "multivalue" indexinfo[].command[].indexname "ratings" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "ratings" +indexinfo[].command[].command "numerical" +indexinfo[].command[].indexname "ratings" indexinfo[].command[].command "type Array<int>" indexinfo[].command[].indexname "a" indexinfo[].command[].command "index" diff --git a/config-model/src/test/derived/attributeprefetch/attributes.cfg b/config-model/src/test/derived/attributeprefetch/attributes.cfg index d05d2d1d5e0..1700cc5ff32 100644 --- a/config-model/src/test/derived/attributeprefetch/attributes.cfg +++ b/config-model/src/test/derived/attributeprefetch/attributes.cfg @@ -3,6 +3,7 @@ attribute[].datatype INT8 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -32,6 +33,7 @@ attribute[].datatype INT8 attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -61,6 +63,7 @@ attribute[].datatype INT8 attribute[].collectiontype WEIGHTEDSET attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -90,6 +93,7 @@ attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -119,6 +123,7 @@ attribute[].datatype INT32 attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -148,6 +153,7 @@ attribute[].datatype INT32 attribute[].collectiontype WEIGHTEDSET attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -177,6 +183,7 @@ attribute[].datatype INT64 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -206,6 +213,7 @@ attribute[].datatype INT64 attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -235,6 +243,7 @@ attribute[].datatype INT64 attribute[].collectiontype WEIGHTEDSET attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -264,6 +273,7 @@ attribute[].datatype FLOAT attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -293,6 +303,7 @@ attribute[].datatype FLOAT attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -322,6 +333,7 @@ attribute[].datatype FLOAT attribute[].collectiontype WEIGHTEDSET attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -351,6 +363,7 @@ attribute[].datatype DOUBLE attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -380,6 +393,7 @@ attribute[].datatype DOUBLE attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -409,6 +423,7 @@ attribute[].datatype DOUBLE attribute[].collectiontype WEIGHTEDSET attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -438,6 +453,7 @@ attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -467,6 +483,7 @@ attribute[].datatype STRING attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -496,6 +513,7 @@ attribute[].datatype STRING attribute[].collectiontype WEIGHTEDSET attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-model/src/test/derived/attributeprefetch/index-info.cfg b/config-model/src/test/derived/attributeprefetch/index-info.cfg index dcf49f787ab..dfce12f9b9f 100644 --- a/config-model/src/test/derived/attributeprefetch/index-info.cfg +++ b/config-model/src/test/derived/attributeprefetch/index-info.cfg @@ -18,6 +18,8 @@ indexinfo[].command[].command "multivalue" indexinfo[].command[].indexname "multibyte" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "multibyte" +indexinfo[].command[].command "numerical" +indexinfo[].command[].indexname "multibyte" indexinfo[].command[].command "type Array<byte>" indexinfo[].command[].indexname "wsbyte" indexinfo[].command[].command "index" @@ -26,6 +28,8 @@ indexinfo[].command[].command "multivalue" indexinfo[].command[].indexname "wsbyte" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "wsbyte" +indexinfo[].command[].command "numerical" +indexinfo[].command[].indexname "wsbyte" indexinfo[].command[].command "type WeightedSet<byte>" indexinfo[].command[].indexname "singleint" indexinfo[].command[].command "index" @@ -42,6 +46,8 @@ indexinfo[].command[].command "multivalue" indexinfo[].command[].indexname "multiint" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "multiint" +indexinfo[].command[].command "numerical" +indexinfo[].command[].indexname "multiint" indexinfo[].command[].command "type Array<int>" indexinfo[].command[].indexname "wsint" indexinfo[].command[].command "index" @@ -50,6 +56,8 @@ indexinfo[].command[].command "multivalue" indexinfo[].command[].indexname "wsint" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "wsint" +indexinfo[].command[].command "numerical" +indexinfo[].command[].indexname "wsint" indexinfo[].command[].command "type WeightedSet<int>" indexinfo[].command[].indexname "singlelong" indexinfo[].command[].command "index" @@ -66,6 +74,8 @@ indexinfo[].command[].command "multivalue" indexinfo[].command[].indexname "multilong" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "multilong" +indexinfo[].command[].command "numerical" +indexinfo[].command[].indexname "multilong" indexinfo[].command[].command "type Array<long>" indexinfo[].command[].indexname "wslong" indexinfo[].command[].command "index" @@ -74,6 +84,8 @@ indexinfo[].command[].command "multivalue" indexinfo[].command[].indexname "wslong" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "wslong" +indexinfo[].command[].command "numerical" +indexinfo[].command[].indexname "wslong" indexinfo[].command[].command "type WeightedSet<long>" indexinfo[].command[].indexname "singlefloat" indexinfo[].command[].command "index" @@ -90,6 +102,8 @@ indexinfo[].command[].command "multivalue" indexinfo[].command[].indexname "multifloat" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "multifloat" +indexinfo[].command[].command "numerical" +indexinfo[].command[].indexname "multifloat" indexinfo[].command[].command "type Array<float>" indexinfo[].command[].indexname "wsfloat" indexinfo[].command[].command "index" @@ -98,6 +112,8 @@ indexinfo[].command[].command "multivalue" indexinfo[].command[].indexname "wsfloat" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "wsfloat" +indexinfo[].command[].command "numerical" +indexinfo[].command[].indexname "wsfloat" indexinfo[].command[].command "type WeightedSet<float>" indexinfo[].command[].indexname "singledouble" indexinfo[].command[].command "index" @@ -114,6 +130,8 @@ indexinfo[].command[].command "multivalue" indexinfo[].command[].indexname "multidouble" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "multidouble" +indexinfo[].command[].command "numerical" +indexinfo[].command[].indexname "multidouble" indexinfo[].command[].command "type Array<double>" indexinfo[].command[].indexname "wsdouble" indexinfo[].command[].command "index" @@ -122,6 +140,8 @@ indexinfo[].command[].command "multivalue" indexinfo[].command[].indexname "wsdouble" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "wsdouble" +indexinfo[].command[].command "numerical" +indexinfo[].command[].indexname "wsdouble" indexinfo[].command[].command "type WeightedSet<double>" indexinfo[].command[].indexname "singlestring" indexinfo[].command[].command "index" diff --git a/config-model/src/test/derived/attributes/attributes.cfg b/config-model/src/test/derived/attributes/attributes.cfg index e9f3f68adc1..632ab97bbfe 100644 --- a/config-model/src/test/derived/attributes/attributes.cfg +++ b/config-model/src/test/derived/attributes/attributes.cfg @@ -3,6 +3,7 @@ attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -32,6 +33,7 @@ attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -61,6 +63,7 @@ attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -90,6 +93,7 @@ attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -119,6 +123,7 @@ attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -148,6 +153,7 @@ attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -177,6 +183,7 @@ attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -206,6 +213,7 @@ attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -235,6 +243,7 @@ attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -264,6 +273,7 @@ attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -293,6 +303,7 @@ attribute[].datatype INT64 attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -322,6 +333,7 @@ attribute[].datatype DOUBLE attribute[].collectiontype WEIGHTEDSET attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -351,6 +363,7 @@ attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -380,6 +393,7 @@ attribute[].datatype INT32 attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch true @@ -409,6 +423,7 @@ attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -438,6 +453,7 @@ attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -467,6 +483,7 @@ attribute[].datatype STRING attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -496,6 +513,7 @@ attribute[].datatype STRING attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-model/src/test/derived/attributes/index-info.cfg b/config-model/src/test/derived/attributes/index-info.cfg index aacd1baa060..77a52fa47ba 100644 --- a/config-model/src/test/derived/attributes/index-info.cfg +++ b/config-model/src/test/derived/attributes/index-info.cfg @@ -112,6 +112,8 @@ indexinfo[].command[].command "multivalue" indexinfo[].command[].indexname "b6" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "b6" +indexinfo[].command[].command "numerical" +indexinfo[].command[].indexname "b6" indexinfo[].command[].command "type Array<long>" indexinfo[].command[].indexname "b7" indexinfo[].command[].command "index" @@ -120,6 +122,8 @@ indexinfo[].command[].command "multivalue" indexinfo[].command[].indexname "b7" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "b7" +indexinfo[].command[].command "numerical" +indexinfo[].command[].indexname "b7" indexinfo[].command[].command "type WeightedSet<double>" indexinfo[].command[].indexname "a9" indexinfo[].command[].command "index" @@ -138,6 +142,8 @@ indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "a10" indexinfo[].command[].command "fast-search" indexinfo[].command[].indexname "a10" +indexinfo[].command[].command "numerical" +indexinfo[].command[].indexname "a10" indexinfo[].command[].command "type Array<int>" indexinfo[].command[].indexname "a11" indexinfo[].command[].command "index" diff --git a/config-model/src/test/derived/complex/attributes.cfg b/config-model/src/test/derived/complex/attributes.cfg index 622fb9f349c..8bfc14e6795 100644 --- a/config-model/src/test/derived/complex/attributes.cfg +++ b/config-model/src/test/derived/complex/attributes.cfg @@ -3,6 +3,7 @@ attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -32,6 +33,7 @@ attribute[].datatype FLOAT attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -61,6 +63,7 @@ attribute[].datatype FLOAT attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -90,6 +93,7 @@ attribute[].datatype INT64 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -119,6 +123,7 @@ attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -148,6 +153,7 @@ attribute[].datatype INT64 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -177,6 +183,7 @@ attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -206,6 +213,7 @@ attribute[].datatype INT32 attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -235,6 +243,7 @@ attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-model/src/test/derived/hnsw_index/attributes.cfg b/config-model/src/test/derived/hnsw_index/attributes.cfg index 4d275787bfd..188ebed6bc2 100644 --- a/config-model/src/test/derived/hnsw_index/attributes.cfg +++ b/config-model/src/test/derived/hnsw_index/attributes.cfg @@ -3,6 +3,7 @@ attribute[].datatype TENSOR attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -32,6 +33,7 @@ attribute[].datatype TENSOR attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-model/src/test/derived/imported_fields_inherited_reference/attributes.cfg b/config-model/src/test/derived/imported_fields_inherited_reference/attributes.cfg index bfdf90ac12c..f2a18de22bf 100644 --- a/config-model/src/test/derived/imported_fields_inherited_reference/attributes.cfg +++ b/config-model/src/test/derived/imported_fields_inherited_reference/attributes.cfg @@ -3,6 +3,7 @@ attribute[].datatype REFERENCE attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -32,6 +33,7 @@ attribute[].datatype REFERENCE attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -61,6 +63,7 @@ attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -90,6 +93,7 @@ attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-model/src/test/derived/imported_position_field/attributes.cfg b/config-model/src/test/derived/imported_position_field/attributes.cfg index f1b20c6e454..0433c41fced 100644 --- a/config-model/src/test/derived/imported_position_field/attributes.cfg +++ b/config-model/src/test/derived/imported_position_field/attributes.cfg @@ -3,6 +3,7 @@ attribute[].datatype REFERENCE attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -32,6 +33,7 @@ attribute[].datatype INT64 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch true diff --git a/config-model/src/test/derived/imported_struct_fields/attributes.cfg b/config-model/src/test/derived/imported_struct_fields/attributes.cfg index 9e0b5f18170..894683e6389 100644 --- a/config-model/src/test/derived/imported_struct_fields/attributes.cfg +++ b/config-model/src/test/derived/imported_struct_fields/attributes.cfg @@ -3,6 +3,7 @@ attribute[].datatype REFERENCE attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -32,6 +33,7 @@ attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch true @@ -61,6 +63,7 @@ attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -90,6 +93,7 @@ attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch true @@ -119,6 +123,7 @@ attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch true @@ -148,6 +153,7 @@ attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -177,6 +183,7 @@ attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch true @@ -206,6 +213,7 @@ attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-model/src/test/derived/importedfields/attributes.cfg b/config-model/src/test/derived/importedfields/attributes.cfg index 68cd917336f..16ddd67ad6d 100644 --- a/config-model/src/test/derived/importedfields/attributes.cfg +++ b/config-model/src/test/derived/importedfields/attributes.cfg @@ -3,6 +3,7 @@ attribute[].datatype REFERENCE attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -32,6 +33,7 @@ attribute[].datatype REFERENCE attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -61,6 +63,7 @@ attribute[].datatype REFERENCE attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -90,6 +93,7 @@ attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -119,6 +123,7 @@ attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -148,6 +153,7 @@ attribute[].datatype INT32 attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -177,6 +183,7 @@ attribute[].datatype INT32 attribute[].collectiontype WEIGHTEDSET attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -206,6 +213,7 @@ attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-model/src/test/derived/importedfields/index-info.cfg b/config-model/src/test/derived/importedfields/index-info.cfg index 76382e4fd80..ec44e4f1b11 100644 --- a/config-model/src/test/derived/importedfields/index-info.cfg +++ b/config-model/src/test/derived/importedfields/index-info.cfg @@ -50,6 +50,8 @@ indexinfo[].command[].command "multivalue" indexinfo[].command[].indexname "my_int_array_field" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "my_int_array_field" +indexinfo[].command[].command "numerical" +indexinfo[].command[].indexname "my_int_array_field" indexinfo[].command[].command "type Array<int>" indexinfo[].command[].indexname "my_int_wset_field" indexinfo[].command[].command "index" @@ -58,6 +60,8 @@ indexinfo[].command[].command "multivalue" indexinfo[].command[].indexname "my_int_wset_field" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "my_int_wset_field" +indexinfo[].command[].command "numerical" +indexinfo[].command[].indexname "my_int_wset_field" indexinfo[].command[].command "type WeightedSet<int>" indexinfo[].command[].indexname "my_ancient_int_field" indexinfo[].command[].command "index" diff --git a/config-model/src/test/derived/inheritance/attributes.cfg b/config-model/src/test/derived/inheritance/attributes.cfg index a931af7af4d..900a93efe23 100644 --- a/config-model/src/test/derived/inheritance/attributes.cfg +++ b/config-model/src/test/derived/inheritance/attributes.cfg @@ -3,6 +3,7 @@ attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -32,6 +33,7 @@ attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -61,6 +63,7 @@ attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-model/src/test/derived/inheritfromparent/attributes.cfg b/config-model/src/test/derived/inheritfromparent/attributes.cfg index 11498de54b1..93dfc9b3234 100644 --- a/config-model/src/test/derived/inheritfromparent/attributes.cfg +++ b/config-model/src/test/derived/inheritfromparent/attributes.cfg @@ -3,6 +3,7 @@ attribute[].datatype FLOAT attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-model/src/test/derived/lowercase/ilscripts.cfg b/config-model/src/test/derived/lowercase/ilscripts.cfg new file mode 100644 index 00000000000..0cf56eb43e0 --- /dev/null +++ b/config-model/src/test/derived/lowercase/ilscripts.cfg @@ -0,0 +1,9 @@ +maxtermoccurrences 100 +fieldmatchmaxlength 1000000 +ilscript[].doctype "lowercase" +ilscript[].docfield[] "single_field_source" +ilscript[].docfield[] "array_field_source" +ilscript[].content[] "clear_state | guard { input array_field_source | for_each { lowercase } | for_each { tokenize normalize stem:\"BEST\" } | summary array_field | index array_field; }" +ilscript[].content[] "clear_state | guard { input single_field_source | lowercase | tokenize normalize stem:\"BEST\" | summary single_field | index single_field; }" +ilscript[].content[] "input array_field_source | passthrough array_field_source" +ilscript[].content[] "input single_field_source | passthrough single_field_source"
\ No newline at end of file diff --git a/config-model/src/test/derived/lowercase/index-info.cfg b/config-model/src/test/derived/lowercase/index-info.cfg new file mode 100644 index 00000000000..b94125de79d --- /dev/null +++ b/config-model/src/test/derived/lowercase/index-info.cfg @@ -0,0 +1,41 @@ +indexinfo[].name "lowercase" +indexinfo[].command[].indexname "sddocname" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "sddocname" +indexinfo[].command[].command "word" +indexinfo[].command[].indexname "single_field_source" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "single_field_source" +indexinfo[].command[].command "type string" +indexinfo[].command[].indexname "array_field_source" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "array_field_source" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "array_field_source" +indexinfo[].command[].command "type Array<string>" +indexinfo[].command[].indexname "array_field" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "array_field" +indexinfo[].command[].command "lowercase" +indexinfo[].command[].indexname "array_field" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "array_field" +indexinfo[].command[].command "stem:BEST" +indexinfo[].command[].indexname "array_field" +indexinfo[].command[].command "normalize" +indexinfo[].command[].indexname "array_field" +indexinfo[].command[].command "plain-tokens" +indexinfo[].command[].indexname "array_field" +indexinfo[].command[].command "type Array<string>" +indexinfo[].command[].indexname "single_field" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "single_field" +indexinfo[].command[].command "lowercase" +indexinfo[].command[].indexname "single_field" +indexinfo[].command[].command "stem:BEST" +indexinfo[].command[].indexname "single_field" +indexinfo[].command[].command "normalize" +indexinfo[].command[].indexname "single_field" +indexinfo[].command[].command "plain-tokens" +indexinfo[].command[].indexname "single_field" +indexinfo[].command[].command "type string"
\ No newline at end of file diff --git a/config-model/src/test/derived/lowercase/lowercase.sd b/config-model/src/test/derived/lowercase/lowercase.sd new file mode 100644 index 00000000000..de8c7ad729e --- /dev/null +++ b/config-model/src/test/derived/lowercase/lowercase.sd @@ -0,0 +1,21 @@ +schema lowercase { + + document lowercase { + + field single_field_source type string { + } + + field array_field_source type array<string> { + } + + } + + field single_field type string { + indexing: input single_field_source | lowercase | summary | index + } + + field array_field type array<string> { + indexing: input array_field_source | for_each { lowercase } | summary | index + } + +}
\ No newline at end of file diff --git a/config-model/src/test/derived/map_attribute/attributes.cfg b/config-model/src/test/derived/map_attribute/attributes.cfg index acbdf119d0d..35b9a1722a9 100644 --- a/config-model/src/test/derived/map_attribute/attributes.cfg +++ b/config-model/src/test/derived/map_attribute/attributes.cfg @@ -3,6 +3,7 @@ attribute[].datatype STRING attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch true @@ -32,6 +33,7 @@ attribute[].datatype STRING attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -61,6 +63,7 @@ attribute[].datatype INT32 attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-model/src/test/derived/map_of_struct_attribute/attributes.cfg b/config-model/src/test/derived/map_of_struct_attribute/attributes.cfg index ecc8c2fd69d..e9471d91c45 100644 --- a/config-model/src/test/derived/map_of_struct_attribute/attributes.cfg +++ b/config-model/src/test/derived/map_of_struct_attribute/attributes.cfg @@ -3,6 +3,7 @@ attribute[].datatype STRING attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch true @@ -32,6 +33,7 @@ attribute[].datatype STRING attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -61,6 +63,7 @@ attribute[].datatype INT32 attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -90,6 +93,7 @@ attribute[].datatype INT32 attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -119,6 +123,7 @@ attribute[].datatype STRING attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch true diff --git a/config-model/src/test/derived/music/attributes.cfg b/config-model/src/test/derived/music/attributes.cfg index a31325e67d0..e1685f55357 100644 --- a/config-model/src/test/derived/music/attributes.cfg +++ b/config-model/src/test/derived/music/attributes.cfg @@ -3,6 +3,7 @@ attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -32,6 +33,7 @@ attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -61,6 +63,7 @@ attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -90,6 +93,7 @@ attribute[].datatype FLOAT attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -119,6 +123,7 @@ attribute[].datatype FLOAT attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -148,6 +153,7 @@ attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -177,6 +183,7 @@ attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -206,6 +213,7 @@ attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -235,6 +243,7 @@ attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -264,6 +273,7 @@ attribute[].datatype STRING attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -293,6 +303,7 @@ attribute[].datatype STRING attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-model/src/test/derived/newrank/attributes.cfg b/config-model/src/test/derived/newrank/attributes.cfg index 68f24871f02..0b3c39c44f6 100644 --- a/config-model/src/test/derived/newrank/attributes.cfg +++ b/config-model/src/test/derived/newrank/attributes.cfg @@ -3,6 +3,7 @@ attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -32,6 +33,7 @@ attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -61,6 +63,7 @@ attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -90,6 +93,7 @@ attribute[].datatype FLOAT attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -119,6 +123,7 @@ attribute[].datatype FLOAT attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -148,6 +153,7 @@ attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -177,6 +183,7 @@ attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -206,6 +213,7 @@ attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -235,6 +243,7 @@ attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -264,6 +273,7 @@ attribute[].datatype INT32 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-model/src/test/derived/position_array/index-info.cfg b/config-model/src/test/derived/position_array/index-info.cfg index c8008e9d440..efbea184232 100644 --- a/config-model/src/test/derived/position_array/index-info.cfg +++ b/config-model/src/test/derived/position_array/index-info.cfg @@ -32,4 +32,6 @@ indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "pos_zcurve" indexinfo[].command[].command "fast-search" indexinfo[].command[].indexname "pos_zcurve" +indexinfo[].command[].command "numerical" +indexinfo[].command[].indexname "pos_zcurve" indexinfo[].command[].command "type Array<long>" diff --git a/config-model/src/test/derived/predicate_attribute/attributes.cfg b/config-model/src/test/derived/predicate_attribute/attributes.cfg index ff45cf1a41e..52ad169550d 100644 --- a/config-model/src/test/derived/predicate_attribute/attributes.cfg +++ b/config-model/src/test/derived/predicate_attribute/attributes.cfg @@ -3,6 +3,7 @@ attribute[].datatype PREDICATE attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-model/src/test/derived/prefixexactattribute/attributes.cfg b/config-model/src/test/derived/prefixexactattribute/attributes.cfg index f9878068372..a0657f5bc27 100644 --- a/config-model/src/test/derived/prefixexactattribute/attributes.cfg +++ b/config-model/src/test/derived/prefixexactattribute/attributes.cfg @@ -3,6 +3,7 @@ attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -32,6 +33,7 @@ attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-model/src/test/derived/reference_fields/attributes.cfg b/config-model/src/test/derived/reference_fields/attributes.cfg index ad3ff5e62f8..1517a96950c 100644 --- a/config-model/src/test/derived/reference_fields/attributes.cfg +++ b/config-model/src/test/derived/reference_fields/attributes.cfg @@ -3,6 +3,7 @@ attribute[].datatype REFERENCE attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -32,6 +33,7 @@ attribute[].datatype REFERENCE attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -61,6 +63,7 @@ attribute[].datatype REFERENCE attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-model/src/test/derived/sorting/attributes.cfg b/config-model/src/test/derived/sorting/attributes.cfg index ebe0e83540e..e878c1f054f 100644 --- a/config-model/src/test/derived/sorting/attributes.cfg +++ b/config-model/src/test/derived/sorting/attributes.cfg @@ -3,6 +3,7 @@ attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -32,6 +33,7 @@ attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -61,6 +63,7 @@ attribute[].datatype STRING attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-model/src/test/derived/tensor/attributes.cfg b/config-model/src/test/derived/tensor/attributes.cfg index 398cdf5f8f8..71789c2987c 100644 --- a/config-model/src/test/derived/tensor/attributes.cfg +++ b/config-model/src/test/derived/tensor/attributes.cfg @@ -3,6 +3,7 @@ attribute[].datatype TENSOR attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -32,6 +33,7 @@ attribute[].datatype TENSOR attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -61,6 +63,7 @@ attribute[].datatype TENSOR attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -90,6 +93,7 @@ attribute[].datatype TENSOR attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -119,6 +123,7 @@ attribute[].datatype FLOAT attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-model/src/test/derived/types/attributes.cfg b/config-model/src/test/derived/types/attributes.cfg index 05e19a15d96..e48d4d1bd09 100644 --- a/config-model/src/test/derived/types/attributes.cfg +++ b/config-model/src/test/derived/types/attributes.cfg @@ -3,6 +3,7 @@ attribute[].datatype INT8 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -32,6 +33,7 @@ attribute[].datatype INT64 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -61,6 +63,7 @@ attribute[].datatype BOOL attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -90,6 +93,7 @@ attribute[].datatype FLOAT16 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -119,6 +123,7 @@ attribute[].datatype INT32 attribute[].collectiontype ARRAY attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -148,6 +153,7 @@ attribute[].datatype STRING attribute[].collectiontype WEIGHTEDSET attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false @@ -177,6 +183,7 @@ attribute[].datatype STRING attribute[].collectiontype WEIGHTEDSET attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero true attribute[].createifnonexistent true attribute[].fastsearch false @@ -206,6 +213,7 @@ attribute[].datatype STRING attribute[].collectiontype WEIGHTEDSET attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero true attribute[].createifnonexistent false attribute[].fastsearch false @@ -235,6 +243,7 @@ attribute[].datatype STRING attribute[].collectiontype WEIGHTEDSET attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent true attribute[].fastsearch false @@ -264,6 +273,7 @@ attribute[].datatype STRING attribute[].collectiontype WEIGHTEDSET attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero true attribute[].createifnonexistent true attribute[].fastsearch false @@ -293,6 +303,7 @@ attribute[].datatype INT64 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch true @@ -322,6 +333,7 @@ attribute[].datatype STRING attribute[].collectiontype WEIGHTEDSET attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero true attribute[].createifnonexistent true attribute[].fastsearch false @@ -351,6 +363,7 @@ attribute[].datatype INT64 attribute[].collectiontype SINGLE attribute[].dictionary.ordering ORDERED attribute[].dictionary.type BTREE +attribute[].dictionary.match CASE_INSENSITIVE attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false diff --git a/config-model/src/test/derived/types/index-info.cfg b/config-model/src/test/derived/types/index-info.cfg index cb10bb561d5..3bcf43060fc 100644 --- a/config-model/src/test/derived/types/index-info.cfg +++ b/config-model/src/test/derived/types/index-info.cfg @@ -42,6 +42,8 @@ indexinfo[].command[].command "multivalue" indexinfo[].command[].indexname "arrayfield" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "arrayfield" +indexinfo[].command[].command "numerical" +indexinfo[].command[].indexname "arrayfield" indexinfo[].command[].command "type Array<int>" indexinfo[].command[].indexname "setfield" indexinfo[].command[].command "index" @@ -222,6 +224,8 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "arraymapfield.value" indexinfo[].command[].command "multivalue" indexinfo[].command[].indexname "arraymapfield.value" +indexinfo[].command[].command "numerical" +indexinfo[].command[].indexname "arraymapfield.value" indexinfo[].command[].command "type Array<int>" indexinfo[].command[].indexname "arraymapfield" indexinfo[].command[].command "index" @@ -252,6 +256,8 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "mystructfield.bytearr" indexinfo[].command[].command "multivalue" indexinfo[].command[].indexname "mystructfield.bytearr" +indexinfo[].command[].command "numerical" +indexinfo[].command[].indexname "mystructfield.bytearr" indexinfo[].command[].command "type Array<byte>" indexinfo[].command[].indexname "mystructfield.mymap.key" indexinfo[].command[].command "index" @@ -290,6 +296,8 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "mystructmap.value.bytearr" indexinfo[].command[].command "multivalue" indexinfo[].command[].indexname "mystructmap.value.bytearr" +indexinfo[].command[].command "numerical" +indexinfo[].command[].indexname "mystructmap.value.bytearr" indexinfo[].command[].command "type Array<byte>" indexinfo[].command[].indexname "mystructmap.value.mymap.key" indexinfo[].command[].command "index" @@ -328,6 +336,8 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "mystructarr.bytearr" indexinfo[].command[].command "multivalue" indexinfo[].command[].indexname "mystructarr.bytearr" +indexinfo[].command[].command "numerical" +indexinfo[].command[].indexname "mystructarr.bytearr" indexinfo[].command[].command "type Array<byte>" indexinfo[].command[].indexname "mystructarr.mymap.key" indexinfo[].command[].command "index" diff --git a/config-model/src/test/examples/indexrewrite.sd b/config-model/src/test/examples/indexrewrite.sd index c7d038a719a..c9fa7a1530a 100644 --- a/config-model/src/test/examples/indexrewrite.sd +++ b/config-model/src/test/examples/indexrewrite.sd @@ -1,19 +1,21 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. search indexrewrite { + document indexrewrite { field title_src type string { - } } + field title type string { indexing: input title_src | lowercase | normalize | tokenize | index - # index-to: title, titleabstract, default indexing-rewrite: none rank-type: about stemming: none alias: headline } + field title_s type string { indexing: input title_src | summary } + } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/LowercaseTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/LowercaseTestCase.java new file mode 100644 index 00000000000..253bb2f4f00 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/LowercaseTestCase.java @@ -0,0 +1,19 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived; + +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +/** + * @author bratseth + */ +public class LowercaseTestCase extends AbstractExportingTestCase { + + @Test + public void testDeriving() throws IOException, ParseException { + assertCorrectDeriving("lowercase"); + } + +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingValuesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingValuesTestCase.java index a1c454da822..823ad556b1d 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingValuesTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingValuesTestCase.java @@ -23,7 +23,8 @@ public class IndexingValuesTestCase { public void requireThatInputOtherFieldThrows() throws IOException, ParseException { assertBuildFails("src/test/examples/indexing_input_other_field.sd", "For search 'indexing_input_other_field', field 'bar': Indexing expression 'input foo' " + - "modifies the value of the document field 'bar'. This is no longer supported -- declare " + - "such fields outside the document."); + "attempts to modify the value of the document field 'bar'. " + + "Use a field outside the document block instead."); } + } diff --git a/config-proxy/src/main/sh/vespa-config-ctl.sh b/config-proxy/src/main/sh/vespa-config-ctl.sh index 2a4905a2da0..b4127cd337e 100755 --- a/config-proxy/src/main/sh/vespa-config-ctl.sh +++ b/config-proxy/src/main/sh/vespa-config-ctl.sh @@ -110,7 +110,7 @@ case $1 in nohup sbin/vespa-retention-enforcer > ${LOGDIR}/vre-start.log 2>&1 </dev/null & configsources=`bin/vespa-print-default configservers_rpc` userargs=$VESPA_CONFIGPROXY_JVMARGS - jvmopts="-Xms32M -Xmx256M -XX:CompressedClassSpaceSize=32m -XX:MaxDirectMemorySize=32m -XX:ThreadStackSize=256 -XX:MaxJavaStackTraceDepth=1000" + jvmopts="-Xms32M -Xmx256M -XX:CompressedClassSpaceSize=32m -XX:MaxDirectMemorySize=32m -XX:ThreadStackSize=256 -XX:MaxJavaStackTraceDepth=1000 -XX:-OmitStackTraceInFastThrow" VESPA_SERVICE_NAME=configproxy export VESPA_SERVICE_NAME diff --git a/configdefinitions/src/vespa/attributes.def b/configdefinitions/src/vespa/attributes.def index 939a0b8476a..f5b5a7cb450 100644 --- a/configdefinitions/src/vespa/attributes.def +++ b/configdefinitions/src/vespa/attributes.def @@ -7,6 +7,7 @@ attribute[].collectiontype enum { SINGLE, ARRAY, WEIGHTEDSET } default=SING # Deprecated/ do-not-use, will soon be GCed. attribute[].dictionary.ordering enum { ORDERED, UNORDERED } default = ORDERED attribute[].dictionary.type enum { BTREE, HASH, BTREE_AND_HASH } default = BTREE +attribute[].dictionary.match enum { CASE_SENSITIVE, CASE_INSENSITIVE } default=CASE_INSENSITIVE attribute[].removeifzero bool default=false attribute[].createifnonexistent bool default=false attribute[].fastsearch bool default=false diff --git a/configdefinitions/src/vespa/lb-services.def b/configdefinitions/src/vespa/lb-services.def index 46438cb6a40..521cadfb457 100644 --- a/configdefinitions/src/vespa/lb-services.def +++ b/configdefinitions/src/vespa/lb-services.def @@ -7,7 +7,7 @@ namespace=cloud.config # Active rotation given as flag 'active' for a prod region in deployment.xml # Default true for now (since code in config-model to set it is not ready yet), should have no default value tenants{}.applications{}.activeRotation bool default=true -tenants{}.applications{}.usePowerOfTwoChoicesLb bool default=false +tenants{}.applications{}.usePowerOfTwoChoicesLb bool default=true tenants{}.applications{}.generateNonMtlsEndpoint bool default=true tenants{}.applications{}.hosts{}.hostname string default="(unknownhostname)" diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java index f77f3112d82..eb179d2d9aa 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java @@ -466,7 +466,7 @@ public class TenantApplications implements RequestHandler, HostValidator<Applica // If all config servers responded, return if (respondents.size() == curator.zooKeeperEnsembleCount()) { - log.log(Level.FINE, barrierCompletedMessage(respondents, startTime)); + logBarrierCompleted(respondents, startTime); break; } @@ -477,7 +477,7 @@ public class TenantApplications implements RequestHandler, HostValidator<Applica // Give up if more than some time has passed since we got quorum, otherwise continue if (Duration.between(Instant.now(), gotQuorumTime.plus(waitForAll)).isNegative()) { - log.log(Level.FINE, barrierCompletedMessage(respondents, startTime)); + logBarrierCompleted(respondents, startTime); break; } } @@ -488,8 +488,14 @@ public class TenantApplications implements RequestHandler, HostValidator<Applica return respondents; } - private String barrierCompletedMessage(List<String> respondents, Instant startTime) { - return barrierPath + " completed in " + Duration.between(startTime, Instant.now()).toString() + + private void logBarrierCompleted(List<String> respondents, Instant startTime) { + Duration duration = Duration.between(startTime, Instant.now()); + Level level = (duration.minus(Duration.ofSeconds(5))).isNegative() ? Level.FINE : Level.INFO; + log.log(level, barrierCompletedMessage(respondents, duration)); + } + + private String barrierCompletedMessage(List<String> respondents, Duration duration) { + return barrierPath + " completed in " + duration.toString() + ", " + respondents.size() + "/" + curator.zooKeeperEnsembleCount() + " responded: " + respondents; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index a1a8548f5f1..b4dd81ad7aa 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -166,7 +166,6 @@ public class ModelContextImpl implements ModelContext { private final boolean skipCommunicationManagerThread; private final boolean skipMbusRequestThread; private final boolean skipMbusReplyThread; - private final boolean useAccessControlTlsHandshakeClientAuth; private final boolean useAsyncMessageHandlingOnSchedule; private final double feedConcurrency; private final boolean useBucketExecutorForLidSpaceCompact; @@ -191,7 +190,6 @@ public class ModelContextImpl implements ModelContext { this.skipCommunicationManagerThread = flagValue(source, appId, Flags.SKIP_COMMUNICATIONMANAGER_THREAD); this.skipMbusRequestThread = flagValue(source, appId, Flags.SKIP_MBUS_REQUEST_THREAD); this.skipMbusReplyThread = flagValue(source, appId, Flags.SKIP_MBUS_REPLY_THREAD); - this.useAccessControlTlsHandshakeClientAuth = flagValue(source, appId, Flags.USE_ACCESS_CONTROL_CLIENT_AUTHENTICATION); this.useAsyncMessageHandlingOnSchedule = flagValue(source, appId, Flags.USE_ASYNC_MESSAGE_HANDLING_ON_SCHEDULE); this.feedConcurrency = flagValue(source, appId, Flags.FEED_CONCURRENCY); this.useBucketExecutorForLidSpaceCompact = flagValue(source, appId, Flags.USE_BUCKET_EXECUTOR_FOR_LID_SPACE_COMPACT); @@ -216,7 +214,6 @@ public class ModelContextImpl implements ModelContext { @Override public boolean skipCommunicationManagerThread() { return skipCommunicationManagerThread; } @Override public boolean skipMbusRequestThread() { return skipMbusRequestThread; } @Override public boolean skipMbusReplyThread() { return skipMbusReplyThread; } - @Override public boolean useAccessControlTlsHandshakeClientAuth() { return useAccessControlTlsHandshakeClientAuth; } @Override public boolean useAsyncMessageHandlingOnSchedule() { return useAsyncMessageHandlingOnSchedule; } @Override public double feedConcurrency() { return feedConcurrency; } @Override public boolean useBucketExecutorForLidSpaceCompact() { return useBucketExecutorForLidSpaceCompact; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java index c195f1a9f96..b2e239c7ebc 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java @@ -35,13 +35,11 @@ public class LbServicesProducer implements LbServicesConfig.Producer { private final Map<TenantName, Set<ApplicationInfo>> models; private final Zone zone; - private final BooleanFlag usePowerOfTwoChoicesLb; private final BooleanFlag generateNonMtlsEndpoint; public LbServicesProducer(Map<TenantName, Set<ApplicationInfo>> models, Zone zone, FlagSource flagSource) { this.models = models; this.zone = zone; - usePowerOfTwoChoicesLb = Flags.USE_POWER_OF_TWO_CHOICES_LOAD_BALANCING.bindTo(flagSource); generateNonMtlsEndpoint = Flags.GENERATE_NON_MTLS_ENDPOINT.bindTo(flagSource); } @@ -74,7 +72,7 @@ public class LbServicesProducer implements LbServicesConfig.Producer { private LbServicesConfig.Tenants.Applications.Builder getAppConfig(ApplicationInfo app) { LbServicesConfig.Tenants.Applications.Builder ab = new LbServicesConfig.Tenants.Applications.Builder(); ab.activeRotation(getActiveRotation(app)); - ab.usePowerOfTwoChoicesLb(usePowerOfTwoChoicesLb(app)); + ab.usePowerOfTwoChoicesLb(true); ab.generateNonMtlsEndpoint(generateNonMtlsEndpoint(app)); app.getModel().getHosts().stream() .sorted((a, b) -> a.getHostname().compareTo(b.getHostname())) @@ -96,10 +94,6 @@ public class LbServicesProducer implements LbServicesConfig.Producer { return activeRotation; } - private boolean usePowerOfTwoChoicesLb(ApplicationInfo app) { - return usePowerOfTwoChoicesLb.with(FetchVector.Dimension.APPLICATION_ID, app.getApplicationId().serializedForm()).value(); - } - private boolean generateNonMtlsEndpoint(ApplicationInfo app) { return generateNonMtlsEndpoint.with(FetchVector.Dimension.APPLICATION_ID, app.getApplicationId().serializedForm()).value(); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/MultiTenantRpcAuthorizer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/MultiTenantRpcAuthorizer.java index 8353e3fab1f..4dfa8384822 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/MultiTenantRpcAuthorizer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/MultiTenantRpcAuthorizer.java @@ -161,7 +161,7 @@ public class MultiTenantRpcAuthorizer implements RpcAuthorizer { JrtErrorCode error = isAuthorizationException ? JrtErrorCode.UNAUTHORIZED : JrtErrorCode.AUTHORIZATION_FAILED; request.setError(error.code, errorMessage); request.returnRequest(); - throwUnchecked(throwable); // rethrow exception to ensure that subsequent completion stages are not executed (don't execute implementation of rpc method). + throw throwUnchecked(throwable); // rethrow exception to ensure that subsequent completion stages are not executed (don't execute implementation of rpc method). } // TODO Make peer identity mandatory once TLS mixed mode is removed diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java index 9fd20453bcd..52d1484897c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java @@ -103,7 +103,8 @@ public class TenantRepository { private final Metrics metrics; private final MetricUpdater metricUpdater; private final ExecutorService zkCacheExecutor; - private final StripedExecutor<TenantName> zkWatcherExecutor; + private final StripedExecutor<TenantName> zkSessionWatcherExecutor; + private final StripedExecutor<TenantName> zkApplicationWatcherExecutor; private final FileDistributionFactory fileDistributionFactory; private final FlagSource flagSource; private final SecretStore secretStore; @@ -141,6 +142,7 @@ public class TenantRepository { configCurator, metrics, new StripedExecutor<>(), + new StripedExecutor<>(), new FileDistributionFactory(configserverConfig), flagSource, Executors.newFixedThreadPool(1, ThreadFactoryFactory.getThreadFactory(TenantRepository.class.getName())), @@ -159,7 +161,8 @@ public class TenantRepository { public TenantRepository(HostRegistry hostRegistry, ConfigCurator configCurator, Metrics metrics, - StripedExecutor<TenantName> zkWatcherExecutor, + StripedExecutor<TenantName> zkApplicationWatcherExecutor , + StripedExecutor<TenantName> zkSessionWatcherExecutor, FileDistributionFactory fileDistributionFactory, FlagSource flagSource, ExecutorService zkCacheExecutor, @@ -181,7 +184,8 @@ public class TenantRepository { this.metrics = metrics; metricUpdater = metrics.getOrCreateMetricUpdater(Collections.emptyMap()); this.zkCacheExecutor = zkCacheExecutor; - this.zkWatcherExecutor = zkWatcherExecutor; + this.zkApplicationWatcherExecutor = zkApplicationWatcherExecutor; + this.zkSessionWatcherExecutor = zkSessionWatcherExecutor; this.fileDistributionFactory = fileDistributionFactory; this.flagSource = flagSource; this.secretStore = secretStore; @@ -318,7 +322,7 @@ public class TenantRepository { TenantApplications applicationRepo = new TenantApplications(tenantName, curator, - zkWatcherExecutor, + zkApplicationWatcherExecutor, zkCacheExecutor, metrics, reloadListener, @@ -342,7 +346,7 @@ public class TenantRepository { sessionPreparer, configCurator, metrics, - zkWatcherExecutor, + zkSessionWatcherExecutor, permanentApplicationPackage, flagSource, zkCacheExecutor, @@ -511,12 +515,12 @@ public class TenantRepository { case CHILD_ADDED: TenantName t1 = getTenantNameFromEvent(event); if ( ! tenants.containsKey(t1)) - zkWatcherExecutor.execute(t1, () -> bootstrapTenant(t1)); + zkApplicationWatcherExecutor.execute(t1, () -> bootstrapTenant(t1)); break; case CHILD_REMOVED: TenantName t2 = getTenantNameFromEvent(event); if (tenants.containsKey(t2)) - zkWatcherExecutor.execute(t2, () -> deleteTenant(t2)); + zkApplicationWatcherExecutor.execute(t2, () -> deleteTenant(t2)); break; default: break; // Nothing to do @@ -537,7 +541,8 @@ public class TenantRepository { try { zkCacheExecutor.shutdown(); checkForRemovedApplicationsService.shutdown(); - zkWatcherExecutor.shutdownAndWait(); + zkApplicationWatcherExecutor.shutdownAndWait(); + zkSessionWatcherExecutor.shutdownAndWait(); zkCacheExecutor.awaitTermination(50, TimeUnit.SECONDS); checkForRemovedApplicationsService.awaitTermination(50, TimeUnit.SECONDS); } diff --git a/configserver/src/main/java/com/yahoo/vespa/serviceview/StateRequestHandler.java b/configserver/src/main/java/com/yahoo/vespa/serviceview/StateRequestHandler.java index 4fbb001b880..63b9def7448 100644 --- a/configserver/src/main/java/com/yahoo/vespa/serviceview/StateRequestHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/serviceview/StateRequestHandler.java @@ -4,9 +4,9 @@ package com.yahoo.vespa.serviceview; import com.google.inject.Inject; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.container.jdisc.LoggingRequestHandler; -import com.yahoo.restapi.JacksonJsonResponse; import com.yahoo.restapi.RestApi; import com.yahoo.restapi.RestApiRequestHandler; +import com.yahoo.restapi.UriBuilder; import com.yahoo.vespa.serviceview.bindings.ApplicationView; import com.yahoo.vespa.serviceview.bindings.ConfigClient; import com.yahoo.vespa.serviceview.bindings.HealthClient; @@ -79,13 +79,13 @@ public class StateRequestHandler extends RestApiRequestHandler<StateRequestHandl .get(self::getUserInfo)) .addRoute(RestApi.route("/serviceview/v1/tenant/{tenantName}/application/{applicationName}/environment/{environmentName}/region/{regionName}/instance/{instanceName}/service/{serviceIdentifier}/{*}") .get(self::singleService)) - .addResponseMapper(HashMap.class, (hashMap, context) -> new JacksonJsonResponse<>(200, hashMap, true)) - .addResponseMapper(ApplicationView.class, (applicationView, context) -> new JacksonJsonResponse<>(200, applicationView, true)) + .registerJacksonResponseEntity(HashMap.class) + .registerJacksonResponseEntity(ApplicationView.class) .build(); } private ApplicationView getDefaultUserInfo(RestApi.RequestContext context) { - return getUserInfo(context.request().getUri(), "default", "default", "default", "default", "default"); + return getUserInfo(context.uriBuilder(), "default", "default", "default", "default", "default"); } private ApplicationView getUserInfo(RestApi.RequestContext context) { @@ -94,7 +94,7 @@ public class StateRequestHandler extends RestApiRequestHandler<StateRequestHandl String environmentName = context.pathParameters().getStringOrThrow("environmentName"); String regionName = context.pathParameters().getStringOrThrow("regionName"); String instanceName = context.pathParameters().getStringOrThrow("instanceName"); - return getUserInfo(context.request().getUri(), tenantName, applicationName, environmentName, regionName, instanceName); + return getUserInfo(context.uriBuilder(), tenantName, applicationName, environmentName, regionName, instanceName); } public HashMap<?, ?> singleService(RestApi.RequestContext context) { @@ -105,14 +105,14 @@ public class StateRequestHandler extends RestApiRequestHandler<StateRequestHandl String instanceName = context.pathParameters().getStringOrThrow("instanceName"); String identifier = context.pathParameters().getStringOrThrow("serviceIdentifier"); String apiParams = context.pathParameters().getString("*").orElse(""); - return singleService(context.request().getUri(), tenantName, applicationName, environmentName, regionName, instanceName, identifier, apiParams); + return singleService(context.uriBuilder(), context.request().getUri(), tenantName, applicationName, environmentName, regionName, instanceName, identifier, apiParams); } - protected ApplicationView getUserInfo(URI requestUri, String tenantName, String applicationName, String environmentName, String regionName, String instanceName) { + protected ApplicationView getUserInfo(UriBuilder uriBuilder, String tenantName, String applicationName, String environmentName, String regionName, String instanceName) { ServiceModel model = new ServiceModel( getModelConfig(tenantName, applicationName, environmentName, regionName, instanceName)); return model.showAllClusters( - baseUri(requestUri).toString(), + baseUri(uriBuilder).toString(), applicationIdentifier(tenantName, applicationName, environmentName, regionName, instanceName)); } @@ -123,13 +123,13 @@ public class StateRequestHandler extends RestApiRequestHandler<StateRequestHandl } protected HashMap<?, ?> singleService( - URI requestUri, String tenantName, String applicationName, String environmentName, String regionName, String instanceName, String identifier, String apiParams) { + UriBuilder uriBuilder, URI requestUri, String tenantName, String applicationName, String environmentName, String regionName, String instanceName, String identifier, String apiParams) { ServiceModel model = new ServiceModel(getModelConfig(tenantName, applicationName, environmentName, regionName, instanceName)); Service s = model.getService(identifier); int requestedPort = s.matchIdentifierWithPort(identifier); HealthClient resource = getHealthClient(apiParams, s, requestedPort, requestUri.getRawQuery(), client); HashMap<?, ?> apiResult = resource.getHealthInfo(); - rewriteResourceLinks(requestUri, apiResult, model, s, applicationIdentifier(tenantName, applicationName, environmentName, regionName, instanceName), identifier); + rewriteResourceLinks(uriBuilder, apiResult, model, s, applicationIdentifier(tenantName, applicationName, environmentName, regionName, instanceName), identifier); return apiResult; } @@ -149,7 +149,7 @@ public class StateRequestHandler extends RestApiRequestHandler<StateRequestHandl + "/instance/" + instance; } - private void rewriteResourceLinks(URI requestUri, + private void rewriteResourceLinks(UriBuilder uriBuilder, Object apiResult, ServiceModel model, Service self, @@ -160,7 +160,7 @@ public class StateRequestHandler extends RestApiRequestHandler<StateRequestHandl Object resource = i.next(); if (resource instanceof String) { try { - StringBuilder buffer = linkBuffer(requestUri, applicationIdentifier); + StringBuilder buffer = linkBuffer(uriBuilder, applicationIdentifier); // if it points to a port and host not part of the application, rewriting will not occur, so this is kind of safe retarget(model, self, buffer, (String) resource); i.set(buffer.toString()); @@ -168,7 +168,7 @@ public class StateRequestHandler extends RestApiRequestHandler<StateRequestHandl break; // assume relatively homogenous lists when doing rewrites to avoid freezing up on scanning long lists } } else { - rewriteResourceLinks(requestUri, resource, model, self, applicationIdentifier, incomingIdentifier); + rewriteResourceLinks(uriBuilder, resource, model, self, applicationIdentifier, incomingIdentifier); } } } else if (apiResult instanceof Map) { @@ -177,14 +177,14 @@ public class StateRequestHandler extends RestApiRequestHandler<StateRequestHandl for (Map.Entry<Object, Object> entry : api.entrySet()) { if (SINGLE_API_LINK.equals(entry.getKey()) && entry.getValue() instanceof String) { try { - rewriteSingleLink(entry, model, self, linkBuffer(requestUri, applicationIdentifier)); + rewriteSingleLink(entry, model, self, linkBuffer(uriBuilder, applicationIdentifier)); } catch (GiveUpLinkRetargetingException e) { // NOP } } else if ("link".equals(entry.getKey()) && entry.getValue() instanceof String) { - buildSingleLink(entry, linkBuffer(requestUri, applicationIdentifier), incomingIdentifier); + buildSingleLink(entry, linkBuffer(uriBuilder, applicationIdentifier), incomingIdentifier); } else { - rewriteResourceLinks(requestUri, entry.getValue(), model, self, applicationIdentifier, incomingIdentifier); + rewriteResourceLinks(uriBuilder, entry.getValue(), model, self, applicationIdentifier, incomingIdentifier); } } } @@ -205,8 +205,8 @@ public class StateRequestHandler extends RestApiRequestHandler<StateRequestHandl } } - private StringBuilder linkBuffer(URI requestUri, String applicationIdentifier) { - return baseUri(requestUri).append(applicationIdentifier); + private StringBuilder linkBuffer(UriBuilder uriBuilder, String applicationIdentifier) { + return baseUri(uriBuilder).append(applicationIdentifier); } private void rewriteSingleLink(Map.Entry<Object, Object> entry, @@ -242,12 +242,7 @@ public class StateRequestHandler extends RestApiRequestHandler<StateRequestHandl newUri.append(link.getRawPath()); } - private static StringBuilder baseUri(URI requestUri) { - return new StringBuilder(requestUri.getScheme()) - .append("://") - .append(requestUri.getHost()) - .append(':') - .append(requestUri.getPort()) - .append("/serviceview/v1/"); + private static StringBuilder baseUri(UriBuilder uriBuilder) { + return new StringBuilder(uriBuilder.withPath("/serviceview/v1/").toString()); } } diff --git a/configserver/src/main/resources/configserver-app/services.xml b/configserver/src/main/resources/configserver-app/services.xml index 51fc26bdbaa..8c792089810 100644 --- a/configserver/src/main/resources/configserver-app/services.xml +++ b/configserver/src/main/resources/configserver-app/services.xml @@ -63,8 +63,29 @@ <component id="com.yahoo.vespa.orchestrator.controller.RetryingClusterControllerClientFactory" bundle="orchestrator" /> <component id="com.yahoo.vespa.orchestrator.OrchestratorImpl" bundle="orchestrator" /> - <rest-api path="orchestrator" jersey2="true"> - <components bundle="orchestrator" /> + <rest-api path="orchestrator/v1/suspensions/applications" jersey2="true"> + <components bundle="orchestrator"> + <package>com.yahoo.vespa.orchestrator.resources.appsuspension</package> + </components> + </rest-api> + <rest-api path="orchestrator/v1/health" jersey2="true"> + <components bundle="orchestrator"> + <package>com.yahoo.vespa.orchestrator.resources.health</package> + </components> + </rest-api> + <rest-api path="orchestrator/v1/hosts" jersey2="true"> + <components bundle="orchestrator"> + <package>com.yahoo.vespa.orchestrator.resources.host</package> + </components> + </rest-api> + <handler id="com.yahoo.vespa.orchestrator.resources.HostSuspensionHandler" bundle="orchestrator"> + <binding>http://*/orchestrator/v1/suspensions/hosts</binding> + <binding>http://*/orchestrator/v1/suspensions/hosts/*</binding> + </handler> + <rest-api path="orchestrator/v1/instances" jersey2="true"> + <components bundle="orchestrator"> + <package>com.yahoo.vespa.orchestrator.resources.instance</package> + </components> </rest-api> <handler id="com.yahoo.vespa.serviceview.StateRequestHandler" bundle="configserver"> diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java index 31746b66806..e94a99f59d7 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java @@ -113,18 +113,6 @@ public class LbServicesProducerTest { } @Test - public void use_power_of_two_lb_is_configured_from_feature_flag() { - RegionName regionName = RegionName.from("us-east-1"); - - LbServicesConfig conf = createModelAndGetLbServicesConfig(regionName); - assertFalse(conf.tenants("foo").applications("foo:prod:" + regionName.value() + ":default").usePowerOfTwoChoicesLb()); - - flagSource.withBooleanFlag(Flags.USE_POWER_OF_TWO_CHOICES_LOAD_BALANCING.id(), true); - conf = createModelAndGetLbServicesConfig(regionName); - assertTrue(conf.tenants("foo").applications("foo:prod:" + regionName.value() + ":default").usePowerOfTwoChoicesLb()); - } - - @Test public void generate_non_mtls_endpoints_from_feature_flag() { RegionName regionName = RegionName.from("us-east-1"); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java index ae9e451b037..6f7e0541cc7 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java @@ -212,7 +212,8 @@ public class TenantRepositoryTest { ConfigCurator.create(new MockCurator()), Metrics.createTestMetrics(), new StripedExecutor<>(new InThreadExecutorService()), - new FileDistributionFactory(configserverConfig), + new StripedExecutor<>(new InThreadExecutorService()), + new FileDistributionFactory(new ConfigserverConfig.Builder().build()), new InMemoryFlagSource(), new InThreadExecutorService(), new MockSecretStore(), diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TestTenantRepository.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TestTenantRepository.java index 8279bf4df5e..687d58fd23b 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TestTenantRepository.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TestTenantRepository.java @@ -50,6 +50,7 @@ public class TestTenantRepository extends TenantRepository { ConfigCurator.create(curator), metrics, new StripedExecutor<>(new InThreadExecutorService()), + new StripedExecutor<>(new InThreadExecutorService()), fileDistributionFactory, flagSource, new InThreadExecutorService(), diff --git a/configserver/src/test/java/com/yahoo/vespa/serviceview/StateRequestHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/serviceview/StateRequestHandlerTest.java index a0bb8a5acc7..68eca62e54b 100644 --- a/configserver/src/test/java/com/yahoo/vespa/serviceview/StateRequestHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/serviceview/StateRequestHandlerTest.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.serviceview; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.jdisc.test.MockMetric; +import com.yahoo.restapi.UriBuilder; import com.yahoo.vespa.serviceview.bindings.ApplicationView; import com.yahoo.vespa.serviceview.bindings.HealthClient; import com.yahoo.vespa.serviceview.bindings.ModelResponse; @@ -74,14 +75,14 @@ public class StateRequestHandlerTest { public final void test() { Service s = correspondingModel.resolve("vespa.yahoo.com", 8080, null); String api = "/state/v1"; - HashMap<?, ?> boom = testHandler.singleService(URI.create(EXTERNAL_BASE_URI), "default", "default", "default", "default", "default", s.getIdentifier(8080), api); + HashMap<?, ?> boom = testHandler.singleService(new UriBuilder("http://someserver:8080"), URI.create(EXTERNAL_BASE_URI), "default", "default", "default", "default", "default", s.getIdentifier(8080), api); assertEquals(EXTERNAL_BASE_URI + "tenant/default/application/default/environment/default/region/default/instance/default/service/" + s.getIdentifier(8080) + api, ((Map<?, ?>) ((List<?>) boom.get("resources")).get(0)).get("url")); } @Test public final void testLinkEquality() { - ApplicationView explicitParameters = testHandler.getUserInfo(URI.create(EXTERNAL_BASE_URI), "default", "default", "default", "default", "default"); + ApplicationView explicitParameters = testHandler.getUserInfo(new UriBuilder("http://someserver:8080"), "default", "default", "default", "default", "default"); assertEquals(EXTERNAL_BASE_URI + "tenant/default/application/default/environment/default/region/default/instance" + "/default/service/container-clustercontroller-2ul67p8psr451t3w8kdd0qwgg/state/v1/", explicitParameters.clusters.get(0).services.get(0).url); diff --git a/container-core/src/main/java/com/yahoo/restapi/ResourceResponse.java b/container-core/src/main/java/com/yahoo/restapi/ResourceResponse.java index 0188136addb..ce8c718fc74 100644 --- a/container-core/src/main/java/com/yahoo/restapi/ResourceResponse.java +++ b/container-core/src/main/java/com/yahoo/restapi/ResourceResponse.java @@ -31,7 +31,7 @@ public class ResourceResponse extends SlimeJsonResponse { var resourceArray = slime.setObject().setArray("resources"); for (var subResource : subResources) { var resourceEntry = resourceArray.addObject(); - resourceEntry.setString("url", new Uri(parentUrl).append(subResource) + resourceEntry.setString("url", new UriBuilder(parentUrl).append(subResource) .withTrailingSlash() .toString()); } diff --git a/container-core/src/main/java/com/yahoo/restapi/RestApi.java b/container-core/src/main/java/com/yahoo/restapi/RestApi.java index 08bac710001..df05723ac14 100644 --- a/container-core/src/main/java/com/yahoo/restapi/RestApi.java +++ b/container-core/src/main/java/com/yahoo/restapi/RestApi.java @@ -1,13 +1,12 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.restapi; -import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.slime.Slime; import java.io.InputStream; +import java.util.List; import java.util.Optional; import java.util.OptionalDouble; import java.util.OptionalLong; @@ -23,7 +22,6 @@ public interface RestApi { static RouteBuilder route(String pathPattern) { return new RestApiImpl.RouteBuilderImpl(pathPattern); } HttpResponse handleRequest(HttpRequest request); - ObjectMapper jacksonJsonMapper(); interface Builder { Builder setObjectMapper(ObjectMapper mapper); @@ -31,7 +29,10 @@ public interface RestApi { Builder addRoute(RouteBuilder route); Builder addFilter(Filter filter); <EXCEPTION extends RuntimeException> Builder addExceptionMapper(Class<EXCEPTION> type, ExceptionMapper<EXCEPTION> mapper); - <ENTITY> Builder addResponseMapper(Class<ENTITY> type, ResponseMapper<ENTITY> mapper); + <RESPONSE_ENTITY> Builder addResponseMapper(Class<RESPONSE_ENTITY> type, ResponseMapper<RESPONSE_ENTITY> mapper); + <REQUEST_ENTITY> Builder addRequestMapper(Class<REQUEST_ENTITY> type, RequestMapper<REQUEST_ENTITY> mapper); + <RESPONSE_ENTITY> Builder registerJacksonResponseEntity(Class<RESPONSE_ENTITY> type); + <REQUEST_ENTITY> Builder registerJacksonRequestEntity(Class<REQUEST_ENTITY> type); Builder disableDefaultExceptionMappers(); Builder disableDefaultResponseMappers(); RestApi build(); @@ -39,28 +40,34 @@ public interface RestApi { interface RouteBuilder { RouteBuilder name(String name); - RouteBuilder get(MethodHandler<?> handler); - RouteBuilder post(MethodHandler<?> handler); - RouteBuilder put(MethodHandler<?> handler); - RouteBuilder delete(MethodHandler<?> handler); - RouteBuilder patch(MethodHandler<?> handler); - RouteBuilder defaultHandler(MethodHandler<?> handler); + RouteBuilder get(Handler<?> handler); + RouteBuilder post(Handler<?> handler); + <REQUEST_ENTITY> RouteBuilder post(Class<REQUEST_ENTITY> type, HandlerWithRequestEntity<REQUEST_ENTITY, ?> handler); + RouteBuilder put(Handler<?> handler); + <REQUEST_ENTITY> RouteBuilder put(Class<REQUEST_ENTITY> type, HandlerWithRequestEntity<REQUEST_ENTITY, ?> handler); + RouteBuilder delete(Handler<?> handler); + RouteBuilder patch(Handler<?> handler); + <REQUEST_ENTITY> RouteBuilder patch(Class<REQUEST_ENTITY> type, HandlerWithRequestEntity<REQUEST_ENTITY, ?> handler); + RouteBuilder defaultHandler(Handler<?> handler); + <REQUEST_ENTITY> RouteBuilder defaultHandler(Class<REQUEST_ENTITY> type, HandlerWithRequestEntity<REQUEST_ENTITY, ?> handler); RouteBuilder addFilter(Filter filter); } - @FunctionalInterface interface ExceptionMapper<EXCEPTION extends RuntimeException> { HttpResponse toResponse(EXCEPTION exception, RequestContext context); } + @FunctionalInterface interface Handler<RESPONSE_ENTITY> { + RESPONSE_ENTITY handleRequest(RequestContext context) throws RestApiException; + } - @FunctionalInterface interface MethodHandler<ENTITY> { ENTITY handleRequest(RequestContext context) throws RestApiException; } + @FunctionalInterface interface HandlerWithRequestEntity<REQUEST_ENTITY, RESPONSE_ENTITY> { + RESPONSE_ENTITY handleRequest(RequestContext context, REQUEST_ENTITY requestEntity) throws RestApiException; + } - @FunctionalInterface interface ResponseMapper<ENTITY> { HttpResponse toHttpResponse(ENTITY responseEntity, RequestContext context); } + @FunctionalInterface interface ExceptionMapper<EXCEPTION extends RuntimeException> { HttpResponse toResponse(RequestContext context, EXCEPTION exception); } - @FunctionalInterface interface Filter { HttpResponse filterRequest(FilterContext context); } + @FunctionalInterface interface ResponseMapper<RESPONSE_ENTITY> { HttpResponse toHttpResponse(RequestContext context, RESPONSE_ENTITY responseEntity) throws RestApiException; } - /** Marker interface required for automatic serialization of Jackson response entities */ - interface JacksonResponseEntity {} + @FunctionalInterface interface RequestMapper<REQUEST_ENTITY> { Optional<REQUEST_ENTITY> toRequestEntity(RequestContext context) throws RestApiException; } - /** Marker interface required for automatic serialization of Jackson request entities */ - interface JacksonRequestEntity {} + @FunctionalInterface interface Filter { HttpResponse filterRequest(FilterContext context); } interface RequestContext { HttpRequest request(); @@ -70,6 +77,8 @@ public interface RestApi { Attributes attributes(); Optional<RequestContent> requestContent(); RequestContent requestContentOrThrow(); + ObjectMapper jacksonJsonMapper(); + UriBuilder uriBuilder(); interface Parameters { Optional<String> getString(String name); @@ -87,7 +96,9 @@ public interface RestApi { } interface PathParameters extends Parameters {} - interface QueryParameters extends Parameters {} + interface QueryParameters extends Parameters { + List<String> getStringList(String name); + } interface Headers extends Parameters {} interface Attributes { @@ -97,13 +108,7 @@ public interface RestApi { interface RequestContent { String contentType(); - InputStream inputStream(); - ObjectMapper jacksonJsonMapper(); - byte[] consumeByteArray(); - String consumeString(); - JsonNode consumeJsonNode(); - Slime consumeSlime(); - <T extends JacksonRequestEntity> T consumeJacksonEntity(Class<T> type); + InputStream content(); } } diff --git a/container-core/src/main/java/com/yahoo/restapi/RestApiImpl.java b/container-core/src/main/java/com/yahoo/restapi/RestApiImpl.java index af816f41411..e6c6d7ccb62 100644 --- a/container-core/src/main/java/com/yahoo/restapi/RestApiImpl.java +++ b/container-core/src/main/java/com/yahoo/restapi/RestApiImpl.java @@ -1,18 +1,18 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.restapi; -import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.jdisc.http.HttpRequest.Method; import com.yahoo.slime.Slime; import com.yahoo.slime.SlimeUtils; import com.yahoo.yolean.Exceptions; import java.io.IOException; import java.io.InputStream; -import java.nio.charset.StandardCharsets; +import java.net.URI; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -22,6 +22,8 @@ import java.util.Optional; import java.util.logging.Level; import java.util.logging.Logger; +import static java.nio.charset.StandardCharsets.UTF_8; + /** * @author bjorncs */ @@ -33,6 +35,7 @@ class RestApiImpl implements RestApi { private final List<Route> routes; private final List<ExceptionMapperHolder<?>> exceptionMappers; private final List<ResponseMapperHolder<?>> responseMappers; + private final List<RequestMapperHolder<?>> requestMappers; private final List<Filter> filters; private final ObjectMapper jacksonJsonMapper; @@ -45,6 +48,8 @@ class RestApiImpl implements RestApi { builderImpl.exceptionMappers, Boolean.TRUE.equals(builderImpl.disableDefaultExceptionMappers)); this.responseMappers = combineWithDefaultResponseMappers( builderImpl.responseMappers, jacksonJsonMapper, Boolean.TRUE.equals(builderImpl.disableDefaultResponseMappers)); + this.requestMappers = combineWithDefaultRequestMappers( + builderImpl.requestMappers, jacksonJsonMapper); this.filters = List.copyOf(builderImpl.filters); this.jacksonJsonMapper = jacksonJsonMapper; } @@ -65,27 +70,53 @@ class RestApiImpl implements RestApi { } } - @Override public ObjectMapper jacksonJsonMapper() { return jacksonJsonMapper; } - private HttpResponse dispatchToRoute(Route route, RequestContextImpl context) { - RestApi.MethodHandler<?> resolvedHandler = route.handlerPerMethod.get(context.request().getMethod()); - if (resolvedHandler == null) { - resolvedHandler = route.defaultHandler; + HandlerHolder<?> resolvedHandler = resolveHandler(context, route); + RequestMapperHolder<?> resolvedRequestMapper = resolveRequestMapper(resolvedHandler); + Object requestEntity; + try { + requestEntity = resolvedRequestMapper.mapper.toRequestEntity(context).orElse(null); + } catch (RuntimeException e) { + return mapException(context, e); + } + Object responseEntity; + try { + responseEntity = resolvedHandler.executeHandler(context, requestEntity); + } catch (RuntimeException e) { + return mapException(context, e); } - Object entity; + if (responseEntity == null) throw new NullPointerException("Handler must return non-null value"); + ResponseMapperHolder<?> resolvedResponseMapper = resolveResponseMapper(responseEntity); try { - entity = resolvedHandler.handleRequest(context); + return resolvedResponseMapper.toHttpResponse(context, responseEntity); } catch (RuntimeException e) { - ExceptionMapperHolder<?> mapper = exceptionMappers.stream() - .filter(holder -> holder.matches(e)) - .findFirst().orElseThrow(() -> e); - return mapper.toResponse(e, context); - } - if (entity == null) throw new NullPointerException("Handler must return non-null value"); - ResponseMapperHolder<?> mapper = responseMappers.stream() - .filter(holder -> holder.matches(entity)) - .findFirst().orElseThrow(() -> new IllegalStateException("No mapper configured for " + entity.getClass())); - return mapper.toHttpResponse(entity, context); + return mapException(context, e); + } + } + + private HandlerHolder<?> resolveHandler(RequestContextImpl context, Route route) { + HandlerHolder<?> resolvedHandler = route.handlerPerMethod.get(context.request().getMethod()); + return resolvedHandler == null ? route.defaultHandler : resolvedHandler; + } + + private RequestMapperHolder<?> resolveRequestMapper(HandlerHolder<?> resolvedHandler) { + return requestMappers.stream() + .filter(holder -> resolvedHandler.type.isAssignableFrom(holder.type)) + .findFirst().orElseThrow(() -> new IllegalStateException("No mapper configured for " + resolvedHandler.type)); + } + + private ResponseMapperHolder<?> resolveResponseMapper(Object responseEntity) { + return responseMappers.stream() + .filter(holder -> holder.type.isAssignableFrom(responseEntity.getClass())) + .findFirst().orElseThrow(() -> new IllegalStateException("No mapper configured for " + responseEntity.getClass())); + } + + private HttpResponse mapException(RequestContextImpl context, RuntimeException e) { + log.log(Level.FINE, e, e::getMessage); + ExceptionMapperHolder<?> mapper = exceptionMappers.stream() + .filter(holder -> holder.type.isAssignableFrom(e.getClass())) + .findFirst().orElseThrow(() -> e); + return mapper.toResponse(context, e); } private Route resolveRoute(Path pathMatcher) { @@ -120,7 +151,7 @@ class RestApiImpl implements RestApi { List<ExceptionMapperHolder<?>> configuredExceptionMappers, boolean disableDefaultMappers) { List<ExceptionMapperHolder<?>> exceptionMappers = new ArrayList<>(configuredExceptionMappers); if (!disableDefaultMappers){ - exceptionMappers.add(new ExceptionMapperHolder<>(RestApiException.class, (exception, context) -> exception.response())); + exceptionMappers.add(new ExceptionMapperHolder<>(RestApiException.class, (context, exception) -> exception.response())); } return exceptionMappers; } @@ -129,19 +160,73 @@ class RestApiImpl implements RestApi { List<ResponseMapperHolder<?>> configuredResponseMappers, ObjectMapper jacksonJsonMapper, boolean disableDefaultMappers) { List<ResponseMapperHolder<?>> responseMappers = new ArrayList<>(configuredResponseMappers); if (!disableDefaultMappers) { - responseMappers.add(new ResponseMapperHolder<>(HttpResponse.class, (entity, context) -> entity)); - responseMappers.add(new ResponseMapperHolder<>(String.class, (entity, context) -> new MessageResponse(entity))); - responseMappers.add(new ResponseMapperHolder<>(Slime.class, (entity, context) -> new SlimeJsonResponse(entity))); - responseMappers.add(new ResponseMapperHolder<>(JsonNode.class, (entity, context) -> new JacksonJsonResponse<>(200, entity, jacksonJsonMapper, true))); - responseMappers.add(new ResponseMapperHolder<>(RestApi.JacksonResponseEntity.class, (entity, context) -> new JacksonJsonResponse<>(200, entity, jacksonJsonMapper, true))); + responseMappers.add(new ResponseMapperHolder<>(HttpResponse.class, (context, entity) -> entity)); + responseMappers.add(new ResponseMapperHolder<>(String.class, (context, entity) -> new MessageResponse(entity))); + responseMappers.add(new ResponseMapperHolder<>(Slime.class, (context, entity) -> new SlimeJsonResponse(entity))); + responseMappers.add(new ResponseMapperHolder<>(JsonNode.class, (context, entity) -> new JacksonJsonResponse<>(200, entity, jacksonJsonMapper, true))); } return responseMappers; } + private static List<RequestMapperHolder<?>> combineWithDefaultRequestMappers( + List<RequestMapperHolder<?>> configuredRequestMappers, ObjectMapper jacksonJsonMapper) { + List<RequestMapperHolder<?>> requestMappers = new ArrayList<>(configuredRequestMappers); + requestMappers.add(new RequestMapperHolder<>(Slime.class, RestApiImpl::toSlime)); + requestMappers.add(new RequestMapperHolder<>(JsonNode.class, ctx -> toJsonNode(ctx, jacksonJsonMapper))); + requestMappers.add(new RequestMapperHolder<>(String.class, RestApiImpl::toString)); + requestMappers.add(new RequestMapperHolder<>(byte[].class, RestApiImpl::toByteArray)); + requestMappers.add(new RequestMapperHolder<>(InputStream.class, RestApiImpl::toInputStream)); + requestMappers.add(new RequestMapperHolder<>(Void.class, ctx -> Optional.empty())); + return requestMappers; + } + + private static Optional<InputStream> toInputStream(RequestContext context) { + return context.requestContent().map(RequestContext.RequestContent::content); + } + + private static Optional<byte[]> toByteArray(RequestContext context) { + InputStream in = toInputStream(context).orElse(null); + if (in == null) return Optional.empty(); + return convertIoException(() -> Optional.of(in.readAllBytes())); + } + + private static Optional<String> toString(RequestContext context) { + try { + return toByteArray(context).map(bytes -> new String(bytes, UTF_8)); + } catch (RuntimeException e) { + throw new RestApiException.BadRequest("Failed parse request content as UTF-8: " + Exceptions.toMessageString(e), e); + } + } + + private static Optional<JsonNode> toJsonNode(RequestContext context, ObjectMapper jacksonJsonMapper) { + if (log.isLoggable(Level.FINE)) { + return toString(context).map(string -> { + log.fine(() -> "Request content: " + string); + return convertIoException("Failed to parse JSON", () -> jacksonJsonMapper.readTree(string)); + }); + } else { + return toInputStream(context) + .map(in -> convertIoException("Invalid JSON", () -> jacksonJsonMapper.readTree(in))); + } + } + + private static Optional<Slime> toSlime(RequestContext context) { + try { + return toString(context).map(string -> { + log.fine(() -> "Request content: " + string); + return SlimeUtils.jsonToSlimeOrThrow(string); + }); + } catch (com.yahoo.slime.JsonParseException e) { + log.log(Level.FINE, e.getMessage(), e); + throw new RestApiException.BadRequest("Invalid JSON: " + Exceptions.toMessageString(e), e); + } + } + static class BuilderImpl implements RestApi.Builder { private final List<Route> routes = new ArrayList<>(); private final List<ExceptionMapperHolder<?>> exceptionMappers = new ArrayList<>(); private final List<ResponseMapperHolder<?>> responseMappers = new ArrayList<>(); + private final List<RequestMapperHolder<?>> requestMappers = new ArrayList<>(); private final List<RestApi.Filter> filters = new ArrayList<>(); private Route defaultRoute; private ObjectMapper jacksonJsonMapper; @@ -161,31 +246,73 @@ class RestApiImpl implements RestApi { responseMappers.add(new ResponseMapperHolder<>(type, mapper)); return this; } + @Override public <ENTITY> Builder addRequestMapper(Class<ENTITY> type, RequestMapper<ENTITY> mapper) { + requestMappers.add(new RequestMapperHolder<>(type, mapper)); return this; + } + + @Override public <ENTITY> Builder registerJacksonResponseEntity(Class<ENTITY> type) { + addResponseMapper(type, new JacksonResponseMapper<>()); return this; + } + + @Override public <ENTITY> Builder registerJacksonRequestEntity(Class<ENTITY> type) { + addRequestMapper(type, new JacksonRequestMapper<>(type)); return this; + } + @Override public Builder disableDefaultExceptionMappers() { this.disableDefaultExceptionMappers = true; return this; } @Override public Builder disableDefaultResponseMappers() { this.disableDefaultResponseMappers = true; return this; } + @Override public RestApi build() { return new RestApiImpl(this); } } public static class RouteBuilderImpl implements RestApi.RouteBuilder { private final String pathPattern; private String name; - private final Map<com.yahoo.jdisc.http.HttpRequest.Method, RestApi.MethodHandler<?>> handlerPerMethod = new HashMap<>(); + private final Map<Method, HandlerHolder<?>> handlerPerMethod = new HashMap<>(); private final List<RestApi.Filter> filters = new ArrayList<>(); - private RestApi.MethodHandler<?> defaultHandler; + private HandlerHolder<?> defaultHandler; RouteBuilderImpl(String pathPattern) { this.pathPattern = pathPattern; } @Override public RestApi.RouteBuilder name(String name) { this.name = name; return this; } - @Override public RestApi.RouteBuilder get(RestApi.MethodHandler<?> handler) { return addHandler(com.yahoo.jdisc.http.HttpRequest.Method.GET, handler); } - @Override public RestApi.RouteBuilder post(RestApi.MethodHandler<?> handler) { return addHandler(com.yahoo.jdisc.http.HttpRequest.Method.POST, handler); } - @Override public RestApi.RouteBuilder put(RestApi.MethodHandler<?> handler) { return addHandler(com.yahoo.jdisc.http.HttpRequest.Method.PUT, handler); } - @Override public RestApi.RouteBuilder delete(RestApi.MethodHandler<?> handler) { return addHandler(com.yahoo.jdisc.http.HttpRequest.Method.DELETE, handler); } - @Override public RestApi.RouteBuilder patch(RestApi.MethodHandler<?> handler) { return addHandler(com.yahoo.jdisc.http.HttpRequest.Method.PATCH, handler); } - @Override public RestApi.RouteBuilder defaultHandler(RestApi.MethodHandler<?> handler) { defaultHandler = handler; return this; } + @Override public RestApi.RouteBuilder get(Handler<?> handler) { + return addHandler(Method.GET, handler); + } + @Override public RestApi.RouteBuilder post(Handler<?> handler) { + return addHandler(Method.POST, handler); + } + @Override public <ENTITY> RouteBuilder post(Class<ENTITY> type, HandlerWithRequestEntity<ENTITY, ?> handler) { + return addHandler(Method.POST, type, handler); + } + @Override public RestApi.RouteBuilder put(Handler<?> handler) { + return addHandler(Method.PUT, handler); + } + @Override public <ENTITY> RouteBuilder put(Class<ENTITY> type, HandlerWithRequestEntity<ENTITY, ?> handler) { + return addHandler(Method.PUT, type, handler); + } + @Override public RestApi.RouteBuilder delete(Handler<?> handler) { + return addHandler(Method.DELETE, handler); + } + @Override public RestApi.RouteBuilder patch(Handler<?> handler) { + return addHandler(Method.PATCH, handler); + } + @Override public <ENTITY> RouteBuilder patch(Class<ENTITY> type, HandlerWithRequestEntity<ENTITY, ?> handler) { + return addHandler(Method.PATCH, type, handler); + } + @Override public RestApi.RouteBuilder defaultHandler(Handler<?> handler) { + defaultHandler = HandlerHolder.of(handler); return this; + } + @Override public <ENTITY> RouteBuilder defaultHandler(Class<ENTITY> type, HandlerWithRequestEntity<ENTITY, ?> handler) { + defaultHandler = HandlerHolder.of(type, handler); return this; + } @Override public RestApi.RouteBuilder addFilter(RestApi.Filter filter) { filters.add(filter); return this; } - private RestApi.RouteBuilder addHandler(com.yahoo.jdisc.http.HttpRequest.Method method, RestApi.MethodHandler<?> handler) { - handlerPerMethod.put(method, handler); return this; + private RestApi.RouteBuilder addHandler(Method method, Handler<?> handler) { + handlerPerMethod.put(method, HandlerHolder.of(handler)); return this; + } + + private <ENTITY> RestApi.RouteBuilder addHandler( + Method method, Class<ENTITY> type, HandlerWithRequestEntity<ENTITY, ?> handler) { + handlerPerMethod.put(method, HandlerHolder.of(type, handler)); return this; } private Route build() { return new Route(this); } @@ -217,6 +344,11 @@ class RestApiImpl implements RestApi { @Override public RequestContent requestContentOrThrow() { return requestContent().orElseThrow(() -> new RestApiException.BadRequest("Request content missing")); } + @Override public ObjectMapper jacksonJsonMapper() { return jacksonJsonMapper; } + @Override public UriBuilder uriBuilder() { + URI uri = request.getUri(); + return new UriBuilder(uri.getScheme() + "://" + uri.getHost() + ':' + uri.getPort()); + } private class PathParametersImpl implements RestApi.RequestContext.PathParameters { @Override @@ -239,6 +371,11 @@ class RestApiImpl implements RestApi { return getString(name) .orElseThrow(() -> new RestApiException.BadRequest("Query parameter '" + name + "' is missing")); } + @Override public List<String> getStringList(String name) { + List<String> result = request.getJDiscRequest().parameters().get(name); + if (result == null) return List.of(); + return List.copyOf(result); + } } private class HeadersImpl implements RestApi.RequestContext.Headers { @@ -251,73 +388,13 @@ class RestApiImpl implements RestApi { private class RequestContentImpl implements RestApi.RequestContext.RequestContent { @Override public String contentType() { return request.getHeader("Content-Type"); } - @Override public InputStream inputStream() { return request.getData(); } - @Override public ObjectMapper jacksonJsonMapper() { return jacksonJsonMapper; } - @Override public byte[] consumeByteArray() { return convertIoException(() -> inputStream().readAllBytes()); } - @Override public String consumeString() { return new String(consumeByteArray(), StandardCharsets.UTF_8); } - - @Override - public JsonNode consumeJsonNode() { - return convertIoException(() -> { - try { - if (log.isLoggable(Level.FINE)) { - String content = consumeString(); - log.fine(() -> "Request content: " + content); - return jacksonJsonMapper.readTree(content); - } else { - return jacksonJsonMapper.readTree(request.getData()); - } - } catch (com.fasterxml.jackson.core.JsonParseException e) { - log.log(Level.FINE, e.getMessage(), e); - throw new RestApiException.BadRequest("Invalid json request content: " + Exceptions.toMessageString(e), e); - } - }); - } - - @Override - public Slime consumeSlime() { - try { - String content = consumeString(); - log.fine(() -> "Request content: " + content); - return SlimeUtils.jsonToSlimeOrThrow(content); - } catch (com.yahoo.slime.JsonParseException e) { - log.log(Level.FINE, e.getMessage(), e); - throw new RestApiException.BadRequest("Invalid json request content: " + Exceptions.toMessageString(e), e); - } - } - - @Override - public <T extends JacksonRequestEntity> T consumeJacksonEntity(Class<T> type) { - return convertIoException(() -> { - try { - if (log.isLoggable(Level.FINE)) { - String content = consumeString(); - log.fine(() -> "Request content: " + content); - return jacksonJsonMapper.readValue(content, type); - } else { - return jacksonJsonMapper.readValue(request.getData(), type); - } - } catch (com.fasterxml.jackson.core.JsonParseException | JsonMappingException e) { - log.log(Level.FINE, e.getMessage(), e); - throw new RestApiException.BadRequest("Invalid json request content: " + Exceptions.toMessageString(e), e); - } - }); - } + @Override public InputStream content() { return request.getData(); } } private class AttributesImpl implements RestApi.RequestContext.Attributes { @Override public Optional<Object> get(String name) { return Optional.ofNullable(request.getJDiscRequest().context().get(name)); } @Override public void set(String name, Object value) { request.getJDiscRequest().context().put(name, value); } } - - @FunctionalInterface private interface SupplierThrowingIoException<T> { T get() throws IOException; } - private static <T> T convertIoException(SupplierThrowingIoException<T> supplier) { - try { - return supplier.get(); - } catch (IOException e) { - throw new RestApiException.InternalServerError("Failed to read request content: " + Exceptions.toMessageString(e), e); - } - } } private class FilterContextImpl implements RestApi.FilterContext { @@ -357,8 +434,7 @@ class RestApiImpl implements RestApi { this.mapper = mapper; } - boolean matches(RuntimeException e) { return type.isAssignableFrom(e.getClass()); } - HttpResponse toResponse(RuntimeException e, RestApi.RequestContext context) { return mapper.toResponse(type.cast(e), context); } + HttpResponse toResponse(RestApi.RequestContext context, RuntimeException e) { return mapper.toResponse(context, type.cast(e)); } } private static class ResponseMapperHolder<ENTITY> { @@ -370,16 +446,47 @@ class RestApiImpl implements RestApi { this.mapper = mapper; } - boolean matches(Object entity) { return type.isAssignableFrom(entity.getClass()); } - HttpResponse toHttpResponse(Object entity, RestApi.RequestContext context) { return mapper.toHttpResponse(type.cast(entity), context); } + HttpResponse toHttpResponse(RestApi.RequestContext context, Object entity) { return mapper.toHttpResponse(context, type.cast(entity)); } + } + + private static class HandlerHolder<REQUEST_ENTITY> { + final Class<REQUEST_ENTITY> type; + final HandlerWithRequestEntity<REQUEST_ENTITY, ?> handler; + + HandlerHolder(Class<REQUEST_ENTITY> type, HandlerWithRequestEntity<REQUEST_ENTITY, ?> handler) { + this.type = type; + this.handler = handler; + } + + static <RESPONSE_ENTITY, REQUEST_ENTITY> HandlerHolder<REQUEST_ENTITY> of( + Class<REQUEST_ENTITY> type, HandlerWithRequestEntity<REQUEST_ENTITY, RESPONSE_ENTITY> handler) { + return new HandlerHolder<>(type, handler); + } + + static <RESPONSE_ENTITY> HandlerHolder<Void> of(Handler<RESPONSE_ENTITY> handler) { + return new HandlerHolder<>( + Void.class, + (HandlerWithRequestEntity<Void, RESPONSE_ENTITY>) (context, nullEntity) -> handler.handleRequest(context)); + } + + Object executeHandler(RestApi.RequestContext context, Object entity) { return handler.handleRequest(context, type.cast(entity)); } } + private static class RequestMapperHolder<ENTITY> { + final Class<ENTITY> type; + final RestApi.RequestMapper<ENTITY> mapper; + + RequestMapperHolder(Class<ENTITY> type, RequestMapper<ENTITY> mapper) { + this.type = type; + this.mapper = mapper; + } + } static class Route { private final String pathPattern; private final String name; - private final Map<com.yahoo.jdisc.http.HttpRequest.Method, RestApi.MethodHandler<?>> handlerPerMethod; - private final RestApi.MethodHandler<?> defaultHandler; + private final Map<Method, HandlerHolder<?>> handlerPerMethod; + private final HandlerHolder<?> defaultHandler; private final List<Filter> filters; private Route(RestApi.RouteBuilder builder) { @@ -391,9 +498,48 @@ class RestApiImpl implements RestApi { this.filters = List.copyOf(builderImpl.filters); } - private RestApi.MethodHandler<?> createDefaultMethodHandler() { - return context -> { throw new RestApiException.MethodNotAllowed(context.request()); }; + private HandlerHolder<?> createDefaultMethodHandler() { + return HandlerHolder.of(context -> { throw new RestApiException.MethodNotAllowed(context.request()); }); + } + } + + private static class JacksonRequestMapper<ENTITY> implements RequestMapper<ENTITY> { + private final Class<ENTITY> type; + + JacksonRequestMapper(Class<ENTITY> type) { this.type = type; } + + @Override + public Optional<ENTITY> toRequestEntity(RequestContext context) throws RestApiException { + if (log.isLoggable(Level.FINE)) { + return RestApiImpl.toString(context).map(string -> { + log.fine(() -> "Request content: " + string); + return convertIoException("Failed to parse JSON", () -> context.jacksonJsonMapper().readValue(string, type)); + }); + } else { + return RestApiImpl.toInputStream(context) + .map(in -> convertIoException("Invalid JSON", () -> context.jacksonJsonMapper().readValue(in, type))); + } + } + } + + private static class JacksonResponseMapper<ENTITY> implements ResponseMapper<ENTITY> { + @Override + public HttpResponse toHttpResponse(RequestContext context, ENTITY responseEntity) throws RestApiException { + return new JacksonJsonResponse<>(200, responseEntity, context.jacksonJsonMapper(), true); + } + } + + @FunctionalInterface private interface SupplierThrowingIoException<T> { T get() throws IOException; } + private static <T> T convertIoException(String messagePrefix, SupplierThrowingIoException<T> supplier) { + try { + return supplier.get(); + } catch (IOException e) { + log.log(Level.FINE, e.getMessage(), e); + throw new RestApiException.InternalServerError(messagePrefix + ": " + Exceptions.toMessageString(e), e); } } + private static <T> T convertIoException(SupplierThrowingIoException<T> supplier) { + return convertIoException("Failed to read request content", supplier); + } } diff --git a/container-core/src/main/java/com/yahoo/restapi/Uri.java b/container-core/src/main/java/com/yahoo/restapi/UriBuilder.java index c1b0d19eb3e..daebb147547 100644 --- a/container-core/src/main/java/com/yahoo/restapi/Uri.java +++ b/container-core/src/main/java/com/yahoo/restapi/UriBuilder.java @@ -10,16 +10,16 @@ import java.net.URISyntaxException; * * @author bratseth */ -public class Uri { +public class UriBuilder { /** The URI instance wrapped by this */ private final URI uri; - public Uri(URI uri) { + public UriBuilder(URI uri) { this.uri = uri; } - public Uri(String uri) { + public UriBuilder(String uri) { try { this.uri = new URI(uri); } @@ -29,21 +29,21 @@ public class Uri { } /** Returns a uri with the given path appended and all parameters removed */ - public Uri append(String pathElement) { - return new Uri(withoutParameters().withTrailingSlash() + pathElement); + public UriBuilder append(String pathElement) { + return new UriBuilder(withoutParameters().withTrailingSlash() + pathElement); } - public Uri withoutParameters() { + public UriBuilder withoutParameters() { int parameterStart = uri.toString().indexOf("?"); if (parameterStart < 0) - return new Uri(uri.toString()); + return new UriBuilder(uri.toString()); else - return new Uri(uri.toString().substring(0, parameterStart)); + return new UriBuilder(uri.toString().substring(0, parameterStart)); } - public Uri withPath(String path) { + public UriBuilder withPath(String path) { try { - return new Uri(new URI(uri.getScheme(), uri.getUserInfo(), uri.getHost(), + return new UriBuilder(new URI(uri.getScheme(), uri.getUserInfo(), uri.getHost(), uri.getPort(), path, uri.getQuery(), uri.getFragment())); } catch (URISyntaxException e) { @@ -51,9 +51,9 @@ public class Uri { } } - public Uri withTrailingSlash() { + public UriBuilder withTrailingSlash() { if (toString().endsWith("/")) return this; - return new Uri(toString() + "/"); + return new UriBuilder(toString() + "/"); } public URI toURI() { return uri; } diff --git a/container-core/src/test/java/com/yahoo/restapi/RestApiImplTest.java b/container-core/src/test/java/com/yahoo/restapi/RestApiImplTest.java index 16cc2353986..1de8184ce22 100644 --- a/container-core/src/test/java/com/yahoo/restapi/RestApiImplTest.java +++ b/container-core/src/test/java/com/yahoo/restapi/RestApiImplTest.java @@ -43,7 +43,7 @@ class RestApiImplTest { @Test void executes_filters_and_handler_in_correct_order() { List<String> actualEvaluationOrdering = new ArrayList<>(); - RestApi.MethodHandler<HttpResponse> handler = context -> { + RestApi.Handler<HttpResponse> handler = context -> { actualEvaluationOrdering.add("handler"); return new MessageResponse("get-method-response"); }; @@ -83,8 +83,8 @@ class RestApiImplTest { .disableDefaultResponseMappers() .addRoute(route("/long").get(ctx -> 123456L)) .addRoute(route("/exception").get(ctx -> 123L / 0L)) - .addResponseMapper(Long.class, (entity, ctx) -> new MessageResponse("long value is " + entity)) - .addExceptionMapper(ArithmeticException.class, (exception, ctx) -> ErrorResponse.internalServerError("oops division by zero")) + .addResponseMapper(Long.class, (ctx, entity) -> new MessageResponse("long value is " + entity)) + .addExceptionMapper(ArithmeticException.class, (ctx, exception) -> ErrorResponse.internalServerError("oops division by zero")) .build(); verifyJsonResponse(restApi, Method.GET, "/long", null, 200, "{\"message\":\"long value is 123456\"}"); verifyJsonResponse(restApi, Method.GET, "/exception", null, 500, "{\"message\":\"oops division by zero\", \"error-code\":\"INTERNAL_SERVER_ERROR\"}"); @@ -92,9 +92,11 @@ class RestApiImplTest { @Test void method_handler_can_consume_and_produce_json() { + RestApi.HandlerWithRequestEntity<TestEntity, TestEntity> handler = (context, requestEntity) -> requestEntity; RestApi restApi = RestApi.builder() - .addRoute(route("/api").post( - ctx -> ctx.requestContent().get().consumeJacksonEntity(TestEntity.class))) + .registerJacksonRequestEntity(TestEntity.class) + .registerJacksonResponseEntity(TestEntity.class) + .addRoute(route("/api").post(TestEntity.class, handler)) .build(); String rawJson = "{\"mystring\":\"my-string-value\", \"myinstant\":\"2000-01-01T00:00:00Z\"}"; verifyJsonResponse(restApi, Method.POST, "/api", rawJson, 200, rawJson); @@ -118,7 +120,7 @@ class RestApiImplTest { } } - public static class TestEntity implements RestApi.JacksonRequestEntity, RestApi.JacksonResponseEntity { + public static class TestEntity { @JsonProperty("mystring") public String stringValue; @JsonProperty("myinstant") public Instant instantValue; } diff --git a/container-disc/pom.xml b/container-disc/pom.xml index b8fbf5fae2a..a78657b7f09 100644 --- a/container-disc/pom.xml +++ b/container-disc/pom.xml @@ -92,17 +92,6 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> - <artifactId>messagebus-disc</artifactId> - <version>${project.version}</version> - <exclusions> - <exclusion> - <groupId>com.yahoo.vespa</groupId> - <artifactId>config</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> <artifactId>vespajlib</artifactId> <version>${project.version}</version> </dependency> diff --git a/container-messagebus/OWNERS b/container-messagebus/OWNERS index 12b533ec610..2fabd640e37 100644 --- a/container-messagebus/OWNERS +++ b/container-messagebus/OWNERS @@ -1 +1,3 @@ +bjorncs +gjoranv havardpe diff --git a/container-messagebus/README b/container-messagebus/README deleted file mode 100644 index e443e31b805..00000000000 --- a/container-messagebus/README +++ /dev/null @@ -1 +0,0 @@ -Provides messeagebus for containers. diff --git a/container-messagebus/README.md b/container-messagebus/README.md new file mode 100644 index 00000000000..e3fe6a12b75 --- /dev/null +++ b/container-messagebus/README.md @@ -0,0 +1,4 @@ +<!-- Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +# JDisc messagebus implementation + +Messagebus protocol implementation for JDisc. diff --git a/container-messagebus/pom.xml b/container-messagebus/pom.xml index 52e3652634b..f98607015f9 100644 --- a/container-messagebus/pom.xml +++ b/container-messagebus/pom.xml @@ -72,12 +72,6 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> - <artifactId>jdisc_messagebus_service</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> <artifactId>messagebus</artifactId> <version>${project.version}</version> <scope>provided</scope> @@ -93,17 +87,6 @@ <build> <plugins> <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-jar-plugin</artifactId> - <configuration> - <archive> - <manifestEntries> - <Bundle-SymbolicName>${project.artifactId}</Bundle-SymbolicName> - </manifestEntries> - </archive> - </configuration> - </plugin> - <plugin> <groupId>com.yahoo.vespa</groupId> <artifactId>config-class-plugin</artifactId> <version>${project.version}</version> diff --git a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/jdisc/IgnoredCompletionHandler.java b/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/IgnoredCompletionHandler.java index c64fea8653b..c64fea8653b 100644 --- a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/jdisc/IgnoredCompletionHandler.java +++ b/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/IgnoredCompletionHandler.java diff --git a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/jdisc/MbusClient.java b/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/MbusClient.java index 922e4140868..922e4140868 100644 --- a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/jdisc/MbusClient.java +++ b/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/MbusClient.java diff --git a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/jdisc/MbusRequest.java b/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/MbusRequest.java index a0bedd678eb..a0bedd678eb 100644 --- a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/jdisc/MbusRequest.java +++ b/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/MbusRequest.java diff --git a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/jdisc/MbusRequestHandler.java b/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/MbusRequestHandler.java index fb5657a9215..fb5657a9215 100644 --- a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/jdisc/MbusRequestHandler.java +++ b/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/MbusRequestHandler.java diff --git a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/jdisc/MbusResponse.java b/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/MbusResponse.java index 37da4d8569f..37da4d8569f 100644 --- a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/jdisc/MbusResponse.java +++ b/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/MbusResponse.java diff --git a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/jdisc/MbusServer.java b/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/MbusServer.java index e26e1e7e134..e26e1e7e134 100644 --- a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/jdisc/MbusServer.java +++ b/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/MbusServer.java diff --git a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/jdisc/StatusCodes.java b/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/StatusCodes.java index 6570c910af3..6570c910af3 100644 --- a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/jdisc/StatusCodes.java +++ b/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/StatusCodes.java diff --git a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/jdisc/package-info.java b/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/package-info.java index 9aea8cf7db8..9aea8cf7db8 100644 --- a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/jdisc/package-info.java +++ b/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/package-info.java diff --git a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/jdisc/test/ClientTestDriver.java b/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/test/ClientTestDriver.java index 111805d61b0..111805d61b0 100644 --- a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/jdisc/test/ClientTestDriver.java +++ b/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/test/ClientTestDriver.java diff --git a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/jdisc/test/MessageQueue.java b/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/test/MessageQueue.java index c5287165e27..c5287165e27 100644 --- a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/jdisc/test/MessageQueue.java +++ b/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/test/MessageQueue.java diff --git a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/jdisc/test/RemoteClient.java b/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/test/RemoteClient.java index 57d0abd980b..57d0abd980b 100644 --- a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/jdisc/test/RemoteClient.java +++ b/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/test/RemoteClient.java diff --git a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/jdisc/test/RemoteServer.java b/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/test/RemoteServer.java index 1f0f82c4903..1f0f82c4903 100644 --- a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/jdisc/test/RemoteServer.java +++ b/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/test/RemoteServer.java diff --git a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/jdisc/test/ReplyQueue.java b/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/test/ReplyQueue.java index 6c48aab5a7f..6c48aab5a7f 100644 --- a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/jdisc/test/ReplyQueue.java +++ b/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/test/ReplyQueue.java diff --git a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/jdisc/test/ServerTestDriver.java b/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/test/ServerTestDriver.java index e59db28e886..e59db28e886 100644 --- a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/jdisc/test/ServerTestDriver.java +++ b/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/test/ServerTestDriver.java diff --git a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/network/package-info.java b/container-messagebus/src/main/java/com/yahoo/messagebus/network/package-info.java index 72f563e8bd7..72f563e8bd7 100644 --- a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/network/package-info.java +++ b/container-messagebus/src/main/java/com/yahoo/messagebus/network/package-info.java diff --git a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/network/rpc/package-info.java b/container-messagebus/src/main/java/com/yahoo/messagebus/network/rpc/package-info.java index 7b468813713..7b468813713 100644 --- a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/network/rpc/package-info.java +++ b/container-messagebus/src/main/java/com/yahoo/messagebus/network/rpc/package-info.java diff --git a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/package-info.java b/container-messagebus/src/main/java/com/yahoo/messagebus/package-info.java index 63b713e70e0..63b713e70e0 100644 --- a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/package-info.java +++ b/container-messagebus/src/main/java/com/yahoo/messagebus/package-info.java diff --git a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/routing/package-info.java b/container-messagebus/src/main/java/com/yahoo/messagebus/routing/package-info.java index ba8fc5fafba..ba8fc5fafba 100644 --- a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/routing/package-info.java +++ b/container-messagebus/src/main/java/com/yahoo/messagebus/routing/package-info.java diff --git a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/ClientSession.java b/container-messagebus/src/main/java/com/yahoo/messagebus/shared/ClientSession.java index 0964a254cf2..0964a254cf2 100644 --- a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/ClientSession.java +++ b/container-messagebus/src/main/java/com/yahoo/messagebus/shared/ClientSession.java diff --git a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/NullNetwork.java b/container-messagebus/src/main/java/com/yahoo/messagebus/shared/NullNetwork.java index ad58d6b9a5e..ad58d6b9a5e 100644 --- a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/NullNetwork.java +++ b/container-messagebus/src/main/java/com/yahoo/messagebus/shared/NullNetwork.java diff --git a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/ServerSession.java b/container-messagebus/src/main/java/com/yahoo/messagebus/shared/ServerSession.java index 56713815c7a..56713815c7a 100644 --- a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/ServerSession.java +++ b/container-messagebus/src/main/java/com/yahoo/messagebus/shared/ServerSession.java diff --git a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/SharedDestinationSession.java b/container-messagebus/src/main/java/com/yahoo/messagebus/shared/SharedDestinationSession.java index 7da164757cd..7da164757cd 100644 --- a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/SharedDestinationSession.java +++ b/container-messagebus/src/main/java/com/yahoo/messagebus/shared/SharedDestinationSession.java diff --git a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/SharedIntermediateSession.java b/container-messagebus/src/main/java/com/yahoo/messagebus/shared/SharedIntermediateSession.java index 5c9fab46e34..5c9fab46e34 100644 --- a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/SharedIntermediateSession.java +++ b/container-messagebus/src/main/java/com/yahoo/messagebus/shared/SharedIntermediateSession.java diff --git a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/SharedMessageBus.java b/container-messagebus/src/main/java/com/yahoo/messagebus/shared/SharedMessageBus.java index dd135a51378..dd135a51378 100644 --- a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/SharedMessageBus.java +++ b/container-messagebus/src/main/java/com/yahoo/messagebus/shared/SharedMessageBus.java diff --git a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/SharedSourceSession.java b/container-messagebus/src/main/java/com/yahoo/messagebus/shared/SharedSourceSession.java index 56071682349..56071682349 100644 --- a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/SharedSourceSession.java +++ b/container-messagebus/src/main/java/com/yahoo/messagebus/shared/SharedSourceSession.java diff --git a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/package-info.java b/container-messagebus/src/main/java/com/yahoo/messagebus/shared/package-info.java index 941a0dc4c5c..941a0dc4c5c 100644 --- a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/package-info.java +++ b/container-messagebus/src/main/java/com/yahoo/messagebus/shared/package-info.java diff --git a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/test/package-info.java b/container-messagebus/src/main/java/com/yahoo/messagebus/test/package-info.java index 42bc03b6e17..42bc03b6e17 100644 --- a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/test/package-info.java +++ b/container-messagebus/src/main/java/com/yahoo/messagebus/test/package-info.java diff --git a/jdisc_messagebus_service/src/test/java/com/yahoo/messagebus/jdisc/ClientThreadingTestCase.java b/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/ClientThreadingTestCase.java index 62a9a864781..62a9a864781 100644 --- a/jdisc_messagebus_service/src/test/java/com/yahoo/messagebus/jdisc/ClientThreadingTestCase.java +++ b/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/ClientThreadingTestCase.java diff --git a/jdisc_messagebus_service/src/test/java/com/yahoo/messagebus/jdisc/MbusClientTestCase.java b/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/MbusClientTestCase.java index 9cfd1fd02b9..9cfd1fd02b9 100644 --- a/jdisc_messagebus_service/src/test/java/com/yahoo/messagebus/jdisc/MbusClientTestCase.java +++ b/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/MbusClientTestCase.java diff --git a/jdisc_messagebus_service/src/test/java/com/yahoo/messagebus/jdisc/MbusRequestHandlerTestCase.java b/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/MbusRequestHandlerTestCase.java index 316ad18bae9..316ad18bae9 100644 --- a/jdisc_messagebus_service/src/test/java/com/yahoo/messagebus/jdisc/MbusRequestHandlerTestCase.java +++ b/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/MbusRequestHandlerTestCase.java diff --git a/jdisc_messagebus_service/src/test/java/com/yahoo/messagebus/jdisc/MbusRequestTestCase.java b/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/MbusRequestTestCase.java index c68ab4e6742..c68ab4e6742 100644 --- a/jdisc_messagebus_service/src/test/java/com/yahoo/messagebus/jdisc/MbusRequestTestCase.java +++ b/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/MbusRequestTestCase.java diff --git a/jdisc_messagebus_service/src/test/java/com/yahoo/messagebus/jdisc/MbusResponseTestCase.java b/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/MbusResponseTestCase.java index eb4cb949770..eb4cb949770 100644 --- a/jdisc_messagebus_service/src/test/java/com/yahoo/messagebus/jdisc/MbusResponseTestCase.java +++ b/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/MbusResponseTestCase.java diff --git a/jdisc_messagebus_service/src/test/java/com/yahoo/messagebus/jdisc/MbusServerConformanceTest.java b/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/MbusServerConformanceTest.java index bf89f3869ed..bf89f3869ed 100644 --- a/jdisc_messagebus_service/src/test/java/com/yahoo/messagebus/jdisc/MbusServerConformanceTest.java +++ b/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/MbusServerConformanceTest.java diff --git a/jdisc_messagebus_service/src/test/java/com/yahoo/messagebus/jdisc/MbusServerTestCase.java b/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/MbusServerTestCase.java index 9d45d2e7abf..9d45d2e7abf 100644 --- a/jdisc_messagebus_service/src/test/java/com/yahoo/messagebus/jdisc/MbusServerTestCase.java +++ b/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/MbusServerTestCase.java diff --git a/jdisc_messagebus_service/src/test/java/com/yahoo/messagebus/jdisc/ServerThreadingTestCase.java b/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/ServerThreadingTestCase.java index a7ee355094f..a7ee355094f 100644 --- a/jdisc_messagebus_service/src/test/java/com/yahoo/messagebus/jdisc/ServerThreadingTestCase.java +++ b/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/ServerThreadingTestCase.java diff --git a/jdisc_messagebus_service/src/test/java/com/yahoo/messagebus/jdisc/test/ClientTestDriverTestCase.java b/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/test/ClientTestDriverTestCase.java index ef290a070cb..ef290a070cb 100644 --- a/jdisc_messagebus_service/src/test/java/com/yahoo/messagebus/jdisc/test/ClientTestDriverTestCase.java +++ b/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/test/ClientTestDriverTestCase.java diff --git a/jdisc_messagebus_service/src/test/java/com/yahoo/messagebus/jdisc/test/ServerTestDriverTestCase.java b/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/test/ServerTestDriverTestCase.java index f6ae2335d12..f6ae2335d12 100644 --- a/jdisc_messagebus_service/src/test/java/com/yahoo/messagebus/jdisc/test/ServerTestDriverTestCase.java +++ b/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/test/ServerTestDriverTestCase.java diff --git a/jdisc_messagebus_service/src/test/java/com/yahoo/messagebus/shared/SharedDestinationSessionTestCase.java b/container-messagebus/src/test/java/com/yahoo/messagebus/shared/SharedDestinationSessionTestCase.java index 78e79da4b9f..78e79da4b9f 100644 --- a/jdisc_messagebus_service/src/test/java/com/yahoo/messagebus/shared/SharedDestinationSessionTestCase.java +++ b/container-messagebus/src/test/java/com/yahoo/messagebus/shared/SharedDestinationSessionTestCase.java diff --git a/jdisc_messagebus_service/src/test/java/com/yahoo/messagebus/shared/SharedIntermediateSessionTestCase.java b/container-messagebus/src/test/java/com/yahoo/messagebus/shared/SharedIntermediateSessionTestCase.java index 87958415149..87958415149 100644 --- a/jdisc_messagebus_service/src/test/java/com/yahoo/messagebus/shared/SharedIntermediateSessionTestCase.java +++ b/container-messagebus/src/test/java/com/yahoo/messagebus/shared/SharedIntermediateSessionTestCase.java diff --git a/jdisc_messagebus_service/src/test/java/com/yahoo/messagebus/shared/SharedMessageBusTestCase.java b/container-messagebus/src/test/java/com/yahoo/messagebus/shared/SharedMessageBusTestCase.java index a54489a89e6..a54489a89e6 100644 --- a/jdisc_messagebus_service/src/test/java/com/yahoo/messagebus/shared/SharedMessageBusTestCase.java +++ b/container-messagebus/src/test/java/com/yahoo/messagebus/shared/SharedMessageBusTestCase.java diff --git a/jdisc_messagebus_service/src/test/java/com/yahoo/messagebus/shared/SharedSourceSessionTestCase.java b/container-messagebus/src/test/java/com/yahoo/messagebus/shared/SharedSourceSessionTestCase.java index 1f0966fc961..1f0966fc961 100644 --- a/jdisc_messagebus_service/src/test/java/com/yahoo/messagebus/shared/SharedSourceSessionTestCase.java +++ b/container-messagebus/src/test/java/com/yahoo/messagebus/shared/SharedSourceSessionTestCase.java diff --git a/container-search-and-docproc/pom.xml b/container-search-and-docproc/pom.xml index 27334a8ddf3..01516cfdd99 100644 --- a/container-search-and-docproc/pom.xml +++ b/container-search-and-docproc/pom.xml @@ -194,12 +194,6 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> - <artifactId>messagebus-disc</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> <artifactId>container-messagebus</artifactId> <version>${project.version}</version> <scope>provided</scope> diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json index b9340636841..75b8814ecb0 100644 --- a/container-search/abi-spec.json +++ b/container-search/abi-spec.json @@ -474,6 +474,7 @@ ], "methods": [ "public void <init>(java.lang.String)", + "public void <init>(java.lang.String, java.util.Map)", "public com.yahoo.prelude.query.Item$ItemType getItemType()" ], "fields": [] @@ -1620,6 +1621,7 @@ ], "methods": [ "public void <init>(java.lang.String, int)", + "public void <init>(java.lang.String, int, java.util.Map)", "public void setScoreThreshold(double)", "public void setThresholdBoostFactor(double)", "public int getTargetNumHits()", @@ -1667,6 +1669,7 @@ ], "methods": [ "public void <init>(java.lang.String)", + "public void <init>(java.lang.String, java.util.Map)", "public java.lang.Integer addToken(long, int)", "public java.lang.Integer addToken(java.lang.String, int)", "public java.lang.Integer addToken(java.lang.String)", diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java index 6f360c8b456..33ebebd8d49 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java @@ -81,7 +81,7 @@ public class FastSearcher extends VespaBackEndSearcher { @Override public Result doSearch2(Query query, Execution execution) { - if (dispatcher.searchCluster().groupSize() == 1) + if (dispatcher.searchCluster().wantedGroupSize() == 1) forceSinglePassGrouping(query); try (SearchInvoker invoker = getSearchInvoker(query)) { Result result = invoker.search(query, execution); diff --git a/container-search/src/main/java/com/yahoo/prelude/query/DotProductItem.java b/container-search/src/main/java/com/yahoo/prelude/query/DotProductItem.java index 455f81c8a90..ee41efd943c 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/DotProductItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/DotProductItem.java @@ -1,6 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.query; +import java.util.Map; + /** * A weighted set query item to be evaluated as a sparse dot product. * @@ -11,6 +13,7 @@ package com.yahoo.prelude.query; public class DotProductItem extends WeightedSetItem { public DotProductItem(String indexName) { super(indexName); } + public DotProductItem(String indexName, Map<Object, Integer> map) { super(indexName, map); } @Override public ItemType getItemType() { return ItemType.DOTPRODUCT; } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/Item.java b/container-search/src/main/java/com/yahoo/prelude/query/Item.java index c4978b2a378..467fbd3127c 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/Item.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/Item.java @@ -65,7 +65,7 @@ public abstract class Item implements Cloneable { public final int code; - private ItemType(int code) { + ItemType(int code) { this.code = code; } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/WandItem.java b/container-search/src/main/java/com/yahoo/prelude/query/WandItem.java index 89e85fe8f12..8cce8fb5720 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/WandItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/WandItem.java @@ -5,6 +5,7 @@ import com.yahoo.compress.IntegerCompressor; import com.yahoo.prelude.query.textualrepresentation.Discloser; import java.nio.ByteBuffer; +import java.util.Map; /** * A weighted set query item to be evaluated as a Wand with dot product scoring. @@ -33,6 +34,18 @@ public class WandItem extends WeightedSetItem { } /** + * Creates an empty WandItem. + * + * @param fieldName the name of the weighted set field to search with this WandItem. + * @param targetNumHits the target for minimum number of hits to produce by the backend search operator handling this WandItem. + * @param tokens the tokens to search for + */ + public WandItem(String fieldName, int targetNumHits, Map<Object, Integer> tokens) { + super(fieldName, tokens); + this.targetNumHits = targetNumHits; + } + + /** * Sets the initial score threshold used by the backend search operator handling this WandItem. * The score of a document must be larger than this threshold in order to be considered a match. * Default value is 0.0. diff --git a/container-search/src/main/java/com/yahoo/prelude/query/WeightedSetItem.java b/container-search/src/main/java/com/yahoo/prelude/query/WeightedSetItem.java index 0e74099174f..15354f5f7d3 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/WeightedSetItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/WeightedSetItem.java @@ -25,9 +25,8 @@ import java.util.Map; */ public class WeightedSetItem extends SimpleTaggableItem { - private String indexName = ""; - - private CopyOnWriteHashMap<Object,Integer> set = new CopyOnWriteHashMap<>(1000); + private String indexName; + private CopyOnWriteHashMap<Object,Integer> set; /** Creates an empty weighted set; note you must provide an index name up front */ public WeightedSetItem(String indexName) { @@ -36,6 +35,15 @@ public class WeightedSetItem extends SimpleTaggableItem { } else { this.indexName = indexName; } + set = new CopyOnWriteHashMap<>(1000); + } + public WeightedSetItem(String indexName, Map<Object, Integer> map) { + if (indexName == null) { + this.indexName = ""; + } else { + this.indexName = indexName; + } + set = new CopyOnWriteHashMap<>(map); } public Integer addToken(long value, int weight) { diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/RuleBase.java b/container-search/src/main/java/com/yahoo/prelude/semantics/RuleBase.java index ccc9c1d2f8f..feab8faf898 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/RuleBase.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/RuleBase.java @@ -26,43 +26,43 @@ public class RuleBase { private String source; /** The name of the automata file used, or null if none */ - protected String automataFileName=null; + protected String automataFileName = null; /** * True if this rule base is default. * The semantics of default is left to the surrounding framework */ - private boolean isDefault=false; + private boolean isDefault = false; - private List<ProductionRule> productionRules=new java.util.ArrayList<>(); + private final List<ProductionRule> productionRules = new java.util.ArrayList<>(); - private Map<String, NamedCondition> namedConditions=new java.util.LinkedHashMap<>(); + private Map<String, NamedCondition> namedConditions = new java.util.LinkedHashMap<>(); /** The analyzer used to do evaluations over this rule base */ - private RuleEngine analyzer=new RuleEngine(this); + private final RuleEngine analyzer = new RuleEngine(this); - private static final PhraseMatcher nullPhraseMatcher=PhraseMatcher.getNullMatcher(); + private static final PhraseMatcher nullPhraseMatcher = PhraseMatcher.getNullMatcher(); /** * The matcher using an automata to match terms and phrases prior to matching rules * or the null matcher if no matcher is used. */ - private PhraseMatcher phraseMatcher=nullPhraseMatcher; + private PhraseMatcher phraseMatcher = nullPhraseMatcher; /** * The names of the rule bases included indirectly or directly in this * Ordered by first to last included */ - private Set<String> includedNames=new java.util.LinkedHashSet<>(); + private final Set<String> includedNames = new java.util.LinkedHashSet<>(); /** * True if this uses an automata, even if an automata is not present right now. Useful to validate without * having automatas available */ - private boolean usesAutomata=false; + private boolean usesAutomata = false; /** Should we allow stemmed matches? */ - private boolean stemming=true; + private boolean stemming = true; /** Creates an empty rule base. TODO: Disallow */ public RuleBase() { @@ -82,8 +82,8 @@ public class RuleBase { * @throws ParseException if the rule file can not be parsed correctly * @throws RuleBaseException if the rule file contains inconsistencies */ - public static RuleBase createFromFile(String ruleFile,String automataFile) throws java.io.IOException, ParseException { - return new RuleImporter().importFile(ruleFile,automataFile); + public static RuleBase createFromFile(String ruleFile, String automataFile) throws java.io.IOException, ParseException { + return new RuleImporter().importFile(ruleFile, automataFile); } /** @@ -96,14 +96,14 @@ public class RuleBase { * @throws com.yahoo.prelude.semantics.parser.ParseException if the rule file can not be parsed correctly * @throws com.yahoo.prelude.semantics.RuleBaseException if the rule file contains inconsistencies */ - public static RuleBase createFromString(String name,String ruleString,String automataFile) throws java.io.IOException, ParseException { - RuleBase base=new RuleImporter().importString(ruleString,automataFile,new RuleBase()); + public static RuleBase createFromString(String name, String ruleString, String automataFile) throws java.io.IOException, ParseException { + RuleBase base = new RuleImporter().importString(ruleString, automataFile, new RuleBase()); base.setName(name); return base; } /** Set to true to enable stemmed matches. True by default */ - public void setStemming(boolean stemming) { this.stemming=stemming; } + public void setStemming(boolean stemming) { this.stemming = stemming; } /** Returns whether stemmed matches are allowed. True by default */ public boolean getStemming() { return stemming; } @@ -125,19 +125,19 @@ public class RuleBase { /** Rules are order based - they are included recursively depth first */ private void inlineIncluded() { // Re-add our own conditions last to - added later overrides - Map<String, NamedCondition> thisConditions=namedConditions; - namedConditions=new LinkedHashMap<>(); + Map<String, NamedCondition> thisConditions = namedConditions; + namedConditions = new LinkedHashMap<>(); - Set<RuleBase> included=new HashSet<>(); + Set<RuleBase> included = new HashSet<>(); included.add(this); - for (ListIterator<ProductionRule> i=productionRules.listIterator(); i.hasNext(); ) { - ProductionRule rule=i.next(); + for (ListIterator<ProductionRule> i = productionRules.listIterator(); i.hasNext(); ) { + ProductionRule rule = i.next(); if ( ! (rule instanceof IncludeDirective) ) continue; i.remove(); - RuleBase toInclude=((IncludeDirective)rule).getIncludedBase(); + RuleBase toInclude = ((IncludeDirective)rule).getIncludedBase(); if ( ! included.contains(toInclude)) - toInclude.inlineIn(this,i,included); + toInclude.inlineIn(this, i, included); } namedConditions.putAll(thisConditions); @@ -147,14 +147,14 @@ public class RuleBase { * Recursively include this and everything it includes into the given rule base. * Skips bases already included in this. */ - private void inlineIn(RuleBase receiver,ListIterator<ProductionRule> receiverRules,Set<RuleBase> included) { + private void inlineIn(RuleBase receiver, ListIterator<ProductionRule> receiverRules, Set<RuleBase> included) { if (included.contains(this)) return; included.add(this); - for (Iterator<ProductionRule> i=productionRules.iterator(); i.hasNext(); ) { - ProductionRule rule=i.next(); + for (Iterator<ProductionRule> i = productionRules.iterator(); i.hasNext(); ) { + ProductionRule rule = i.next(); if (rule instanceof IncludeDirective) - ((IncludeDirective)rule).getIncludedBase().inlineIn(receiver,receiverRules,included); + ((IncludeDirective)rule).getIncludedBase().inlineIn(receiver, receiverRules, included); else receiverRules.add(rule); } @@ -164,11 +164,11 @@ public class RuleBase { /** Adds a named condition which can be referenced by rules */ public void addCondition(NamedCondition namedCondition) { - namedConditions.put(namedCondition.getName(),namedCondition); + namedConditions.put(namedCondition.getName(), namedCondition); - Condition condition=namedCondition.getCondition(); - Condition superCondition=findIncludedCondition(namedCondition.getName()); - resolveSuper(condition,superCondition); + Condition condition = namedCondition.getCondition(); + Condition superCondition = findIncludedCondition(namedCondition.getName()); + resolveSuper(condition, superCondition); } private void resolveSuper(Condition condition,Condition superCondition) { @@ -176,24 +176,22 @@ public class RuleBase { ((SuperCondition)condition).setCondition(superCondition); } else if (condition instanceof CompositeCondition) { - for (Iterator<Condition> i=((CompositeCondition)condition).conditionIterator(); i.hasNext(); ) { - Condition subCondition=i.next(); - resolveSuper(subCondition,superCondition); + for (Iterator<Condition> i = ((CompositeCondition)condition).conditionIterator(); i.hasNext(); ) { + Condition subCondition = i.next(); + resolveSuper(subCondition, superCondition); } } } private Condition findIncludedCondition(String name) { - for (Iterator<ProductionRule> i=productionRules.iterator(); i.hasNext(); ) { - ProductionRule rule=i.next(); + for (Iterator<ProductionRule> i = productionRules.iterator(); i.hasNext(); ) { + ProductionRule rule = i.next(); if ( ! (rule instanceof IncludeDirective) ) continue; - RuleBase included=((IncludeDirective)rule).getIncludedBase(); - NamedCondition condition=included.getCondition(name); - if (condition!=null) return condition.getCondition(); + RuleBase included = ((IncludeDirective)rule).getIncludedBase(); + NamedCondition condition = included.getCondition(name); + if (condition != null) return condition.getCondition(); included.findIncludedCondition(name); - // FIXME: dead code commented out - // if (condition!=null) return condition.getCondition(); } return null; } @@ -212,8 +210,8 @@ public class RuleBase { * change, and then re-added */ public void setName(String name) { - Validator.ensureNotNull("Rule base name",name); - this.name=name; + Validator.ensureNotNull("Rule base name", name); + this.name = name; } /** Returns the name of this rule base. This is never null. */ @@ -230,27 +228,27 @@ public class RuleBase { if ( ! new File(automataFile).exists()) throw new IllegalArgumentException("Automata file '" + automataFile + "' " + "included in " + this + " not found"); - phraseMatcher=new PhraseMatcher(automataFile); + phraseMatcher = new PhraseMatcher(automataFile); phraseMatcher.setIgnorePluralForm(true); phraseMatcher.setMatchAll(true); phraseMatcher.setMatchPhraseItems(true); phraseMatcher.setMatchSingleItems(true); setPhraseMatcher(phraseMatcher); - this.automataFileName=automataFile; + this.automataFileName = automataFile; } /** Returns the name of the automata file used, or null if none */ public String getAutomataFile() { return automataFileName; } /** Sets whether this base is default, the semantics of default is left to the application */ - public void setDefault(boolean isDefault) { this.isDefault=isDefault; } + public void setDefault(boolean isDefault) { this.isDefault = isDefault; } /** Returns whether this base is default, the semantics of default is left to the application */ public boolean isDefault() { return isDefault; } /** Thread safely sets the phrase matcher to use in this, or null to not use a phrase matcher */ public synchronized void setPhraseMatcher(PhraseMatcher matcher) { - if (matcher==null) + if (matcher == null) this.phraseMatcher = nullPhraseMatcher; else this.phraseMatcher = matcher; @@ -282,7 +280,7 @@ public class RuleBase { * Set to truew if this uses an automata, even if an automata is not present right now. * Useful to validate without having automatas available */ - void setUsesAutomata(boolean usesAutomata) { this.usesAutomata=usesAutomata; } + void setUsesAutomata(boolean usesAutomata) { this.usesAutomata = usesAutomata; } // Note that included rules are added though a list iterator, not this */ public void addRule(ProductionRule productionRule) { @@ -399,13 +397,13 @@ public class RuleBase { public String toContentString() { StringBuilder b = new StringBuilder(); for (Iterator<ProductionRule> i = productionRules.iterator(); i.hasNext(); ) { - b.append(i.next().toString()); + b.append(i.next()); b.append("\n"); } b.append("\n"); b.append("\n"); for (Iterator<NamedCondition> i = namedConditions.values().iterator(); i.hasNext(); ) { - b.append(i.next().toString()); + b.append(i.next()); b.append("\n"); } return b.toString(); @@ -414,10 +412,10 @@ public class RuleBase { /** A placeholder for an included rule base until it is inlined */ private static class IncludeDirective extends ProductionRule { - private RuleBase includedBase; + private final RuleBase includedBase; public IncludeDirective(RuleBase ruleBase) { - this.includedBase=ruleBase; + this.includedBase = ruleBase; } public RuleBase getIncludedBase() { return includedBase; } @@ -425,7 +423,6 @@ public class RuleBase { /** Not used */ public String getSymbol() { return ""; } - } } diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/RuleBaseException.java b/container-search/src/main/java/com/yahoo/prelude/semantics/RuleBaseException.java index d851c2648d5..5108777d938 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/RuleBaseException.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/RuleBaseException.java @@ -6,7 +6,6 @@ package com.yahoo.prelude.semantics; * * @author bratseth */ -@SuppressWarnings("serial") public class RuleBaseException extends RuntimeException { public RuleBaseException(String message) { @@ -14,7 +13,7 @@ public class RuleBaseException extends RuntimeException { } public RuleBaseException(String message,Exception cause) { - super(message,cause); + super(message, cause); } } diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/RuleImporter.java b/container-search/src/main/java/com/yahoo/prelude/semantics/RuleImporter.java index 754d14ddcb4..ac643469ab6 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/RuleImporter.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/RuleImporter.java @@ -28,13 +28,13 @@ public class RuleImporter { * If this is set, imported rule bases are looked up in this config * otherwise, they are looked up as files */ - private SemanticRulesConfig config = null; + private SemanticRulesConfig config; /** * Ignore requests to read automata files. * Useful to validate rule bases without having automatas present */ - private boolean ignoreAutomatas = false; + private boolean ignoreAutomatas; /** * Ignore requests to include files. @@ -61,8 +61,8 @@ public class RuleImporter { } public RuleImporter(SemanticRulesConfig config, boolean ignoreAutomatas) { - this.config=config; - this.ignoreAutomatas=ignoreAutomatas; + this.config = config; + this.ignoreAutomatas = ignoreAutomatas; } public RuleImporter(SemanticRulesConfig config, boolean ignoreAutomatas, boolean ignoreIncludes) { @@ -79,7 +79,7 @@ public class RuleImporter { * @throws ParseException if the file does not contain a valid semantic rule set */ public RuleBase importFile(String fileName) throws IOException, ParseException { - return importFile(fileName,null); + return importFile(fileName, null); } /** @@ -90,8 +90,8 @@ public class RuleImporter { * @throws java.io.IOException if the file can not be read for some reason * @throws ParseException if the file does not contain a valid semantic rule set */ - public RuleBase importFile(String fileName,String automataFile) throws IOException, ParseException { - return importFile(fileName,automataFile,null); + public RuleBase importFile(String fileName, String automataFile) throws IOException, ParseException { + return importFile(fileName, automataFile, null); } /** @@ -99,27 +99,26 @@ public class RuleImporter { * * @param fileName the rule file to use * @param automataFile the automata file to use, or null to not use any - * @param ruleBase an existing rule base to import these rules into, or null - * to create a new + * @param ruleBase an existing rule base to import these rules into, or null to create a new * @throws java.io.IOException if the file can not be read for some reason * @throws ParseException if the file does not contain a valid semantic rule set */ - public RuleBase importFile(String fileName,String automataFile,RuleBase ruleBase) throws IOException, ParseException { - ruleBase=privateImportFile(fileName,automataFile,ruleBase); + public RuleBase importFile(String fileName, String automataFile, RuleBase ruleBase) throws IOException, ParseException { + ruleBase = privateImportFile(fileName, automataFile, ruleBase); ruleBase.initialize(); return ruleBase; } - public RuleBase privateImportFile(String fileName,String automataFile,RuleBase ruleBase) throws IOException, ParseException { - BufferedReader reader=null; + public RuleBase privateImportFile(String fileName, String automataFile, RuleBase ruleBase) throws IOException, ParseException { + BufferedReader reader = null; try { - reader= IOUtils.createReader(fileName, "utf-8"); - File file=new File(fileName); - String absoluteFileName=file.getAbsolutePath(); - if (ruleBase==null) - ruleBase=new RuleBase(); + reader = IOUtils.createReader(fileName, "utf-8"); + File file = new File(fileName); + String absoluteFileName = file.getAbsolutePath(); + if (ruleBase == null) + ruleBase = new RuleBase(); ruleBase.setName(stripLastName(file.getName())); - privateImportFromReader(reader,absoluteFileName,automataFile,ruleBase); + privateImportFromReader(reader, absoluteFileName, automataFile, ruleBase); return ruleBase; } finally { @@ -129,14 +128,14 @@ public class RuleImporter { /** Imports all the rule files (files ending by "sr") in the given directory */ public List<RuleBase> importDir(String ruleBaseDir) throws IOException, ParseException { - File ruleBaseDirFile=new File(ruleBaseDir); - if (!ruleBaseDirFile.exists()) + File ruleBaseDirFile = new File(ruleBaseDir); + if ( ! ruleBaseDirFile.exists()) throw new IOException("Rule base dir '" + ruleBaseDirFile.getAbsolutePath() + "' does not exist"); - File[] files=ruleBaseDirFile.listFiles(); + File[] files = ruleBaseDirFile.listFiles(); Arrays.sort(files); - List<RuleBase> ruleBases=new java.util.ArrayList<>(); + List<RuleBase> ruleBases = new java.util.ArrayList<>(); for (File file : files) { - if (!file.getName().endsWith(".sr")) continue; + if ( ! file.getName().endsWith(".sr")) continue; RuleBase base = importFile(file.getAbsolutePath()); ruleBases.add(base); } @@ -144,50 +143,50 @@ public class RuleImporter { } /** Read and include a rule base in another */ - public void include(String ruleBaseName,RuleBase ruleBase) throws java.io.IOException, ParseException { + public void include(String ruleBaseName, RuleBase ruleBase) throws java.io.IOException, ParseException { if (ignoreIncludes) return; RuleBase include; - if (config==null) { - include=privateImportFromDirectory(ruleBaseName,ruleBase); + if (config == null) { + include = privateImportFromDirectory(ruleBaseName, ruleBase); } else { - include=privateImportFromConfig(ruleBaseName); + include = privateImportFromConfig(ruleBaseName); } ruleBase.include(include); } /** Returns an unitialized rule base */ - private RuleBase privateImportFromDirectory(String ruleBaseName,RuleBase ruleBase) throws IOException, ParseException { + private RuleBase privateImportFromDirectory(String ruleBaseName, RuleBase ruleBase) throws IOException, ParseException { RuleBase include = new RuleBase(); - String includeDir=new File(ruleBase.getSource()).getParentFile().getAbsolutePath(); + String includeDir = new File(ruleBase.getSource()).getParentFile().getAbsolutePath(); if (!ruleBaseName.endsWith(".sr")) - ruleBaseName=ruleBaseName + ".sr"; - File importFile=new File(includeDir,ruleBaseName); - if (!importFile.exists()) + ruleBaseName = ruleBaseName + ".sr"; + File importFile = new File(includeDir, ruleBaseName); + if ( ! importFile.exists()) throw new IOException("No file named '" + shortenPath(importFile.getPath()) + "'"); - return privateImportFile(importFile.getPath(),null,include); + return privateImportFile(importFile.getPath(), null, include); } /** Returns an unitialized rule base */ private RuleBase privateImportFromConfig(String ruleBaseName) throws IOException, ParseException { - SemanticRulesConfig.Rulebase ruleBaseConfig=findRuleBaseConfig(config,ruleBaseName); - if (ruleBaseConfig==null) - ruleBaseConfig=findRuleBaseConfig(config,stripLastName(ruleBaseName)); - if (ruleBaseConfig==null) + SemanticRulesConfig.Rulebase ruleBaseConfig = findRuleBaseConfig(config,ruleBaseName); + if (ruleBaseConfig == null) + ruleBaseConfig = findRuleBaseConfig(config, stripLastName(ruleBaseName)); + if (ruleBaseConfig == null) throw new ParseException("Could not find included rule base '" + ruleBaseName + "'"); return privateImportConfig(ruleBaseConfig); } - private SemanticRulesConfig.Rulebase findRuleBaseConfig(SemanticRulesConfig config,String ruleBaseName) { + private SemanticRulesConfig.Rulebase findRuleBaseConfig(SemanticRulesConfig config, String ruleBaseName) { for (Object aRulebase : config.rulebase()) { - SemanticRulesConfig.Rulebase ruleBaseConfig = (SemanticRulesConfig.Rulebase) aRulebase; + SemanticRulesConfig.Rulebase ruleBaseConfig = (SemanticRulesConfig.Rulebase)aRulebase; if (ruleBaseConfig.name().equals(ruleBaseName)) return ruleBaseConfig; } return null; } - public void setAutomata(RuleBase base,String automata) { + public void setAutomata(RuleBase base, String automata) { if (ignoreAutomatas) base.setUsesAutomata(true); // Stop it from failing on automata condition references else @@ -195,9 +194,9 @@ public class RuleImporter { } static String stripLastName(String fileName) { - int lastDotIndex=fileName.lastIndexOf("."); - if (lastDotIndex<0) return fileName; - return fileName.substring(0,lastDotIndex); + int lastDotIndex = fileName.lastIndexOf("."); + if (lastDotIndex < 0) return fileName; + return fileName.substring(0, lastDotIndex); } public RuleBase importString(String string, String automataFile) throws IOException, ParseException { @@ -217,22 +216,23 @@ public class RuleImporter { } public RuleBase importConfig(SemanticRulesConfig.Rulebase ruleBaseConfig) throws IOException, ParseException { - RuleBase ruleBase=privateImportConfig(ruleBaseConfig); + RuleBase ruleBase = privateImportConfig(ruleBaseConfig); ruleBase.initialize(); return ruleBase; } /** Imports an unitialized rule base */ - public RuleBase privateImportConfig(SemanticRulesConfig.Rulebase ruleBaseConfig) throws IOException, ParseException { - if (config==null) throw new IllegalStateException("Must initialize with config if importing from config"); + public RuleBase privateImportConfig(SemanticRulesConfig.Rulebase ruleBaseConfig) throws ParseException { + if (config == null) throw new IllegalStateException("Must initialize with config if importing from config"); RuleBase ruleBase = new RuleBase(); ruleBase.setName(ruleBaseConfig.name()); - return privateImportFromReader(new StringReader(ruleBaseConfig.rules()),"semantic-rules.cfg", - ruleBaseConfig.automata(),ruleBase); + return privateImportFromReader(new StringReader(ruleBaseConfig.rules()), + "semantic-rules.cfg", + ruleBaseConfig.automata(),ruleBase); } - public RuleBase importFromReader(Reader reader,String sourceInfo,String automataFile) throws ParseException { - return importFromReader(reader,sourceInfo,automataFile,null); + public RuleBase importFromReader(Reader reader, String sourceInfo, String automataFile) throws ParseException { + return importFromReader(reader, sourceInfo, automataFile, null); } /** @@ -245,7 +245,7 @@ public class RuleImporter { * @throws ParseException if the reader contains illegal rule syntax */ public RuleBase importFromReader(Reader reader, String sourceName, String automataFile, RuleBase ruleBase) throws ParseException { - ruleBase=privateImportFromReader(reader, sourceName, automataFile,ruleBase); + ruleBase = privateImportFromReader(reader, sourceName, automataFile, ruleBase); ruleBase.initialize(); return ruleBase; } @@ -253,19 +253,19 @@ public class RuleImporter { /** Returns an unitialized rule base */ public RuleBase privateImportFromReader(Reader reader, String sourceName, String automataFile, RuleBase ruleBase) throws ParseException { try { - if (ruleBase==null) { - ruleBase=new RuleBase(); + if (ruleBase == null) { + ruleBase = new RuleBase(); if (sourceName == null) sourceName = "anonymous"; ruleBase.setName(sourceName); } - ruleBase.setSource(sourceName.replace('\\','/')); + ruleBase.setSource(sourceName.replace('\\', '/')); new SemanticsParser(reader).semanticRules(ruleBase, this); - if (automataFile!=null && !automataFile.isEmpty()) - ruleBase.setAutomataFile(automataFile.replace('\\','/')); + if (automataFile != null && !automataFile.isEmpty()) + ruleBase.setAutomataFile(automataFile.replace('\\', '/')); return ruleBase; } catch (Throwable t) { // also catches token mgr errors - ParseException p=new ParseException("Could not parse '" + shortenPath(sourceName) + "'"); + ParseException p = new ParseException("Could not parse '" + shortenPath(sourceName) + "'"); p.initCause(t); throw p; } @@ -277,8 +277,8 @@ public class RuleImporter { * (if rules/ is present, these rules are read from an applicatino package) */ private static String shortenPath(String path) { - int rulesIndex=path.indexOf("rules/"); - if (rulesIndex<0) return path; + int rulesIndex = path.indexOf("rules/"); + if (rulesIndex < 0) return path; return path.substring(rulesIndex); } diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/engine/Evaluation.java b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/Evaluation.java index b1b317379ff..989f3040cc7 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/engine/Evaluation.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/Evaluation.java @@ -244,15 +244,16 @@ public class Evaluation { * @param desiredParentType the desired type of the composite which contains item when this returns */ public void insertItem(Item item, CompositeItem parent, int index, TermType desiredParentType) { - if (parent == null) { // TODO: Accommodate for termtype in this case too - query.getModel().getQueryTree().setRoot(item); - + if (isEmpty(parent)) { + CompositeItem newParent = (CompositeItem)desiredParentType.createItemClass(); + newParent.addItem(item); + query.getModel().getQueryTree().setRoot(newParent); return; } - if (parent.getItemCount()>0 && parent instanceof QueryTree && parent.getItem(0) instanceof CompositeItem) { + if (parent.getItemCount() > 0 && parent instanceof QueryTree && parent.getItem(0) instanceof CompositeItem) { // combine with the existing root instead - parent=(CompositeItem)parent.getItem(0); + parent = (CompositeItem)parent.getItem(0); if (index == 1) { // that means adding it after the existing root index = parent.getItemCount(); } @@ -260,11 +261,25 @@ public class Evaluation { if (( desiredParentType == TermType.DEFAULT || desiredParentType.hasItemClass(parent.getClass()) ) && equalIndexNameIfParentIsPhrase(item, parent)) { - addItem(parent,index,item,desiredParentType); + addItem(parent, index, item, desiredParentType); } - else { + else if (incompatible(desiredParentType, parent)) { insertIncompatibleItem(item, parent, query, desiredParentType); } + else { + insertIncompatibleItemAsParent(item, parent, query, desiredParentType); + } + } + + private boolean isEmpty(Item item) { + if (item == null) return true; + if (item instanceof QueryTree && ((QueryTree) item).isEmpty()) return true; + return false; + } + + /** Returns true if the desired type cannot have childCandidate as a child */ + private boolean incompatible(TermType desiredParentType, Item childCandidate) { + return desiredParentType == TermType.EQUIV && childCandidate.getItemType() != Item.ItemType.EQUIV; } private void addItem(CompositeItem parent, int index, Item item, TermType desiredParentType) { @@ -273,27 +288,27 @@ public class Evaluation { parent.setItem(0, item); } else if (index<=1 && !(parent.getItem(0) instanceof CompositeItem)) { // Case 2: The positive must become a composite - CompositeItem positiveComposite=(CompositeItem)desiredParentType.createItemClass(); + CompositeItem positiveComposite = (CompositeItem)desiredParentType.createItemClass(); positiveComposite.addItem(parent.getItem(0)); - positiveComposite.addItem(index,item); - parent.setItem(0,positiveComposite); + positiveComposite.addItem(index, item); + parent.setItem(0, positiveComposite); } else if (parent.getItem(0)!=null && parent.getItem(0) instanceof CompositeItem // Case 3: Add to the positive composite - && index<=((CompositeItem)parent.getItem(0)).getItemCount()) { - ((CompositeItem)parent.getItem(0)).addItem(index,item); + && index <= ((CompositeItem)parent.getItem(0)).getItemCount()) { + ((CompositeItem)parent.getItem(0)).addItem(index, item); } else { // Case 4: Add negative - parent.addItem(index,item); + parent.addItem(index, item); } } - else if (parent.getItemCount()>0 && parent instanceof QueryTree) { - CompositeItem composite=(CompositeItem)desiredParentType.createItemClass(); + else if (parent.getItemCount() > 0 && parent instanceof QueryTree) { + CompositeItem composite = (CompositeItem)desiredParentType.createItemClass(); composite.addItem(parent.getItem(0)); - composite.addItem(index,item); - parent.setItem(0,composite); + composite.addItem(index, item); + parent.setItem(0, composite); } else { - parent.addItem(index,item); + parent.addItem(index, item); } } @@ -305,32 +320,39 @@ public class Evaluation { return ((PhraseItem)parent).getIndexName().equals(((IndexedItem)item).getIndexName()); } - private void insertIncompatibleItem(Item item,CompositeItem parent,Query query,TermType desiredParentType) { + private void insertIncompatibleItem(Item item, CompositeItem parent, Query query, TermType desiredParentType) { + CompositeItem newParent; + if (desiredParentType == TermType.DEFAULT) + newParent = new AndItem(); + else + newParent = (CompositeItem)desiredParentType.createItemClass(); + + newParent.addItem(item); + parent.addItem(newParent); + } + + private void insertIncompatibleItemAsParent(Item item, CompositeItem parent, Query query, TermType desiredParentType) { // Create new parent CompositeItem newParent; - if (desiredParentType==TermType.DEFAULT) - newParent=new AndItem(); + if (desiredParentType == TermType.DEFAULT) + newParent = new AndItem(); else - newParent=(CompositeItem)desiredParentType.createItemClass(); + newParent = (CompositeItem)desiredParentType.createItemClass(); // Save previous parent parent - CompositeItem parentsParent=parent.getParent(); + CompositeItem parentsParent = parent.getParent(); // Add items to new parent newParent.addItem(parent); newParent.addItem(item); // Insert new parent as root or child of old parents parent - if (parentsParent==null) { + if (parentsParent == null) { query.getModel().getQueryTree().setRoot(newParent); } else { - int parentIndex=0; - if (parentsParent!=null) { - parentIndex=parentsParent.getItemIndex(parent); - } - parentsParent.setItem(parentIndex,newParent); + parentsParent.setItem(parentsParent.getItemIndex(parent), newParent); } } @@ -338,19 +360,19 @@ public class Evaluation { if (first instanceof NullItem) { return second; } else if (first instanceof NotItem) { - NotItem notItem=(NotItem)first; - if (termType==TermType.NOT) { + NotItem notItem = (NotItem)first; + if (termType == TermType.NOT) { notItem.addNegativeItem(second); } else { - Item newPositive=combineItems(notItem.getPositiveItem(),second,termType); + Item newPositive = combineItems(notItem.getPositiveItem(), second, termType); notItem.setPositiveItem(newPositive); } return notItem; } else if (first instanceof CompositeItem) { - CompositeItem composite=(CompositeItem)first; - CompositeItem combined=createType(termType); + CompositeItem composite = (CompositeItem)first; + CompositeItem combined = createType(termType); if (combined.getClass().equals(composite.getClass())) { composite.addItem(second); return composite; @@ -362,7 +384,7 @@ public class Evaluation { } } else if (first instanceof TermItem) { - CompositeItem combined=createType(termType); + CompositeItem combined = createType(termType); combined.addItem(first); combined.addItem(second); return combined; diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/engine/Match.java b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/Match.java index 9e7c761deba..a87338879fe 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/engine/Match.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/Match.java @@ -6,6 +6,8 @@ import com.yahoo.prelude.query.Item; import com.yahoo.prelude.query.TermItem; import com.yahoo.prelude.query.WordItem; +import java.util.Objects; + /** * A match * @@ -14,15 +16,15 @@ import com.yahoo.prelude.query.WordItem; public class Match { /** The start position of this match */ - private int position; + private final int position; - private TermItem item; + private final TermItem item; /** The string to replace the match by, usually item.getIndexedString() */ - private String replaceValue; + private final String replaceValue; /** The parent of the matched item */ - private CompositeItem parent=null; + private final CompositeItem parent; /** * Creates a match @@ -56,23 +58,25 @@ public class Match { */ public CompositeItem getParent() { return parent; } - public int hashCode() { - return - 17*item.getIndexedString().hashCode()+ - 33*item.getIndexName().hashCode(); - } - /** Returns a new item representing this match */ public Item toItem(String label) { - return new WordItem(getReplaceValue(),label); + var newItem = new WordItem(getReplaceValue(), label); + newItem.setWeight(item.getWeight()); + return newItem; + } + + @Override + public int hashCode() { + return Objects.hash(item.getIndexedString(), item.getIndexName()); } + @Override public boolean equals(Object o) { if (! (o instanceof Match)) return false; - Match other=(Match)o; - if (other.position!=position) return false; - if (!other.item.equals(item)) return false; + Match other = (Match)o; + if (other.position != position) return false; + if ( ! other.item.equals(item)) return false; return true; } diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/engine/ReferencedMatches.java b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/ReferencedMatches.java index 274528f1fb3..f21f861a801 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/engine/ReferencedMatches.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/ReferencedMatches.java @@ -14,12 +14,12 @@ import com.yahoo.prelude.query.PhraseItem; */ public class ReferencedMatches { - private String contextName; + private final String contextName; - private List<Match> matches=new java.util.ArrayList<>(1); + private final List<Match> matches = new java.util.ArrayList<>(1); public ReferencedMatches(String contextName) { - this.contextName=contextName; + this.contextName = contextName; } public void addMatch(Match match) { @@ -38,12 +38,12 @@ public class ReferencedMatches { * @param label the label of the matches */ public Item toItem(String label) { - if (matches.size()==0) return null; - if (matches.size()==1) return matches.get(0).toItem(label); + if (matches.size() == 0) return null; + if (matches.size() == 1) return matches.get(0).toItem(label); - PhraseItem phrase=new PhraseItem(); // TODO: Somehow allow AND items instead here + PhraseItem phrase = new PhraseItem(); // TODO: Somehow allow AND items instead here phrase.setIndexName(label); - for (Iterator<Match> i=matches.iterator(); i.hasNext(); ) { + for (Iterator<Match> i = matches.iterator(); i.hasNext(); ) { phrase.addItem(i.next().toItem(label)); } return phrase; diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/engine/RuleEvaluation.java b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/RuleEvaluation.java index 29a781b1e68..09fcb6a424f 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/engine/RuleEvaluation.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/RuleEvaluation.java @@ -10,7 +10,7 @@ import com.yahoo.prelude.semantics.rule.ProductionRule; import java.util.*; /** - * A particular evalutation of a particular rule. + * A particular evaluation of a particular rule. * * @author bratseth */ @@ -23,7 +23,7 @@ public class RuleEvaluation { // Remember that whenever state is added to this class, you // must consider whether/how to make that state backtrackable - // by savinginformation in choicepoint.state + // by saving information in choicepoint.state /** The items to match in this evaluation */ private List<FlattenedItem> items; @@ -41,12 +41,12 @@ public class RuleEvaluation { private String currentContext; /** A list of referencedMatches */ - private List<ReferencedMatches> referencedMatchesList = new java.util.ArrayList<>(); + private final List<ReferencedMatches> referencedMatchesList = new java.util.ArrayList<>(); - private List<Match> nonreferencedMatches = new java.util.ArrayList<>(); + private final List<Match> nonreferencedMatches = new java.util.ArrayList<>(); /** The evaluation owning this */ - private Evaluation evaluation; + private final Evaluation evaluation; /** The choice points saved in this evaluation */ private Stack<Choicepoint> choicepoints = null; @@ -82,7 +82,7 @@ public class RuleEvaluation { choicepoints.clear(); } - public void setMatchReferences(Set<String> matchReferences) { this.matchReferences=matchReferences; } + public void setMatchReferences(Set<String> matchReferences) { this.matchReferences = matchReferences; } /** * <p>Calculates an id which is unique for each match (the totality of the matched terms) @@ -96,23 +96,23 @@ public class RuleEvaluation { * we add other kinds of conditions.</p> */ int calculateMatchDigest(ProductionRule rule) { - int matchDigest=rule.hashCode(); - int matchCounter=1; - for (Iterator<ReferencedMatches> i=referencedMatchesList.iterator(); i.hasNext(); ) { - ReferencedMatches matches=i.next(); - int termCounter=0; - for (Iterator<Match> j=matches.matchIterator(); j.hasNext(); ) { - Match match=j.next(); - matchDigest=7*matchDigest*matchCounter+ - 71*termCounter+ - match.hashCode(); + int matchDigest = rule.hashCode(); + int matchCounter = 1; + for (Iterator<ReferencedMatches> i = referencedMatchesList.iterator(); i.hasNext(); ) { + ReferencedMatches matches = i.next(); + int termCounter = 0; + for (Iterator<Match> j = matches.matchIterator(); j.hasNext(); ) { + Match match = j.next(); + matchDigest = 7 * matchDigest * matchCounter+ + 71 * termCounter + + match.hashCode(); termCounter++; } matchCounter++; } - for (Iterator<Match> i=nonreferencedMatches.iterator(); i.hasNext(); ) { - Match match=i.next(); - matchDigest=7*matchDigest*matchCounter+match.hashCode(); + for (Iterator<Match> i = nonreferencedMatches.iterator(); i.hasNext(); ) { + Match match = i.next(); + matchDigest = 7 * matchDigest * matchCounter + match.hashCode(); matchCounter++; } return matchDigest; @@ -139,7 +139,7 @@ public class RuleEvaluation { /** Sets the current position */ public void setPosition(int position) { - this.position=position; + this.position = position; } /** Returns the total number of items to match in this evaluation */ @@ -151,21 +151,21 @@ public class RuleEvaluation { public Object getValue() { return value; } /** Sets the last value returned by a condition in this evaluatiino, or null */ - public void setValue(Object value) { this.value=value; } + public void setValue(Object value) { this.value = value; } /** Returns whether we are evaluating inside a condition which inverts the truth value */ public boolean isInNegation() { return inNegation; } /** sets whether we are evaluating inside a condition which inverts the truth value */ - public void setInNegation(boolean inNegation) { this.inNegation=inNegation; } + public void setInNegation(boolean inNegation) { this.inNegation = inNegation; } /** Returns the current position into the terms this evaluates over */ public int getPosition() { return position; } /** Sets a new current label and returns the previous one */ public String setCurrentLabel(String currentLabel) { - String oldLabel=currentLabel; - this.currentLabel=currentLabel; + String oldLabel = currentLabel; + this.currentLabel = currentLabel; return oldLabel; } @@ -179,8 +179,8 @@ public class RuleEvaluation { public FlattenedItem next() { position++; - if (position>=items.size()) { - position=items.size(); + if (position >= items.size()) { + position = items.size(); return null; } @@ -189,17 +189,16 @@ public class RuleEvaluation { // TODO: Simplistic yet. Nedd to support context nesting etc. public void entering(String context) { - if (context==null) return; - if (matchReferences!=null && matchReferences.contains(context)) - currentContext=context; - + if (context == null) return; + if (matchReferences != null && matchReferences.contains(context)) + currentContext = context; } public void leaving(String context) { - if (context==null) return; - if (currentContext==null) return; + if (context == null) return; + if (currentContext == null) return; if (currentContext.equals(context)) - currentContext=null; + currentContext = null; } /** @@ -269,7 +268,7 @@ public class RuleEvaluation { * @param termType the kind of item to index, this decides the resulting structure */ public void insertItem(Item item, CompositeItem parent, int index, TermType termType) { - evaluation.insertItem(item,parent,index,termType); + evaluation.insertItem(item, parent, index, termType); } /** Returns a read-only view of the items of this */ @@ -282,7 +281,7 @@ public class RuleEvaluation { } public void trace(int level,String string) { - evaluation.trace(level,string); + evaluation.trace(level, string); } public int getTraceLevel() { @@ -304,24 +303,24 @@ public class RuleEvaluation { * @param create true to create this choicepoint if it is missing * @return the choicepoint, or null if not present, and create is false */ - public Choicepoint getChoicepoint(Condition condition,boolean create) { - if (choicepoints==null) { - if (!create) return null; - choicepoints=new java.util.Stack<>(); + public Choicepoint getChoicepoint(Condition condition, boolean create) { + if (choicepoints == null) { + if ( ! create) return null; + choicepoints = new java.util.Stack<>(); } Choicepoint choicepoint=lookupChoicepoint(condition); - if (choicepoint==null) { - if (!create) return null; - choicepoint=new Choicepoint(this,condition); + if (choicepoint == null) { + if ( ! create) return null; + choicepoint = new Choicepoint(this, condition); choicepoints.push(choicepoint); } return choicepoint; } private Choicepoint lookupChoicepoint(Condition condition) { - for (Iterator<Choicepoint> i=choicepoints.iterator(); i.hasNext(); ) { - Choicepoint choicepoint=i.next(); - if (condition==choicepoint.getCondition()) + for (Iterator<Choicepoint> i = choicepoints.iterator(); i.hasNext(); ) { + Choicepoint choicepoint = i.next(); + if (condition == choicepoint.getCondition()) return choicepoint; } return null; @@ -337,8 +336,8 @@ public class RuleEvaluation { /** Remove all the terms recognized by this match */ public void removeMatches(ReferencedMatches matches) { - for (Iterator<Match> i=matches.matchIterator(); i.hasNext(); ) { - Match match=i.next(); + for (Iterator<Match> i = matches.matchIterator(); i.hasNext(); ) { + Match match = i.next(); removeItemByIdentity(match.getItem()); } } diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/LiteralPhraseProduction.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/LiteralPhraseProduction.java index cad753a570b..d06c4144fbc 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/LiteralPhraseProduction.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/LiteralPhraseProduction.java @@ -19,7 +19,7 @@ import com.yahoo.protect.Validator; */ public class LiteralPhraseProduction extends TermProduction { - private List<String> terms=new ArrayList<>(); + private final List<String> terms = new ArrayList<>(); /** Creates a new produced literal phrase */ public LiteralPhraseProduction() { @@ -45,20 +45,20 @@ public class LiteralPhraseProduction extends TermProduction { public List<String> getTerms() { return Collections.unmodifiableList(terms); } public void produce(RuleEvaluation e,int offset) { - PhraseItem newPhrase=new PhraseItem(); + PhraseItem newPhrase = new PhraseItem(); newPhrase.setIndexName(getLabel()); for (String term : terms) newPhrase.addItem(new WordItem(term)); if (replacing) { - Match matched=e.getNonreferencedMatch(0); - insertMatch(e,matched,newPhrase,offset); + Match matched = e.getNonreferencedMatch(0); + insertMatch(e, matched, newPhrase, offset); } else { newPhrase.setWeight(getWeight()); - if (e.getTraceLevel()>=6) - e.trace(6,"Adding '" + newPhrase + "'"); - e.addItem(newPhrase,getTermType()); + if (e.getTraceLevel() >= 6) + e.trace(6, "Adding '" + newPhrase + "'"); + e.addItem(newPhrase, getTermType()); } } @@ -67,8 +67,8 @@ public class LiteralPhraseProduction extends TermProduction { } private String getSpaceSeparated(List<String> terms) { - StringBuilder builder=new StringBuilder(); - for (Iterator<String> i=terms.iterator(); i.hasNext(); ) { + StringBuilder builder = new StringBuilder(); + for (Iterator<String> i = terms.iterator(); i.hasNext(); ) { builder.append(i.next()); if (i.hasNext()) builder.append(" "); diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/LiteralTermProduction.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/LiteralTermProduction.java index 69a4a87c04e..66d1df690bb 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/LiteralTermProduction.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/LiteralTermProduction.java @@ -44,31 +44,32 @@ public class LiteralTermProduction extends TermProduction { * @param literal this term word * @param termType the type of term to produce */ - public LiteralTermProduction(String label,String literal, TermType termType) { - super(label,termType); + public LiteralTermProduction(String label, String literal, TermType termType) { + super(label, termType); setLiteral(literal); } /** The literal term value, never null */ public void setLiteral(String literal) { - Validator.ensureNotNull("A produced term",literal); + Validator.ensureNotNull("A produced term", literal); this.literal=literal; } /** Returns the term word produced, never null */ public String getLiteral() { return literal; } - public void produce(RuleEvaluation e,int offset) { - WordItem newItem=new WordItem(literal,getLabel()); + public void produce(RuleEvaluation e, int offset) { + WordItem newItem = new WordItem(literal, getLabel()); if (replacing) { - Match matched=e.getNonreferencedMatch(0); - insertMatch(e,matched,newItem,offset); + Match matched = e.getNonreferencedMatch(0); + newItem.setWeight(matched.getItem().getWeight()); + insertMatch(e, matched, newItem, offset); } else { newItem.setWeight(getWeight()); - if (e.getTraceLevel()>=6) - e.trace(6,"Adding '" + newItem + "'"); - e.addItem(newItem,getTermType()); + if (e.getTraceLevel() >= 6) + e.trace(6, "Adding '" + newItem + "'"); + e.addItem(newItem, getTermType()); } } diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/Production.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/Production.java index 3073e2f7727..716622a4849 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/Production.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/Production.java @@ -33,7 +33,7 @@ public abstract class Production { public void setPosition(int position) { this.position = position; } /** Sets the weight of this production as a percentage (default is 100) */ - public void setWeight(int weight) { this.weight=weight; } + public void setWeight(int weight) { this.weight = weight; } /** Returns the weight of this production as a percentage (default is 100) */ public int getWeight() { return weight; } diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ProductionList.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ProductionList.java index 5b973228b1f..49d15710cd7 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ProductionList.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ProductionList.java @@ -15,10 +15,10 @@ import com.yahoo.prelude.semantics.engine.RuleEvaluation; */ public class ProductionList { - private List<Production> productions =new java.util.ArrayList<>(); + private final List<Production> productions = new java.util.ArrayList<>(); /** True to replace by the production, false to add it */ - private boolean replacing=true; + private boolean replacing = true; public void addProduction(Production term) { term.setReplacing(replacing); @@ -28,12 +28,12 @@ public class ProductionList { /** True to replace, false to add, default true */ void setReplacing(boolean replacing) { - for (Iterator<Production> i=productions.iterator(); i.hasNext(); ) { - Production production=i.next(); + for (Iterator<Production> i = productions.iterator(); i.hasNext(); ) { + Production production = i.next(); production.setReplacing(replacing); } - this.replacing=replacing; + this.replacing = replacing; } /** Returns an unmodifiable view of the productions in this */ @@ -42,22 +42,22 @@ public class ProductionList { public int getTermCount() { return productions.size(); } void addMatchReferences(Set<String> matchReferences) { - for (Iterator<Production> i=productions.iterator(); i.hasNext(); ) { - Production term=i.next(); + for (Iterator<Production> i = productions.iterator(); i.hasNext(); ) { + Production term = i.next(); term.addMatchReferences(matchReferences); } } public void produce(RuleEvaluation e) { - for (int i=0; i<productions.size(); i++) { - productions.get(i).produce(e,i); + for (int i = 0; i < productions.size(); i++) { + productions.get(i).produce(e, i); } } public String toString() { - StringBuilder buffer=new StringBuilder(); - for (Iterator<Production> i=productions.iterator(); i.hasNext(); ) { - buffer.append(i.next().toString()); + StringBuilder buffer = new StringBuilder(); + for (Iterator<Production> i = productions.iterator(); i.hasNext(); ) { + buffer.append(i.next()); if (i.hasNext()) buffer.append(" "); } diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ReferenceTermProduction.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ReferenceTermProduction.java index d4593a5dbb9..e1bb8ff102d 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ReferenceTermProduction.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ReferenceTermProduction.java @@ -48,7 +48,7 @@ public class ReferenceTermProduction extends TermProduction { * @param label the label of the produced term * @param reference the label of the condition this should take it's value from */ - public ReferenceTermProduction(String label,String reference) { + public ReferenceTermProduction(String label, String reference) { super(label); setReference(reference); } @@ -60,15 +60,15 @@ public class ReferenceTermProduction extends TermProduction { * @param reference the label of the condition this should take it's value from * @param termType the type of term to produce */ - public ReferenceTermProduction(String label,String reference, TermType termType) { - super(label,termType); + public ReferenceTermProduction(String label, String reference, TermType termType) { + super(label, termType); setReference(reference); } /** The label of the condition this should take its value from, never null */ public void setReference(String reference) { Validator.ensureNotNull("reference name of a produced reference term",reference); - this.reference =reference; + this.reference = reference; } /** Returns the label of the condition this should take its value from, never null */ @@ -78,29 +78,29 @@ public class ReferenceTermProduction extends TermProduction { matchReferences.add(reference); } - public void produce(RuleEvaluation e,int offset) { - ReferencedMatches referencedMatches=e.getReferencedMatches(reference); - if (referencedMatches==null) + public void produce(RuleEvaluation e, int offset) { + ReferencedMatches referencedMatches = e.getReferencedMatches(reference); + if (referencedMatches == null) throw new EvaluationException("Referred match '" + reference + "' not found"); if (replacing) { - replaceMatches(e,referencedMatches); + replaceMatches(e, referencedMatches); } else { - addMatches(e,referencedMatches); + addMatches(e, referencedMatches); } } - public void replaceMatches(RuleEvaluation e,ReferencedMatches referencedMatches) { - Item referencedItem=referencedMatches.toItem(getLabel()); - if (referencedItem==null) return; + public void replaceMatches(RuleEvaluation e, ReferencedMatches referencedMatches) { + Item referencedItem = referencedMatches.toItem(getLabel()); + if (referencedItem == null) return; e.removeMatches(referencedMatches); - insertMatch(e, referencedMatches.matchIterator().next(),referencedItem,0); + insertMatch(e, referencedMatches.matchIterator().next(), referencedItem, 0); } - private void addMatches(RuleEvaluation e,ReferencedMatches referencedMatches) { - Item referencedItem=referencedMatches.toItem(getLabel()); - if (referencedItem==null) return; - e.addItem(referencedItem,getTermType()); + private void addMatches(RuleEvaluation e, ReferencedMatches referencedMatches) { + Item referencedItem = referencedMatches.toItem(getLabel()); + if (referencedItem == null) return; + e.addItem(referencedItem, getTermType()); } public String toInnerTermString() { diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ReplacingProductionRule.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ReplacingProductionRule.java index dfa423ec889..70151ea479b 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ReplacingProductionRule.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ReplacingProductionRule.java @@ -14,7 +14,7 @@ public class ReplacingProductionRule extends ProductionRule { /** Carries out the production of this rule */ public void produce(RuleEvaluation e) { removeNonreferencedMatches(e); - if (e.getTraceLevel()>=5) { + if (e.getTraceLevel() >= 5) { e.trace(5,"Removed terms to get '" + e.getEvaluation().getQuery().getModel().getQueryTree().getRoot() + "', will add terms"); } super.produce(e); @@ -22,16 +22,16 @@ public class ReplacingProductionRule extends ProductionRule { /** Remove items until there's only one item left */ private void removeNonreferencedMatches(RuleEvaluation e) { - int itemCount=e.getEvaluation().getQuerySize(); + int itemCount = e.getEvaluation().getQuerySize(); // Remove items backwards to ease index handling - for (int i=e.getNonreferencedMatchCount()-1; i>=0; i--) { + for (int i = e.getNonreferencedMatchCount() - 1; i >= 0; i--) { // Ensure we don't produce an empty query - if (getProduction().getTermCount()==0 && itemCount==1) + if (getProduction().getTermCount() == 0 && itemCount == 1) break; itemCount--; - Match match=e.getNonreferencedMatch(i); + Match match = e.getNonreferencedMatch(i); match.getItem().getParent().removeItem(match.getPosition()); } } diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/TermProduction.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/TermProduction.java index 847014ff646..d1a74a991da 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/TermProduction.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/TermProduction.java @@ -51,7 +51,7 @@ public abstract class TermProduction extends Production { /** Sets the term type to produce */ public void setTermType(TermType termType) { - Validator.ensureNotNull("Type of produced Term",termType); + Validator.ensureNotNull("Type of produced Term", termType); this.termType = termType; } @@ -60,19 +60,21 @@ public abstract class TermProduction extends Production { * TODO: Move to ruleevaluation */ protected void insertMatch(RuleEvaluation e, Match matched, Item newItem, int offset) { - newItem.setWeight(getWeight()); - int insertPosition=matched.getPosition()+offset; + if (getWeight() != 100) + newItem.setWeight(getWeight()); + int insertPosition = matched.getPosition()+offset; // This check is necessary (?) because earlier items may have been removed // after we recorded the match position. It is sort of hackish. A cleaner // solution would be to update the match position on changes - if (insertPosition>matched.getParent().getItemCount()) { - insertPosition=matched.getParent().getItemCount(); + if (insertPosition > matched.getParent().getItemCount()) { + insertPosition = matched.getParent().getItemCount(); } - e.insertItem(newItem,matched.getParent(),insertPosition,getTermType()); - if (e.getTraceLevel()>=6) - e.trace(6,"Inserted item '" + newItem + "' at position " + insertPosition + " producing " + e.getEvaluation().getQuery().getModel().getQueryTree()); + e.insertItem(newItem, matched.getParent(), insertPosition,getTermType()); + if (e.getTraceLevel() >= 6) + e.trace(6, "Inserted item '" + newItem + "' at position " + insertPosition + " producing " + + e.getEvaluation().getQuery().getModel().getQueryTree()); } protected String getLabelString() { diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java index f777fdeca26..036592dcf23 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java @@ -110,6 +110,8 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM InvokerResult result = new InvokerResult(query, query.getHits()); List<LeanHit> merged = Collections.emptyList(); long nextTimeout = query.getTimeLeft(); + boolean extraDebug = (query.getOffset() == 0) && (query.getHits() == 7) && log.isLoggable(java.util.logging.Level.FINE); + List<InvokerResult> processed = new ArrayList<>(); try { while (!invokers.isEmpty() && nextTimeout >= 0) { SearchInvoker invoker = availableForProcessing.poll(nextTimeout, TimeUnit.MILLISECONDS); @@ -117,7 +119,11 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM log.fine(() -> "Search timed out with " + askedNodes + " requests made, " + answeredNodes + " responses received"); break; } else { - merged = mergeResult(result.getResult(), invoker.getSearchResult(execution), merged); + InvokerResult toMerge = invoker.getSearchResult(execution); + if (extraDebug) { + processed.add(toMerge); + } + merged = mergeResult(result.getResult(), toMerge, merged); ejectInvoker(invoker); } nextTimeout = nextTimeout(); @@ -128,6 +134,33 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM insertNetworkErrors(result.getResult()); result.getResult().setCoverage(createCoverage()); + + if (extraDebug && merged.size() > 0) { + int firstPartId = merged.get(0).getPartId(); + for (int index = 1; index < merged.size(); index++) { + if (merged.get(index).getPartId() != firstPartId) { + extraDebug = false; + log.fine("merged["+index+"/"+merged.size()+"] from partId "+merged.get(index).getPartId()+", first "+firstPartId); + break; + } + } + } + if (extraDebug) { + log.fine("Interleaved "+processed.size()+" results"); + for (int pIdx = 0; pIdx < processed.size(); ++pIdx) { + var p = processed.get(pIdx); + log.fine("InvokerResult "+pIdx+" total hits "+p.getResult().getTotalHitCount()); + var lean = p.getLeanHits(); + for (int idx = 0; idx < lean.size(); ++idx) { + var hit = lean.get(idx); + log.fine("lean hit "+idx+" relevance "+hit.getRelevance()+" partid "+hit.getPartId()); + } + } + for (int mIdx = 0; mIdx < merged.size(); ++mIdx) { + var hit = merged.get(mIdx); + log.fine("merged hit "+mIdx+" relevance "+hit.getRelevance()+" partid "+hit.getPartId()); + } + } int needed = query.getOffset() + query.getHits(); for (int index = query.getOffset(); (index < merged.size()) && (index < needed); index++) { result.getLeanHits().add(merged.get(index)); @@ -235,22 +268,18 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM List<LeanHit> merged = new ArrayList<>(needed); int indexCurrent = 0; int indexPartial = 0; - log.fine(() -> "Merging "+current.size()+" and "+partial.size()+" lean hits (want "+needed+")"); while (indexCurrent < current.size() && indexPartial < partial.size() && merged.size() < needed) { LeanHit incomingHit = partial.get(indexPartial); LeanHit currentHit = current.get(indexCurrent); int cmpRes = currentHit.compareTo(incomingHit); if (cmpRes < 0) { - log.fine(() -> "Prefer current "+dbg(currentHit)); merged.add(currentHit); indexCurrent++; } else if (cmpRes > 0) { - log.fine(() -> "Prefer incoming "+dbg(incomingHit)); merged.add(incomingHit); indexPartial++; } else { // Duplicates - log.fine(() -> "Equal, prefer current "+dbg(currentHit)+" before incoming "+dbg(incomingHit)); merged.add(currentHit); indexCurrent++; indexPartial++; @@ -258,23 +287,12 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM } while ((indexCurrent < current.size()) && (merged.size() < needed)) { LeanHit currentHit = current.get(indexCurrent++); - log.fine(() -> "Also use current "+dbg(currentHit)); - merged.add(currentHit); - } - while (indexCurrent < current.size()) { - LeanHit currentHit = current.get(indexCurrent++); - log.fine(() -> "Do not use current "+dbg(currentHit)); merged.add(currentHit); } while ((indexPartial < partial.size()) && (merged.size() < needed)) { LeanHit incomingHit = partial.get(indexPartial++); - log.fine(() -> "Also use incoming "+dbg(incomingHit)); merged.add(incomingHit); } - while (indexPartial < partial.size()) { - LeanHit incomingHit = partial.get(indexPartial++); - log.fine(() -> "Do not use incoming "+dbg(incomingHit)); - } return merged; } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/TopKEstimator.java b/container-search/src/main/java/com/yahoo/search/dispatch/TopKEstimator.java index d48e337b0c1..aef1ef2f498 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/TopKEstimator.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/TopKEstimator.java @@ -13,6 +13,8 @@ public class TopKEstimator { private final double defaultP; private final boolean estimate; private final double skewFactor; + private final double [] defaultCumulativeProbability; + private final static int MIN_N = 2; private static boolean needEstimate(double p) { return (0.0 < p) && (p < 1.0); @@ -25,24 +27,40 @@ public class TopKEstimator { defaultP = defaultProbability; estimate = needEstimate(defaultP); this.skewFactor = skewFactor; + defaultCumulativeProbability = new double[64]; + for (int i=0; i < defaultCumulativeProbability.length; i++) { + defaultCumulativeProbability[i] = computeCumulativeProbability(i+MIN_N, defaultP); + } } - double estimateExactK(double k, double n, double p) { + private double inverseCumulativeProbability(int n, double p) { + if (p == defaultP && (n >= MIN_N) && (n < defaultCumulativeProbability.length + MIN_N)) { + return defaultCumulativeProbability[n - MIN_N]; + } + return computeCumulativeProbability(n, p); + } + private double computeCumulativeProbability(int n, double p) { + double p_inverse = 1 - (1 - p)/computeN(n); + return studentT.inverseCumulativeProbability(p_inverse); + } + private double computeN(double n) { double p_max = (1 + skewFactor)/n; - n = Math.max(1, 1/p_max); + return Math.max(1, 1/p_max); + } + double estimateExactK(double k, int n_i, double p) { + double n = computeN(n_i); double variance = k * 1/n * (1 - 1/n); - double p_inverse = 1 - (1 - p)/n; - return k/n + studentT.inverseCumulativeProbability(p_inverse) * Math.sqrt(variance); + return k/n + inverseCumulativeProbability(n_i, p) * Math.sqrt(variance); } - double estimateExactK(double k, double n) { + double estimateExactK(double k, int n) { return estimateExactK(k, n, defaultP); } public int estimateK(int k, int n) { - return (estimate && n > 1) + return (estimate && (n >= MIN_N)) ? Math.min(k, (int)Math.ceil(estimateExactK(k, n, defaultP))) : k; } public int estimateK(int k, int n, double p) { - return (needEstimate(p) && (n > 1)) + return (needEstimate(p) && (n >= MIN_N)) ? Math.min(k, (int)Math.ceil(estimateExactK(k, n, p))) : k; } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java index 1897c0af8bc..b3b2c23e7dc 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java @@ -94,7 +94,7 @@ public class SearchCluster implements NodeManager<Node> { this(clusterId, dispatchConfig, 1, vipStatus, pingFactory); } - public void addMonitoring(ClusterMonitor clusterMonitor) { + public void addMonitoring(ClusterMonitor<Node> clusterMonitor) { for (var group : orderedGroups()) { for (var node : group.nodes()) clusterMonitor.add(node, true); @@ -158,8 +158,11 @@ public class SearchCluster implements NodeManager<Node> { } } - /** Returns the number of nodes per group - size()/groups.size() */ - public int groupSize() { + /** + * Returns the wanted number of nodes per group - size()/groups.size(). + * The actual node count for a given group may differ due to node retirements. + */ + public int wantedGroupSize() { if (groups().size() == 0) return size(); return size() / groups().size(); } @@ -305,9 +308,10 @@ public class SearchCluster implements NodeManager<Node> { // With just one group sufficient coverage may not be the same as full coverage, as the // group will always be marked sufficient for use. updateSufficientCoverage(group, true); - boolean fullCoverage = isGroupCoverageSufficient(group.workingNodes(), group.nodes().size(), group.getActiveDocuments(), - group.getActiveDocuments()); - trackGroupCoverageChanges(0, group, fullCoverage, group.getActiveDocuments()); + boolean sufficientCoverage = isGroupCoverageSufficient(group.workingNodes(), + group.getActiveDocuments(), + group.getActiveDocuments()); + trackGroupCoverageChanges(0, group, sufficientCoverage, group.getActiveDocuments()); } private void pingIterationCompletedMultipleGroups() { @@ -327,7 +331,7 @@ public class SearchCluster implements NodeManager<Node> { Group group = orderedGroups().get(i); long activeDocuments = activeDocumentsInGroup[i]; long averageDocumentsInOtherGroups = (sumOfActiveDocuments - activeDocuments) / (numGroups - 1); - boolean sufficientCoverage = isGroupCoverageSufficient(group.workingNodes(), group.nodes().size(), activeDocuments, averageDocumentsInOtherGroups); + boolean sufficientCoverage = isGroupCoverageSufficient(group.workingNodes(), activeDocuments, averageDocumentsInOtherGroups); anyGroupsSufficientCoverage = anyGroupsSufficientCoverage || sufficientCoverage; updateSufficientCoverage(group, sufficientCoverage); trackGroupCoverageChanges(i, group, sufficientCoverage, averageDocumentsInOtherGroups); @@ -349,23 +353,22 @@ public class SearchCluster implements NodeManager<Node> { } } - private boolean isGroupCoverageSufficient(int workingNodes, int nodesInGroup, long activeDocuments, long averageDocumentsInOtherGroups) { - boolean sufficientCoverage = true; + private boolean isGroupCoverageSufficient(int workingNodesInGroup, long activeDocuments, long averageDocumentsInOtherGroups) { + double documentCoverage = 100.0 * (double) activeDocuments / averageDocumentsInOtherGroups; - if (averageDocumentsInOtherGroups > 0) { - double coverage = 100.0 * (double) activeDocuments / averageDocumentsInOtherGroups; - sufficientCoverage = coverage >= dispatchConfig.minActivedocsPercentage(); - } - if (sufficientCoverage) { - sufficientCoverage = isGroupNodeCoverageSufficient(workingNodes, nodesInGroup); - } - return sufficientCoverage; + if (averageDocumentsInOtherGroups > 0 && documentCoverage < dispatchConfig.minActivedocsPercentage()) + return false; + + if ( ! isGroupNodeCoverageSufficient(workingNodesInGroup)) + return false; + + return true; } - private boolean isGroupNodeCoverageSufficient(int workingNodes, int nodesInGroup) { + private boolean isGroupNodeCoverageSufficient(int workingNodesInGroup) { int nodesAllowedDown = dispatchConfig.maxNodesDownPerGroup() - + (int) (((double) nodesInGroup * (100.0 - dispatchConfig.minGroupCoverage())) / 100.0); - return workingNodes + nodesAllowedDown >= nodesInGroup; + + (int) (((double) wantedGroupSize() * (100.0 - dispatchConfig.minGroupCoverage())) / 100.0); + return workingNodesInGroup + nodesAllowedDown >= wantedGroupSize(); } public boolean isGroupWellBalanced(OptionalInt groupId) { @@ -379,7 +382,7 @@ public class SearchCluster implements NodeManager<Node> { */ public boolean isPartialGroupCoverageSufficient(OptionalInt knownGroupId, List<Node> nodes) { if (orderedGroups().size() == 1) { - boolean sufficient = nodes.size() >= groupSize() - dispatchConfig.maxNodesDownPerGroup(); + boolean sufficient = nodes.size() >= wantedGroupSize() - dispatchConfig.maxNodesDownPerGroup(); return sufficient; } @@ -391,7 +394,6 @@ public class SearchCluster implements NodeManager<Node> { if (group == null) { return false; } - int nodesInGroup = group.nodes().size(); long sumOfActiveDocuments = 0; int otherGroups = 0; for (Group g : orderedGroups()) { @@ -405,7 +407,7 @@ public class SearchCluster implements NodeManager<Node> { activeDocuments += n.getActiveDocuments(); } long averageDocumentsInOtherGroups = sumOfActiveDocuments / otherGroups; - return isGroupCoverageSufficient(nodes.size(), nodesInGroup, activeDocuments, averageDocumentsInOtherGroups); + return isGroupCoverageSufficient(nodes.size(), activeDocuments, averageDocumentsInOtherGroups); } private void trackGroupCoverageChanges(int index, Group group, boolean fullCoverage, long averageDocuments) { @@ -413,19 +415,21 @@ public class SearchCluster implements NodeManager<Node> { boolean changed = group.isFullCoverageStatusChanged(fullCoverage); if (changed || (!fullCoverage && System.currentTimeMillis() > nextLogTime)) { nextLogTime = System.currentTimeMillis() + 30 * 1000; - int requiredNodes = groupSize() - dispatchConfig.maxNodesDownPerGroup(); + int requiredNodes = group.nodes().size() - dispatchConfig.maxNodesDownPerGroup(); if (fullCoverage) { - log.info(() -> String.format("Group %d is now good again (%d/%d active docs, coverage %d/%d)", - index, group.getActiveDocuments(), averageDocuments, group.workingNodes(), groupSize())); + log.info(() -> String.format("Cluster %s: Group %d is now good again (%d/%d active docs, coverage %d/%d)", + clusterId, index, group.getActiveDocuments(), averageDocuments, + group.workingNodes(), group.nodes().size())); } else { StringBuilder missing = new StringBuilder(); for (var node : group.nodes()) { if (node.isWorking() != Boolean.TRUE) { - missing.append('\n').append(node.toString()); + missing.append('\n').append(node); } } - log.warning(() -> String.format("Coverage of group %d is only %d/%d (requires %d) (%d/%d active docs) Failed nodes are:%s", - index, group.workingNodes(), groupSize(), requiredNodes, group.getActiveDocuments(), averageDocuments, missing.toString())); + log.warning(() -> String.format("Cluster %s: Coverage of group %d is only %d/%d (requires %d) (%d/%d active docs) Failed nodes are:%s", + clusterId, index, group.workingNodes(), group.nodes().size(), requiredNodes, + group.getActiveDocuments(), averageDocuments, missing)); } } } diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/WandSearcher.java b/container-search/src/main/java/com/yahoo/search/querytransform/WandSearcher.java index f3804af1a9e..4745bd23642 100644 --- a/container-search/src/main/java/com/yahoo/search/querytransform/WandSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/querytransform/WandSearcher.java @@ -3,7 +3,13 @@ package com.yahoo.search.querytransform; import com.yahoo.prelude.Index; import com.yahoo.prelude.IndexFacts; -import com.yahoo.prelude.query.*; +import com.yahoo.prelude.query.CompositeItem; +import com.yahoo.prelude.query.DotProductItem; +import com.yahoo.prelude.query.Item; +import com.yahoo.prelude.query.OrItem; +import com.yahoo.prelude.query.WandItem; +import com.yahoo.prelude.query.WeakAndItem; +import com.yahoo.prelude.query.WordItem; import com.yahoo.processing.IllegalInputException; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.Query; @@ -11,11 +17,11 @@ import com.yahoo.search.Result; import com.yahoo.search.Searcher; import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.searchchain.Execution; -import com.yahoo.text.MapParser; import java.util.LinkedHashMap; import java.util.Map; +import com.yahoo.text.SimpleMapParser; import com.yahoo.yolean.Exceptions; /** @@ -65,7 +71,7 @@ public class WandSearcher extends Searcher { private static final CompoundName WAND_THRESHOLD_BOOST_FACTOR = new CompoundName("wand.thresholdBoostFactor"); private final String fieldName; private final WandType wandType; - private final Map<String, Integer> tokens; + private final Map<Object, Integer> tokens; private final int heapSize; private final double scoreThreshold; private final double thresholdBoostFactor; @@ -75,8 +81,14 @@ public class WandSearcher extends Searcher { if (fieldName != null) { String tokens = query.properties().getString(WAND_TOKENS); if (tokens != null) { - wandType = resolveWandType(execution.context().getIndexFacts().newSession(query), query); - this.tokens = new IntegerMapParser().parse(tokens, new LinkedHashMap<>()); + IndexFacts.Session indexFacts = execution.context().getIndexFacts().newSession(query); + Index index = indexFacts.getIndex(fieldName); + wandType = resolveWandType(index, indexFacts, query); + if (index.isNumerical() && (wandType == WandType.DOT_PRODUCT || wandType == WandType.PARALLEL)) { + this.tokens = new LongIntegerMapParser().parse(tokens, new LinkedHashMap<>(200)); + } else { + this.tokens = new MapObjectIntegerParser().parse(tokens, new LinkedHashMap<>(200)); + } heapSize = resolveHeapSize(query); scoreThreshold = resolveScoreThreshold(query); thresholdBoostFactor = resolveThresholdBoostFactor(query); @@ -90,8 +102,7 @@ public class WandSearcher extends Searcher { thresholdBoostFactor = 1; } - private WandType resolveWandType(IndexFacts.Session indexFacts, Query query) { - Index index = indexFacts.getIndex(fieldName); + private WandType resolveWandType(Index index, IndexFacts.Session indexFacts, Query query) { if (index.isNull()) { throw new IllegalInputException("Field '" + fieldName + "' was not found in " + indexFacts); } else { @@ -120,7 +131,7 @@ public class WandSearcher extends Searcher { return fieldName; } - public Map<String, Integer> getTokens() { + public Map<Object, Integer> getTokens() { return tokens; } @@ -162,17 +173,17 @@ public class WandSearcher extends Searcher { } else if (inputs.getWandType().equals(WandType.OR)) { return populate(new OrItem(), inputs.getFieldName(), inputs.getTokens()); } else if (inputs.getWandType().equals(WandType.PARALLEL)) { - return populate(new WandItem(inputs.getFieldName(), inputs.getHeapSize()), - inputs.getScoreThreshold(), inputs.getThresholdBoostFactor(), inputs.getTokens()); + return populate(new WandItem(inputs.getFieldName(), inputs.getHeapSize(), inputs.getTokens()), + inputs.getScoreThreshold(), inputs.getThresholdBoostFactor()); } else if (inputs.getWandType().equals(WandType.DOT_PRODUCT)) { - return populate(new DotProductItem(inputs.getFieldName()), inputs.getTokens()); + return new DotProductItem(inputs.getFieldName(), inputs.getTokens()); } throw new IllegalInputException("Unknown type '" + inputs.getWandType() + "'"); } - private CompositeItem populate(CompositeItem parent, String fieldName, Map<String,Integer> tokens) { - for (Map.Entry<String,Integer> entry : tokens.entrySet()) { - WordItem wordItem = new WordItem(entry.getKey(), fieldName); + private CompositeItem populate(CompositeItem parent, String fieldName, Map<Object,Integer> tokens) { + for (Map.Entry<Object,Integer> entry : tokens.entrySet()) { + WordItem wordItem = new WordItem(entry.getKey().toString(), fieldName); wordItem.setWeight(entry.getValue()); wordItem.setStemmed(true); wordItem.setNormalizable(false); @@ -181,25 +192,32 @@ public class WandSearcher extends Searcher { return parent; } - private WeightedSetItem populate(WeightedSetItem item, Map<String,Integer> tokens) { - for (Map.Entry<String,Integer> entry : tokens.entrySet()) { - item.addToken(entry.getKey(), entry.getValue()); - } - return item; - } - - private WandItem populate(WandItem item, double scoreThreshold, double thresholdBoostFactor, Map<String,Integer> tokens) { - populate(item, tokens); + private WandItem populate(WandItem item, double scoreThreshold, double thresholdBoostFactor) { item.setScoreThreshold(scoreThreshold); item.setThresholdBoostFactor(thresholdBoostFactor); return item; } - private static class IntegerMapParser extends MapParser<Integer> { + private static class MapObjectIntegerParser extends SimpleMapParser { + protected Map<Object, Integer> map; + + public Map<Object,Integer> parse(String string, Map<Object,Integer> map) { + this.map = map; + parse(string); + return this.map; + } + + @Override + protected void handleKeyValue(String key, String value) { + map.put(key, Integer.parseInt(value)); + } + } + + private static class LongIntegerMapParser extends MapObjectIntegerParser { @Override - protected Integer parseValue(String s) { - return Integer.parseInt(s); + protected void handleKeyValue(String key, String value) { + map.put(Long.parseLong(key), Integer.parseInt(value)); } } diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/ExpansionTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/ExpansionTestCase.java index b2d0d60c8fa..fa6b4eefdd5 100644 --- a/container-search/src/test/java/com/yahoo/prelude/semantics/test/ExpansionTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/ExpansionTestCase.java @@ -15,8 +15,19 @@ public class ExpansionTestCase extends RuleBaseAbstractTestCase { } @Test - public void testEquivExpansion() { + public void testEquivExpansion1() { assertSemantics("EQUIV equiv1 equiv2 equiv3", "equiv1"); } + @Test + public void testEquivExpansion2() { + assertSemantics("EQUIV testfield:e1 testfield:e2 testfield:e3", "testfield:foo"); + } + + @Test + public void testEquivExpansion3() { + assertSemantics("AND testfield:e1 testfield:e2 testfield:e3 testfield:e1 testfield:e2 testfield:e3", + "testfield:foo testfield:bar"); + } + } diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/ProductionRuleTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/ProductionRuleTestCase.java index 3513904af02..6c0084d1bdc 100644 --- a/container-search/src/test/java/com/yahoo/prelude/semantics/test/ProductionRuleTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/ProductionRuleTestCase.java @@ -50,7 +50,7 @@ public class ProductionRuleTestCase { RuleEvaluation e = new Evaluation(query).freshRuleEvaluation(); assertTrue(rule.matches(e)); rule.produce(e); - assertEquals("brand:sony", query.getModel().getQueryTree().getRoot().toString()); + assertEquals("AND brand:sony", query.getModel().getQueryTree().getRoot().toString()); } } diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/RuleBaseAbstractTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/RuleBaseAbstractTestCase.java index 41597a22832..81d359a804d 100644 --- a/container-search/src/test/java/com/yahoo/prelude/semantics/test/RuleBaseAbstractTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/RuleBaseAbstractTestCase.java @@ -57,8 +57,8 @@ public abstract class RuleBaseAbstractTestCase { } protected Query assertSemantics(String result, String input, int tracelevel, Query.Type queryType) { - Query query=new Query("?query=" + QueryTestCase.httpEncode(input) + "&tracelevel=0&tracelevel.rules=" + tracelevel + - "&language=und&type=" + queryType.toString()); + Query query = new Query("?query=" + QueryTestCase.httpEncode(input) + "&tracelevel=0&tracelevel.rules=" + tracelevel + + "&language=und&type=" + queryType); return assertSemantics(result, query); } diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/SemanticSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/SemanticSearcherTestCase.java index 5e5df9d2005..2e43eae3775 100644 --- a/container-search/src/test/java/com/yahoo/prelude/semantics/test/SemanticSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/SemanticSearcherTestCase.java @@ -24,7 +24,6 @@ import static org.junit.Assert.assertEquals; * * @author bratseth */ -@SuppressWarnings("deprecation") public class SemanticSearcherTestCase extends RuleBaseAbstractTestCase { public SemanticSearcherTestCase() { @@ -35,6 +34,8 @@ public class SemanticSearcherTestCase extends RuleBaseAbstractTestCase { public void testSingleShopping() { assertSemantics("brand:sony", "sony"); + assertSemantics("brand:sony!150", + "sony!150"); } @Test @@ -63,7 +64,7 @@ public class SemanticSearcherTestCase extends RuleBaseAbstractTestCase { @Test public void testLiteralReplacing() { - assertSemantics("AND lord of rings","lotr"); + assertSemantics("AND lord of rings", "lotr"); } @Test diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/WeightingTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/WeightingTestCase.java index 349a3502c0f..3e684ca6c04 100644 --- a/container-search/src/test/java/com/yahoo/prelude/semantics/test/WeightingTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/WeightingTestCase.java @@ -14,12 +14,12 @@ public class WeightingTestCase extends RuleBaseAbstractTestCase { @Test public void testWeighting() { - assertSemantics("foo!150","foo"); - assertSemantics("AND foo!150 snip","foo snip"); - assertSemantics("AND foo!150 bar","foo bar"); - assertSemantics("AND bar!57 foo","bar foo"); - assertSemantics("AND foo!150 fu","foo fu"); - assertSemantics("AND foo!150 bar kanoo boat!237","foo bar kanoo"); + assertSemantics("foo!150", "foo"); + assertSemantics("AND foo!150 snip", "foo snip"); + assertSemantics("AND foo!150 bar", "foo bar"); + assertSemantics("AND bar!57 foo!150", "bar foo"); + assertSemantics("AND foo!150 fu", "foo fu"); + assertSemantics("AND foo!150 bar kanoo boat!237", "foo bar kanoo"); } } diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/expansion.sr b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/expansion.sr index 728c494682b..d03f060cbde 100644 --- a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/expansion.sr +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/expansion.sr @@ -2,3 +2,7 @@ or1 +> ?or2 ?or3; equiv1 +> =equiv2 =equiv3; + +testfield:[test] -> =testfield:e1 =testfield:e2 =testfield:e3; + +[test] :- foo, bar, baz; diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java b/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java index 4afb6186c60..fe43658ee75 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java @@ -71,7 +71,7 @@ public class MockSearchCluster extends SearchCluster { } @Override - public int groupSize() { + public int wantedGroupSize() { return numNodesPerGroup; } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/LogEntry.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/LogEntry.java index 53ac0e42cb7..c315563f730 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/LogEntry.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/LogEntry.java @@ -69,6 +69,7 @@ public class LogEntry { parts[6].replaceAll("\\\\n", "\n") .replaceAll("\\\\t", "\t"))) .filter(entry -> entry.at().isAfter(from)) + .limit(100_000) .collect(Collectors.toUnmodifiableList()); } catch (IOException e) { diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveBucket.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveBucket.java new file mode 100644 index 00000000000..6bdaced2642 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveBucket.java @@ -0,0 +1,71 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.archive; + +import com.google.common.collect.Sets; +import com.yahoo.config.provision.TenantName; + +import java.util.Objects; +import java.util.Set; + +/** + * Represents an S3 bucket used to store archive data - logs, heap/core dumps, etc. + * + * @author andreer + */ +public class ArchiveBucket { + private final String bucketName; + private final String keyArn; + private final Set<TenantName> tenants; + + public ArchiveBucket(String bucketName, String keyArn) { + this(bucketName, keyArn, Set.of()); + } + + private ArchiveBucket(String bucketName, String keyArn, Set<TenantName> tenants) { + this.bucketName = bucketName; + this.keyArn = keyArn; + this.tenants = Set.copyOf(tenants); + } + + public String bucketName() { + return bucketName; + } + + public String keyArn() { + return keyArn; + } + + public Set<TenantName> tenants() { + return tenants; + } + + public ArchiveBucket withTenant(TenantName tenant) { + return withTenants(Set.of(tenant)); + } + + public ArchiveBucket withTenants(Set<TenantName> tenants) { + return new ArchiveBucket(bucketName, keyArn, Sets.union(this.tenants, tenants)); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ArchiveBucket that = (ArchiveBucket) o; + return bucketName.equals(that.bucketName) && keyArn.equals(that.keyArn) && tenants.equals(that.tenants); + } + + @Override + public int hashCode() { + return Objects.hash(bucketName, keyArn, tenants); + } + + @Override + public String toString() { + return "ArchiveBucket{" + + "bucketName='" + bucketName + '\'' + + ", keyArn='" + keyArn + '\'' + + ", tenants=" + tenants + + '}'; + } +}
\ No newline at end of file diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveBucketDb.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveBucketDb.java new file mode 100644 index 00000000000..91c7cd68ea2 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveBucketDb.java @@ -0,0 +1,16 @@ +// Copyright 2021 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.archive; + +import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.zone.ZoneId; + +import java.net.URI; +import java.util.Optional; +import java.util.Set; + +public interface ArchiveBucketDb { + + Optional<URI> archiveUriFor(ZoneId zoneId, TenantName tenant); + + Set<ArchiveBucket> buckets(ZoneId zoneId); +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveService.java index cdde20b4554..47aaf1740e1 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveService.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveService.java @@ -4,20 +4,18 @@ package com.yahoo.vespa.hosted.controller.api.integration.archive; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; -import java.net.URI; -import java.util.Optional; +import java.util.Map; +import java.util.Set; /** * Service that manages archive storage URIs for tenant nodes. * * @author freva + * @author andreer */ public interface ArchiveService { - Optional<URI> archiveUriFor(ZoneId zoneId, TenantName tenant); - - // TODO: Method to configure archive permissions/access for a tenant - - // TODO: Method to revoke permission/access for a tenant + ArchiveBucket createArchiveBucketFor(ZoneId zoneId); + void updateBucketAndKeyPolicy(ZoneId zoneId, ArchiveBucket bucket, Map<TenantName, String> authorizeIamRoleByTenantName); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/MockArchiveService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/MockArchiveService.java index 66f753a6186..9e9492a982c 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/MockArchiveService.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/MockArchiveService.java @@ -4,24 +4,24 @@ package com.yahoo.vespa.hosted.controller.api.integration.archive; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; -import java.net.URI; import java.util.HashMap; -import java.util.Optional; import java.util.Map; /** * @author freva + * @author andreer */ public class MockArchiveService implements ArchiveService { - private final Map<ZoneId, Map<TenantName, URI>> archiveUris = new HashMap<>(); + public Map<ArchiveBucket, Map<TenantName, String>> authorizedIamRoles = new HashMap<>(); @Override - public Optional<URI> archiveUriFor(ZoneId zoneId, TenantName tenant) { - return Optional.ofNullable(archiveUris.get(zoneId)).map(uris -> uris.get(tenant)); + public ArchiveBucket createArchiveBucketFor(ZoneId zoneId) { + return new ArchiveBucket("bucketName", "keyArn"); } - public void setArchiveUri(ZoneId zone, TenantName tenantName, URI archiveUri) { - archiveUris.computeIfAbsent(zone, z -> new HashMap<>()).put(tenantName, archiveUri); + @Override + public void updateBucketAndKeyPolicy(ZoneId zoneId, ArchiveBucket bucket, Map<TenantName, String> authorizeIamRoleByTenantName) { + authorizedIamRoles.put(bucket, authorizeIamRoleByTenantName); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/AlreadyExistsException.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/AlreadyExistsException.java deleted file mode 100644 index ffe7cb6ef67..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/AlreadyExistsException.java +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller; - -import com.yahoo.vespa.hosted.controller.api.identifiers.Identifier; - -/** - * @author Tony Vaagenes - */ -public class AlreadyExistsException extends IllegalArgumentException { - - /** - * Example message: Tenant 'myId' already exists. - * - * @param capitalizedType e.g. Tenant, Application - * @param id The id of the entity that didn't exist. - * - */ - public AlreadyExistsException(String capitalizedType, String id) { - super(String.format("%s '%s' already exists", capitalizedType, id)); - } - - public AlreadyExistsException(Identifier identifier) { - this(identifier.capitalizedType(), identifier.id()); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 8de50aa348a..69d723edbe8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -21,7 +21,7 @@ import com.yahoo.vespa.flags.FetchVector; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.flags.StringFlag; -import com.yahoo.vespa.hosted.controller.api.ActivateResult; +import com.yahoo.vespa.hosted.controller.application.ActivateResult; import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeploymentData; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.identifiers.InstanceId; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java index abc0784396c..c6dddc0f223 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java @@ -17,6 +17,7 @@ import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.hosted.controller.api.integration.ServiceRegistry; import com.yahoo.vespa.hosted.controller.api.integration.maven.MavenRepository; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; +import com.yahoo.vespa.hosted.controller.archive.CuratorArchiveBucketDb; import com.yahoo.vespa.hosted.controller.auditlog.AuditLogger; import com.yahoo.vespa.hosted.controller.config.ControllerConfig; import com.yahoo.vespa.hosted.controller.deployment.JobController; @@ -80,6 +81,7 @@ public class Controller extends AbstractComponent { private final RoutingController routingController; private final ControllerConfig controllerConfig; private final SecretStore secretStore; + private final CuratorArchiveBucketDb archiveBucketDb; /** * Creates a controller @@ -115,6 +117,7 @@ public class Controller extends AbstractComponent { routingController = new RoutingController(this, Objects.requireNonNull(rotationsConfig, "RotationsConfig cannot be null")); auditLogger = new AuditLogger(curator, clock); jobControl = new JobControl(new JobControlFlags(curator, flagSource)); + archiveBucketDb = new CuratorArchiveBucketDb(this); this.controllerConfig = controllerConfig; this.secretStore = secretStore; @@ -302,4 +305,8 @@ public class Controller extends AbstractComponent { public JobControl jobControl() { return jobControl; } + + public CuratorArchiveBucketDb archiveBucketDb() { + return archiveBucketDb; + } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java index fdd0e02e2ec..f3e192aef90 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java @@ -8,7 +8,7 @@ import com.yahoo.vespa.flags.FetchVector; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; -import com.yahoo.vespa.hosted.controller.athenz.impl.AthenzFacade; +import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.concurrent.Once; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.vespa.hosted.controller.security.AccessControl; @@ -92,6 +92,17 @@ public class TenantController { return get(name).orElseThrow(() -> new IllegalArgumentException("No such tenant '" + name + "'.")); } + /** Returns the tenant with the given name, and ensures the type */ + public <T extends Tenant> T require(TenantName name, Class<T> tenantType) { + return get(name) + .map(t -> { + try { return tenantType.cast(t); } catch (ClassCastException e) { + throw new IllegalArgumentException("Tenant '" + name + "' was of type '" + t.getClass().getSimpleName() + "' and not '" + tenantType.getSimpleName() + "'"); + } + }) + .orElseThrow(() -> new IllegalArgumentException("No such tenant '" + name + "'.")); + } + /** Replace and store any previous version of given tenant */ public void store(LockedTenant tenant) { curator.writeTenant(tenant.get()); @@ -164,7 +175,7 @@ public class TenantController { } private void requireNonExistent(TenantName name) { - if ("hosted-vespa".equals(name.value()) + if (SystemApplication.TENANT.equals(name) || get(name).isPresent() // Underscores are allowed in existing tenant names, but tenants with - and _ cannot co-exist. E.g. // my-tenant cannot be created if my_tenant exists. diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/api/package-info.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/api/package-info.java deleted file mode 100644 index 4b405f55e10..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/api/package-info.java +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * @author Tony Vaagenes - */ -@ExportPackage -package com.yahoo.vespa.hosted.controller.api; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/api/ActivateResult.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ActivateResult.java index 5379a08afc0..20400be0d1f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/api/ActivateResult.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ActivateResult.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api; +package com.yahoo.vespa.hosted.controller.application; import com.yahoo.vespa.hosted.controller.api.identifiers.RevisionId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareResponse; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Change.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Change.java index 8d082222721..33eafecf60a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Change.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Change.java @@ -40,7 +40,6 @@ public final class Change { throw new IllegalArgumentException("Application version to deploy must be a known version"); } this.pinned = pinned; - } public Change withoutPlatform() { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java index b742c45bf09..cfc9393efe2 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java @@ -3,7 +3,9 @@ package com.yahoo.vespa.hosted.controller.application; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; @@ -23,19 +25,22 @@ import java.util.Optional; */ public enum SystemApplication { - controllerHost(ApplicationId.from("hosted-vespa", "controller-host", "default"), NodeType.controllerhost), - configServerHost(ApplicationId.from("hosted-vespa", "configserver-host", "default"), NodeType.confighost), - configServer(ApplicationId.from("hosted-vespa", "zone-config-servers", "default"), NodeType.config), - proxyHost(ApplicationId.from("hosted-vespa", "proxy-host", "default"), NodeType.proxyhost), - proxy(ApplicationId.from("hosted-vespa", "routing", "default"), NodeType.proxy, proxyHost, configServer), - tenantHost(ApplicationId.from("hosted-vespa", "tenant-host", "default"), NodeType.host); + controllerHost("controller-host", NodeType.controllerhost), + configServerHost("configserver-host", NodeType.confighost), + configServer("zone-config-servers", NodeType.config), + proxyHost("proxy-host", NodeType.proxyhost), + proxy( "routing", NodeType.proxy, proxyHost, configServer), + tenantHost("tenant-host", NodeType.host); + + /** The tenant owning all system applications */ + public static final TenantName TENANT = TenantName.from(Constants.TENANT_NAME); private final ApplicationId id; private final NodeType nodeType; private final List<SystemApplication> dependencies; - SystemApplication(ApplicationId id, NodeType nodeType, SystemApplication... dependencies) { - this.id = id; + SystemApplication(String application, NodeType nodeType, SystemApplication... dependencies) { + this.id = ApplicationId.from(Constants.TENANT_NAME, application, InstanceName.defaultName().value()); this.nodeType = nodeType; this.dependencies = List.of(dependencies); } @@ -103,4 +108,8 @@ public enum SystemApplication { return String.format("system application %s of type %s", id, nodeType); } + private static class Constants { + private static final String TENANT_NAME = "hosted-vespa"; + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/archive/CuratorArchiveBucketDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/archive/CuratorArchiveBucketDb.java new file mode 100644 index 00000000000..0037048fca8 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/archive/CuratorArchiveBucketDb.java @@ -0,0 +1,134 @@ +// Copyright 2021 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.archive; + +import com.google.inject.Inject; +import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.flags.FetchVector; +import com.yahoo.vespa.flags.Flags; +import com.yahoo.vespa.flags.StringFlag; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveBucket; +import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveBucketDb; +import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveService; +import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; +import org.jetbrains.annotations.NotNull; + +import java.net.URI; +import java.util.HashSet; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; + +/** + * This class decides which tenant goes in what bucket, and creates new buckets when required. + * + * @author andreer + */ +public class CuratorArchiveBucketDb implements ArchiveBucketDb { + + /** + * Due to policy limits, we can't put data for more than this many tenants in a bucket. + * Policy size limit is 20kb, with approx. 500 bytes of policy required per tenant = 40 tenants. + * We set the maximum a bit lower to have a solid margin of error. + */ + private final static int TENANTS_PER_BUCKET = 30; + + /** + * Archive URIs are often requested because they are returned in /application/v4 API. Since they + * never change, it's safe to cache them and only update on misses + */ + private final Map<ZoneId, Map<TenantName, String>> archiveUriCache = new ConcurrentHashMap<>(); + + private final ArchiveService archiveService; + private final CuratorDb curatorDb; + private final StringFlag bucketNameFlag; + + @Inject + public CuratorArchiveBucketDb(Controller controller) { + this.archiveService = controller.serviceRegistry().archiveService(); + this.curatorDb = controller.curator(); + this.bucketNameFlag = Flags.SYNC_HOST_LOGS_TO_S3_BUCKET.bindTo(controller.flagSource()); + } + + @Override + public Optional<URI> archiveUriFor(ZoneId zoneId, TenantName tenant) { + String bucketName = bucketNameFlag + .with(FetchVector.Dimension.ZONE_ID, zoneId.value()) + .with(FetchVector.Dimension.TENANT_ID, tenant.value()) + .value(); + + if (bucketName.isBlank()) return Optional.empty(); + + if ("auto".equals(bucketName)) bucketName = findOrAssignBucket(zoneId, tenant); + + return Optional.of(URI.create(String.format("s3://%s/%s/", bucketName, tenant.value()))); + } + + private String findOrAssignBucket(ZoneId zoneId, TenantName tenant) { + return getBucketNameFromCache(zoneId, tenant) + .or(() -> findAndUpdateArchiveUriCache(zoneId, tenant, buckets(zoneId))) + .orElseGet(() -> assignToBucket(zoneId, tenant)); + } + + private String assignToBucket(ZoneId zoneId, TenantName tenant) { + try (var lock = curatorDb.lockArchiveBuckets(zoneId)) { + Set<ArchiveBucket> zoneBuckets = new HashSet<>(buckets(zoneId)); + + return findAndUpdateArchiveUriCache(zoneId, tenant, zoneBuckets) // Some other thread might have assigned it before we grabbed the lock + .orElseGet(() -> { + // If not, find an existing bucket with space + Optional<ArchiveBucket> unfilledBucket = zoneBuckets.stream() + .filter(bucket -> bucket.tenants().size() < TENANTS_PER_BUCKET) + .findAny(); + + // And place the tenant in that bucket. + if (unfilledBucket.isPresent()) { + var unfilled = unfilledBucket.get(); + + zoneBuckets.remove(unfilled); + zoneBuckets.add(unfilled.withTenant(tenant)); + curatorDb.writeArchiveBuckets(zoneId, zoneBuckets); + + return unfilled.bucketName(); + } + + // We'll have to create a new bucket + var newBucket = archiveService.createArchiveBucketFor(zoneId).withTenant(tenant); + zoneBuckets.add(newBucket); + curatorDb.writeArchiveBuckets(zoneId, zoneBuckets); + updateArchiveUriCache(zoneId, zoneBuckets); + return newBucket.bucketName(); + }); + } + } + + @Override + public Set<ArchiveBucket> buckets(ZoneId zoneId) { + return curatorDb.readArchiveBuckets(zoneId); + } + + @NotNull + private Optional<String> findAndUpdateArchiveUriCache(ZoneId zoneId, TenantName tenant, Set<ArchiveBucket> zoneBuckets) { + Optional<String> bucketName = zoneBuckets.stream() + .filter(bucket -> bucket.tenants().contains(tenant)) + .findAny() + .map(ArchiveBucket::bucketName); + if (bucketName.isPresent()) updateArchiveUriCache(zoneId, zoneBuckets); + return bucketName; + } + + private Optional<String> getBucketNameFromCache(ZoneId zoneId, TenantName tenantName) { + return Optional.ofNullable(archiveUriCache.get(zoneId)).map(map -> map.get(tenantName)); + } + + private void updateArchiveUriCache(ZoneId zoneId, Set<ArchiveBucket> zoneBuckets) { + Map<TenantName, String> bucketNameByTenant = zoneBuckets.stream() + .flatMap(bucket -> bucket.tenants().stream() + .map(tenant -> Map.entry(tenant, bucket.bucketName()))) + .collect(Collectors.toUnmodifiableMap(Map.Entry::getKey, Map.Entry::getValue)); + archiveUriCache.put(zoneId, bucketNameByTenant); + } +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/archive/package-info.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/archive/package-info.java new file mode 100644 index 00000000000..c93eb56d294 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/archive/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2021 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.hosted.controller.archive; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index f2df1cce15b..d3b8aa2a201 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -25,7 +25,7 @@ import com.yahoo.security.X509CertificateUtils; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.Instance; -import com.yahoo.vespa.hosted.controller.api.ActivateResult; +import com.yahoo.vespa.hosted.controller.application.ActivateResult; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.LogEntry; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; @@ -954,7 +954,7 @@ public class InternalStepRunner implements StepRunner { return new Timeouts(system); } - Duration capacity() { return Duration.ofMinutes(system.isCd() ? 5 : 0); } + Duration capacity() { return Duration.ofMinutes(system.isCd() ? 15 : 0); } Duration endpoint() { return Duration.ofMinutes(15); } Duration endpointCertificate() { return Duration.ofMinutes(20); } Duration tester() { return Duration.ofMinutes(30); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/LockedStep.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/LockedStep.java index d1d3e6238fd..966c9a2dbc3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/LockedStep.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/LockedStep.java @@ -3,6 +3,9 @@ package com.yahoo.vespa.hosted.controller.deployment; import com.yahoo.vespa.curator.Lock; +/** + * @author jonmv + */ public class LockedStep { private final Step step; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainer.java new file mode 100644 index 00000000000..826b411df9e --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainer.java @@ -0,0 +1,52 @@ +// Copyright 2021 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.google.common.collect.Maps; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveBucketDb; +import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveService; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; +import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; +import com.yahoo.vespa.hosted.controller.tenant.Tenant; + +import java.time.Duration; +import java.util.stream.Collectors; + +/** + * Update archive access permissions with roles from tenants + * + * @author andreer + */ +public class ArchiveAccessMaintainer extends ControllerMaintainer { + + private final ArchiveBucketDb archiveBucketDb; + private final ArchiveService archiveService; + private final ZoneRegistry zoneRegistry; + + public ArchiveAccessMaintainer(Controller controller, Duration interval) { + super(controller, interval); + this.archiveBucketDb = controller.archiveBucketDb(); + this.archiveService = controller.serviceRegistry().archiveService(); + this.zoneRegistry = controller().zoneRegistry(); + } + + @Override + protected boolean maintain() { + var tenantArchiveAccessRoles = controller().tenants().asList().stream() + .filter(t -> t instanceof CloudTenant) + .map(t -> (CloudTenant) t) + .filter(t -> t.archiveAccessRole().isPresent()) + .collect(Collectors.toUnmodifiableMap( + Tenant::name, cloudTenant -> cloudTenant.archiveAccessRole().orElseThrow())); + + zoneRegistry.zones().controllerUpgraded().ids().forEach(zoneId -> + archiveBucketDb.buckets(zoneId).forEach(archiveBucket -> + archiveService.updateBucketAndKeyPolicy(zoneId, archiveBucket, + Maps.filterEntries(tenantArchiveAccessRoles, + entry -> archiveBucket.tenants().contains(entry.getKey()))) + ) + ); + + return true; + } +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java index 5dce4d7b344..faa4813e6b0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java @@ -6,8 +6,9 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.ApplicationController; import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveService; +import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveBucketDb; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; +import com.yahoo.vespa.hosted.controller.application.SystemApplication; import java.net.URI; import java.time.Duration; @@ -23,17 +24,17 @@ import java.util.Set; */ public class ArchiveUriUpdater extends ControllerMaintainer { - private static final Set<TenantName> INFRASTRUCTURE_TENANTS = Set.of(TenantName.from("hosted-vespa")); + private static final Set<TenantName> INFRASTRUCTURE_TENANTS = Set.of(SystemApplication.TENANT); private final ApplicationController applications; private final NodeRepository nodeRepository; - private final ArchiveService archiveService; + private final ArchiveBucketDb archiveBucketDb; public ArchiveUriUpdater(Controller controller, Duration duration) { super(controller, duration, ArchiveUriUpdater.class.getSimpleName(), SystemName.all()); this.applications = controller.applications(); this.nodeRepository = controller.serviceRegistry().configServer().nodeRepository(); - this.archiveService = controller.serviceRegistry().archiveService(); + this.archiveBucketDb = controller.archiveBucketDb(); } @Override @@ -52,13 +53,13 @@ public class ArchiveUriUpdater extends ControllerMaintainer { tenantsByZone.forEach((zone, tenants) -> { Map<TenantName, URI> zoneArchiveUris = nodeRepository.getArchiveUris(zone); for (TenantName tenant : tenants) { - archiveService.archiveUriFor(zone, tenant) + archiveBucketDb.archiveUriFor(zone, tenant) .filter(uri -> !uri.equals(zoneArchiveUris.get(tenant))) .ifPresent(uri -> nodeRepository.setArchiveUri(zone, tenant, uri)); } zoneArchiveUris.keySet().stream() - .filter(tenant -> ! tenants.contains(tenant)) + .filter(tenant -> !tenants.contains(tenant)) .forEach(tenant -> nodeRepository.removeArchiveUri(zone, tenant)); }); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeManagementAssessor.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeManagementAssessor.java index 11f36201f32..a84d1c3ad7e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeManagementAssessor.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ChangeManagementAssessor.java @@ -3,20 +3,16 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.Cluster; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryNode; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeState; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeType; -import java.util.Arrays; import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Optional; import java.util.stream.Collectors; public class ChangeManagementAssessor { @@ -50,12 +46,14 @@ public class ChangeManagementAssessor { .flatMap(Collection::stream) .collect(Collectors.groupingBy(ChangeManagementAssessor::clusterKey)); - boolean allHostsReplacable = nodeRepository.isReplaceable( + var tenantHosts = prParentHost.keySet().stream() + .filter(node -> node.getType() == NodeType.host) + .map(node -> HostName.from(node.getHostname())) + .collect(Collectors.toList()); + + boolean allHostsReplacable = tenantHosts.isEmpty() || nodeRepository.isReplaceable( zone, - prParentHost.keySet().stream() - .filter(node -> node.getType() == NodeType.host) - .map(node -> HostName.from(node.getHostname())) - .collect(Collectors.toList()) + tenantHosts ); // Report assessment pr cluster @@ -120,6 +118,8 @@ public class ChangeManagementAssessor { } private static Cluster clusterKey(NodeRepositoryNode node) { + if (node.getOwner() == null) + return Cluster.EMPTY; String appId = String.format("%s:%s:%s", node.getOwner().tenant, node.getOwner().application, node.getOwner().instance); return new Cluster(Node.ClusterType.valueOf(node.getMembership().clustertype), node.getMembership().clusterid, appId, node.getType()); } @@ -225,6 +225,8 @@ public class ChangeManagementAssessor { private String app; private NodeType nodeType; + public final static Cluster EMPTY = new Cluster(Node.ClusterType.unknown, "na", "na", NodeType.tenant); + public Cluster(Node.ClusterType clusterType, String clusterId, String app, NodeType nodeType) { this.clusterType = clusterType; this.clusterId = clusterId; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContainerImageExpirer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContainerImageExpirer.java index 0f976458257..ff5fc4d2051 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContainerImageExpirer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContainerImageExpirer.java @@ -38,7 +38,7 @@ public class ContainerImageExpirer extends ControllerMaintainer { Instant now = controller().clock().instant(); VersionStatus versionStatus = controller().readVersionStatus(); List<ContainerImage> imagesToExpire = controller().serviceRegistry().containerRegistry().list().stream() - .filter(image -> canExpire(image, now, versionStatus)) + .filter(image -> isExpired(image, now, versionStatus)) .collect(Collectors.toList()); if (!imagesToExpire.isEmpty()) { log.log(Level.INFO, "Expiring " + imagesToExpire.size() + " container images: " + imagesToExpire); @@ -47,8 +47,8 @@ public class ContainerImageExpirer extends ControllerMaintainer { return true; } - /** Returns whether given image can be expired */ - private boolean canExpire(ContainerImage image, Instant now, VersionStatus versionStatus) { + /** Returns whether given image is expired */ + private boolean isExpired(ContainerImage image, Instant now, VersionStatus versionStatus) { List<VespaVersion> versions = versionStatus.versions(); if (versions.isEmpty()) return false; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index 8433afaf006..ad41fc7c9e8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -66,6 +66,7 @@ public class ControllerMaintenance extends AbstractComponent { maintainers.add(new EndpointCertificateMaintainer(controller, intervals.endpointCertificateMaintainer)); maintainers.add(new TrafficShareUpdater(controller, intervals.trafficFractionUpdater)); maintainers.add(new ArchiveUriUpdater(controller, intervals.archiveUriUpdater)); + maintainers.add(new ArchiveAccessMaintainer(controller, intervals.archiveAccessMaintainer)); maintainers.add(new TenantRoleMaintainer(controller, intervals.tenantRoleMaintainer)); maintainers.add(new ChangeRequestMaintainer(controller, intervals.changeRequestMaintainer)); } @@ -119,6 +120,7 @@ public class ControllerMaintenance extends AbstractComponent { private final Duration endpointCertificateMaintainer; private final Duration trafficFractionUpdater; private final Duration archiveUriUpdater; + private final Duration archiveAccessMaintainer; private final Duration tenantRoleMaintainer; private final Duration changeRequestMaintainer; @@ -149,6 +151,7 @@ public class ControllerMaintenance extends AbstractComponent { this.endpointCertificateMaintainer = duration(12, HOURS); this.trafficFractionUpdater = duration(5, MINUTES); this.archiveUriUpdater = duration(5, MINUTES); + this.archiveAccessMaintainer = duration(10, MINUTES); this.tenantRoleMaintainer = duration(5, MINUTES); this.changeRequestMaintainer = duration(12, HOURS); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java index 8de6bdbb99c..fd375c80218 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java @@ -11,6 +11,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; import com.yahoo.vespa.hosted.controller.api.integration.resource.MeteringClient; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshot; +import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.yolean.Exceptions; @@ -100,7 +101,7 @@ public class ResourceMeterMaintainer extends ControllerMaintainer { private Collection<ResourceSnapshot> createResourceSnapshotsFromNodes(ZoneId zoneId, List<Node> nodes) { return nodes.stream() - .filter(this::unlessNodeOwnerIsHostedVespa) + .filter(this::unlessNodeOwnerIsSystemApplication) .filter(this::isNodeStateMeterable) .filter(this::isNodeTypeMeterable) .collect(Collectors.groupingBy(node -> @@ -113,10 +114,10 @@ public class ResourceMeterMaintainer extends ControllerMaintainer { )).values(); } - private boolean unlessNodeOwnerIsHostedVespa(Node node) { + private boolean unlessNodeOwnerIsSystemApplication(Node node) { return node.owner() - .map(owner -> !owner.tenant().value().equals("hosted-vespa")) - .orElse(false); + .map(owner -> !owner.tenant().equals(SystemApplication.TENANT)) + .orElse(false); } /** diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/metric/CostCalculator.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/metric/CostCalculator.java index 4cc4ee0386c..5d92166d758 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/metric/CostCalculator.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/metric/CostCalculator.java @@ -8,6 +8,7 @@ import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.identifiers.Property; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceAllocation; +import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import com.yahoo.vespa.hosted.controller.tenant.Tenant; @@ -48,7 +49,7 @@ public class CostCalculator { var nodes = controller.zoneRegistry().zones() .reachable().in(Environment.prod).ofCloud(cloudName).zones().stream() .flatMap(zone -> uncheck(() -> nodeRepository.list(zone.getId()).stream())) - .filter(node -> node.owner().isPresent() && !node.owner().get().tenant().value().equals("hosted-vespa")) + .filter(node -> node.owner().isPresent() && !node.owner().get().tenant().equals(SystemApplication.TENANT)) .collect(Collectors.toList()); var totalAllocation = ResourceAllocation.ZERO; for (var node : nodes) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ArchiveBucketsSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ArchiveBucketsSerializer.java new file mode 100644 index 00000000000..3a625c5c42c --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ArchiveBucketsSerializer.java @@ -0,0 +1,71 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.persistence; + +import com.yahoo.config.provision.TenantName; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.Slime; +import com.yahoo.slime.SlimeUtils; +import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveBucket; + +import java.util.Set; +import java.util.stream.Collectors; + +/** + * (de)serializes tenant/bucket mappings for a zone + * <p> + * + * @author andreer + */ +public class ArchiveBucketsSerializer { + + // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one + // (and rewrite all nodes on startup), changes to the serialized format must be made + // such that what is serialized on version N+1 can be read by version N: + // - ADDING FIELDS: Always ok + // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. + // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. + + private final static String bucketsFieldName = "buckets"; + private final static String bucketNameFieldName = "bucketName"; + private final static String keyArnFieldName = "keyArn"; + private final static String tenantsFieldName = "tenantIds"; + + public static Slime toSlime(Set<ArchiveBucket> archiveBuckets) { + Slime slime = new Slime(); + Cursor rootObject = slime.setObject(); + Cursor bucketsArray = rootObject.setArray(bucketsFieldName); + + archiveBuckets.forEach(bucket -> { + Cursor cursor = bucketsArray.addObject(); + cursor.setString(bucketNameFieldName, bucket.bucketName()); + cursor.setString(keyArnFieldName, bucket.keyArn()); + Cursor tenants = cursor.setArray(tenantsFieldName); + bucket.tenants().forEach(tenantName -> tenants.addString(tenantName.value())); + } + ); + + return slime; + } + + public static Set<ArchiveBucket> fromSlime(Inspector inspector) { + return SlimeUtils.entriesStream(inspector.field(bucketsFieldName)) + .map(ArchiveBucketsSerializer::fromInspector) + .collect(Collectors.toUnmodifiableSet()); + } + + private static ArchiveBucket fromInspector(Inspector inspector) { + Set<TenantName> tenants = SlimeUtils.entriesStream(inspector.field(tenantsFieldName)) + .map(i -> TenantName.from(i.asString())) + .collect(Collectors.toUnmodifiableSet()); + + return new ArchiveBucket( + inspector.field(bucketNameFieldName).asString(), + inspector.field(keyArnFieldName).asString()) + .withTenants(tenants); + } + + public static Set<ArchiveBucket> fromJsonString(String zkData) { + return fromSlime(SlimeUtils.jsonToSlime(zkData).get()); + } +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java index 010a2e3f8e4..34741bcaedf 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java @@ -15,6 +15,7 @@ import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveBucket; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; @@ -87,6 +88,7 @@ public class CuratorDb { private static final Path routingPoliciesRoot = root.append("routingPolicies"); private static final Path zoneRoutingPoliciesRoot = root.append("zoneRoutingPolicies"); private static final Path endpointCertificateRoot = root.append("applicationCertificates"); + private static final Path archiveBucketsRoot = root.append("archiveBuckets"); private final NodeVersionSerializer nodeVersionSerializer = new NodeVersionSerializer(); private final VersionStatusSerializer versionStatusSerializer = new VersionStatusSerializer(nodeVersionSerializer); @@ -198,6 +200,10 @@ public class CuratorDb { return tryLock(lockRoot.append("meteringRefreshTime")); } + public Lock lockArchiveBuckets(ZoneId zoneId) { + return curator.lock(lockRoot.append("archiveBuckets").append(zoneId.value()), defaultLockTimeout); + } + // -------------- Helpers ------------------------------------------ /** Try locking with a low timeout, meaning it is OK to fail lock acquisition. @@ -546,6 +552,17 @@ public class CuratorDb { .orElse(0L); } + // -------------- Archive buckets ----------------------------------------- + + public Set<ArchiveBucket> readArchiveBuckets(ZoneId zoneId) { + return curator.getData(archiveBucketsPath(zoneId)).map(String::new).map(ArchiveBucketsSerializer::fromJsonString) + .orElse(Set.of()); + } + + public void writeArchiveBuckets(ZoneId zoneid, Set<ArchiveBucket> archiveBuckets) { + curator.set(archiveBucketsPath(zoneid), asJson(ArchiveBucketsSerializer.toSlime(archiveBuckets))); + } + // -------------- Paths --------------------------------------------------- private Path lockPath(TenantName tenant) { @@ -667,4 +684,8 @@ public class CuratorDb { return root.append("meteringRefreshTime"); } + private static Path archiveBucketsPath(ZoneId zoneId) { + return archiveBucketsRoot.append(zoneId.value()); + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index a7b872630e7..ffb5e040517 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -41,7 +41,6 @@ import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.LockedTenant; import com.yahoo.vespa.hosted.controller.NotExistsException; -import com.yahoo.vespa.hosted.controller.api.ActivateResult; import com.yahoo.vespa.hosted.controller.api.application.v4.EnvironmentResource; import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus; import com.yahoo.vespa.hosted.controller.api.application.v4.model.ProtonMetrics; @@ -70,6 +69,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretSto import com.yahoo.vespa.hosted.controller.api.role.Role; import com.yahoo.vespa.hosted.controller.api.role.RoleDefinition; import com.yahoo.vespa.hosted.controller.api.role.SecurityContext; +import com.yahoo.vespa.hosted.controller.application.ActivateResult; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.AssignedRotation; import com.yahoo.vespa.hosted.controller.application.Change; @@ -590,17 +590,13 @@ public class ApplicationApiHandler extends LoggingRequestHandler { } private HttpResponse validateSecretStore(String tenantName, String secretStoreName, HttpRequest request) { - var awsRegion = request.getProperty("aws-region"); var parameterName = request.getProperty("parameter-name"); var applicationId = ApplicationId.fromFullString(request.getProperty("application-id")); var zoneId = ZoneId.from(request.getProperty("zone")); var deploymentId = new DeploymentId(applicationId, zoneId); - var tenant = (CloudTenant)controller.tenants().require(applicationId.tenant()); - if (tenant.type() != Tenant.Type.cloud) { - return ErrorResponse.badRequest("Tenant '" + applicationId.tenant() + "' is not a cloud tenant"); - } + var tenant = controller.tenants().require(applicationId.tenant(), CloudTenant.class); var tenantSecretStore = tenant.tenantSecretStores() .stream() @@ -630,7 +626,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { String pemDeveloperKey = toSlime(request.getData()).get().field("key").asString(); PublicKey developerKey = KeyUtils.fromPemEncodedPublicKey(pemDeveloperKey); - Principal user = ((CloudTenant) controller.tenants().require(TenantName.from(tenantName))).developerKeys().get(developerKey); + Principal user = controller.tenants().require(TenantName.from(tenantName), CloudTenant.class).developerKeys().get(developerKey); Slime root = new Slime(); controller.tenants().lockOrThrow(TenantName.from(tenantName), LockedTenant.Cloud.class, tenant -> { tenant = tenant.withoutDeveloperKey(developerKey); @@ -685,7 +681,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { var externalId = mandatory("externalId", data).asString(); var role = mandatory("role", data).asString(); - var tenant = (CloudTenant) controller.tenants().require(TenantName.from(tenantName)); + var tenant = controller.tenants().require(TenantName.from(tenantName), CloudTenant.class); var tenantSecretStore = new TenantSecretStore(name, awsId, role); if (!tenantSecretStore.isValid()) { @@ -703,14 +699,14 @@ public class ApplicationApiHandler extends LoggingRequestHandler { controller.tenants().store(lockedTenant); }); - tenant = (CloudTenant) controller.tenants().require(TenantName.from(tenantName)); + tenant = controller.tenants().require(TenantName.from(tenantName), CloudTenant.class); var slime = new Slime(); toSlime(slime.setObject(), tenant.tenantSecretStores()); return new SlimeJsonResponse(slime); } private HttpResponse deleteSecretStore(String tenantName, String name, HttpRequest request) { - var tenant = (CloudTenant) controller.tenants().require(TenantName.from(tenantName)); + var tenant = controller.tenants().require(TenantName.from(tenantName), CloudTenant.class); var optionalSecretStore = tenant.tenantSecretStores().stream() .filter(secretStore -> secretStore.getName().equals(name)) @@ -727,7 +723,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { controller.tenants().store(lockedTenant); }); - tenant = (CloudTenant) controller.tenants().require(TenantName.from(tenantName)); + tenant = controller.tenants().require(TenantName.from(tenantName), CloudTenant.class); var slime = new Slime(); toSlime(slime.setObject(), tenant.tenantSecretStores()); return new SlimeJsonResponse(slime); @@ -1320,6 +1316,9 @@ public class ApplicationApiHandler extends LoggingRequestHandler { }); } + controller.archiveBucketDb().archiveUriFor(deploymentId.zoneId(), deploymentId.applicationId().tenant()) + .ifPresent(archiveUri -> response.setString("archiveUri", archiveUri.toString())); + Cursor activity = response.setObject("activity"); deployment.activity().lastQueried().ifPresent(instant -> activity.setLong("lastQueried", instant.toEpochMilli())); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponse.java index 3166a380f47..9255cbfd7b2 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponse.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponse.java @@ -7,7 +7,7 @@ import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.slime.Cursor; import com.yahoo.slime.JsonFormat; import com.yahoo.slime.Slime; -import com.yahoo.restapi.Uri; +import com.yahoo.restapi.UriBuilder; import com.yahoo.vespa.serviceview.bindings.ApplicationView; import com.yahoo.vespa.serviceview.bindings.ClusterView; import com.yahoo.vespa.serviceview.bindings.ServiceView; @@ -34,7 +34,7 @@ class ServiceApiResponse extends HttpResponse { private final ApplicationId application; private final List<URI> configServerURIs; private final Slime slime; - private final Uri requestUri; + private final UriBuilder requestUri; // Only set for one of the setResponse calls private String serviceName = null; @@ -46,7 +46,7 @@ class ServiceApiResponse extends HttpResponse { this.application = application; this.configServerURIs = configServerURIs; this.slime = new Slime(); - this.requestUri = new Uri(requestUri).withoutParameters(); + this.requestUri = new UriBuilder(requestUri).withoutParameters(); } public void setResponse(ApplicationView applicationView) { @@ -138,7 +138,7 @@ class ServiceApiResponse extends HttpResponse { mapToSlime((Map)entry, array.addObject()); } - private String rewriteIfUrl(String urlOrAnyString, Uri requestUri) { + private String rewriteIfUrl(String urlOrAnyString, UriBuilder requestUri) { if (urlOrAnyString == null) return null; String hostPattern = "(" + @@ -169,11 +169,11 @@ class ServiceApiResponse extends HttpResponse { } } - private Uri generateLocalLinkPrefix(String identifier, String restPath) { + private UriBuilder generateLocalLinkPrefix(String identifier, String restPath) { String proxiedPath = identifier + "/" + restPath; if (this.requestUri.toString().endsWith(proxiedPath)) { - return new Uri(this.requestUri.toString().substring(0, this.requestUri.toString().length() - proxiedPath.length())); + return new UriBuilder(this.requestUri.toString().substring(0, this.requestUri.toString().length() - proxiedPath.length())); } else { throw new IllegalStateException("Expected the resource path '" + this.requestUri + "' to end with '" + proxiedPath + "'"); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/ChangeManagementApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/ChangeManagementApiHandler.java index cd9c467582c..2077278ee0c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/ChangeManagementApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/ChangeManagementApiHandler.java @@ -68,7 +68,7 @@ public class ChangeManagementApiHandler extends AuditLoggingRequestHandler { private HttpResponse post(HttpRequest request) { Path path = new Path(request.getUri()); - if (path.matches("/changemanagement/v1/assessment")) return new SlimeJsonResponse(doAssessment(request)); + if (path.matches("/changemanagement/v1/assessment")) return doAssessment(request); return ErrorResponse.notFoundError("Nothing at " + path); } @@ -97,31 +97,24 @@ public class ChangeManagementApiHandler extends AuditLoggingRequestHandler { return ErrorResponse.notFoundError("Could not find any upcoming change requests with id " + changeRequestId); var changeRequest = optionalChangeRequest.get(); - var zone = affectedZone(changeRequest); - if (zone.isEmpty()) - return ErrorResponse.notFoundError("Could not find prod zone affected by change request " + changeRequestId); - - var assessment = doAssessment(changeRequest.getImpactedHosts(), zone.get()); - return new SlimeJsonResponse(assessment); + return doAssessment(changeRequest.getImpactedHosts()); } // The structure here should be // // { - // zone: string // hosts: string[] // switches: string[] // switchInSequence: boolean // } // - // Only zone and host are supported right now - private Slime doAssessment(HttpRequest request) { + // Only hosts is supported right now + private HttpResponse doAssessment(HttpRequest request) { Inspector inspector = inspectorOrThrow(request); // For now; mandatory fields - String zoneStr = getInspectorFieldOrThrow(inspector, "zone").asString(); Inspector hostArray = getInspectorFieldOrThrow(inspector, "hosts"); // The impacted hostnames @@ -130,11 +123,15 @@ public class ChangeManagementApiHandler extends AuditLoggingRequestHandler { hostArray.traverse((ArrayTraverser) (i, host) -> hostNames.add(host.asString())); } - return doAssessment(hostNames, ZoneId.from(zoneStr)); + return doAssessment(hostNames); } - private Slime doAssessment(List<String> hostNames, ZoneId zoneId) { - ChangeManagementAssessor.Assessment assessments = assessor.assessment(hostNames, zoneId); + private HttpResponse doAssessment(List<String> hostNames) { + var zone = affectedZone(hostNames); + if (zone.isEmpty()) + return ErrorResponse.notFoundError("Could not infer prod zone from host list: " + hostNames); + + ChangeManagementAssessor.Assessment assessments = assessor.assessment(hostNames, zone.get()); Slime slime = new Slime(); Cursor root = slime.setObject(); @@ -171,12 +168,11 @@ public class ChangeManagementApiHandler extends AuditLoggingRequestHandler { hostObject.setLong("numberOfProblematicChildren", assessment.numberOfProblematicChildren); }); - return slime; + return new SlimeJsonResponse(slime); } - private Optional<ZoneId> affectedZone(ChangeRequest changeRequest) { - var affectedHosts = changeRequest.getImpactedHosts() - .stream() + private Optional<ZoneId> affectedZone(List<String> hosts) { + var affectedHosts = hosts.stream() .map(HostName::from) .collect(Collectors.toList()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java index 9c609c8341b..5d819ec9804 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java @@ -11,7 +11,7 @@ import com.yahoo.container.jdisc.LoggingRequestHandler; import com.yahoo.restapi.ErrorResponse; import com.yahoo.restapi.Path; import com.yahoo.restapi.SlimeJsonResponse; -import com.yahoo.restapi.Uri; +import com.yahoo.restapi.UriBuilder; import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; import com.yahoo.vespa.hosted.controller.Controller; @@ -227,7 +227,7 @@ public class DeploymentApiHandler extends LoggingRequestHandler { object.setString("tenant", id.tenant().value()); object.setString("application", id.application().value()); object.setString("instance", id.instance().value()); - object.setString("url", new Uri(request.getUri()).withPath("/application/v4/tenant/" + + object.setString("url", new UriBuilder(request.getUri()).withPath("/application/v4/tenant/" + id.tenant().value() + "/application/" + id.application().value()).toString()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java index 039e9b64df7..6e069b2b5ec 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java @@ -21,7 +21,6 @@ import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.flags.BooleanFlag; import com.yahoo.vespa.flags.FetchVector; import com.yahoo.vespa.flags.FlagSource; -import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.IntFlag; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.controller.Controller; @@ -76,7 +75,7 @@ public class UserApiHandler extends LoggingRequestHandler { this.users = users; this.controller = controller; this.enable_public_signup_flow = PermanentFlags.ENABLE_PUBLIC_SIGNUP_FLOW.bindTo(flagSource); - this.maxTrialTenants = Flags.MAX_TRIAL_TENANTS.bindTo(flagSource); + this.maxTrialTenants = PermanentFlags.MAX_TRIAL_TENANTS.bindTo(flagSource); } @Override diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java index f87c6e2d11c..a0fecbdf9e1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java @@ -73,8 +73,8 @@ public class RoutingPolicy { public Endpoint endpointIn(SystemName system, RoutingMethod routingMethod, ZoneRegistry zoneRegistry) { Optional<Endpoint> infraEndpoint = SystemApplication.matching(id.owner()) .flatMap(app -> app.endpointIn(id.zone(), zoneRegistry)); - if (infraEndpoint.isPresent()) return infraEndpoint.get(); - return endpoint(routingMethod).target(id.cluster(), id.zone()).in(system); + return infraEndpoint.orElseGet(() -> endpoint(routingMethod).target(id.cluster(), id.zone()) + .in(system)); } /** Returns the region endpoint of this */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java index 563c230e4f0..48f8d3e43cb 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java @@ -6,7 +6,6 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.vespa.flags.BooleanFlag; import com.yahoo.vespa.flags.FetchVector; import com.yahoo.vespa.flags.FlagSource; -import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.IntFlag; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.controller.Application; @@ -47,7 +46,7 @@ public class CloudAccessControl implements AccessControl { public CloudAccessControl(UserManagement userManagement, FlagSource flagSource, ServiceRegistry serviceRegistry) { this.userManagement = userManagement; this.enablePublicSignup = PermanentFlags.ENABLE_PUBLIC_SIGNUP_FLOW.bindTo(flagSource); - this.maxTrialTenants = Flags.MAX_TRIAL_TENANTS.bindTo(flagSource); + this.maxTrialTenants = PermanentFlags.MAX_TRIAL_TENANTS.bindTo(flagSource); billingController = serviceRegistry.billingController(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java index 8fd5f07b9ea..69808a987fc 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java @@ -75,8 +75,8 @@ public class OsVersionStatus { var nodeVersion = new NodeVersion(node.hostname(), zone.getVirtualId(), node.currentOsVersion(), targetOsVersion, suspendedAt); var osVersion = new OsVersion(nodeVersion.currentVersion(), zone.getCloudName()); - osVersions.putIfAbsent(osVersion, new ArrayList<>()); - osVersions.get(osVersion).add(nodeVersion); + osVersions.computeIfAbsent(osVersion, (k) -> new ArrayList<>()) + .add(nodeVersion); } } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/archive/CuratorArchiveBucketDbTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/archive/CuratorArchiveBucketDbTest.java new file mode 100644 index 00000000000..57fa7cc8e44 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/archive/CuratorArchiveBucketDbTest.java @@ -0,0 +1,66 @@ +package com.yahoo.vespa.hosted.controller.archive; + +import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.flags.Flags; +import com.yahoo.vespa.flags.InMemoryFlagSource; +import com.yahoo.vespa.hosted.controller.ControllerTester; +import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveBucket; +import org.apache.curator.shaded.com.google.common.collect.Streams; +import org.junit.Test; + +import java.net.URI; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import java.util.stream.Stream; + +import static org.junit.Assert.*; + +public class CuratorArchiveBucketDbTest { + + @Test + public void archiveUriFor() { + ControllerTester tester = new ControllerTester(); + InMemoryFlagSource flagSource = (InMemoryFlagSource) tester.controller().flagSource(); + CuratorArchiveBucketDb bucketDb = new CuratorArchiveBucketDb(tester.controller()); + + tester.curator().writeArchiveBuckets(ZoneId.defaultId(), + Set.of(new ArchiveBucket("existingBucket", "keyArn").withTenant(TenantName.defaultName()))); + + // Nothing when feature flag is not set. + assertEquals(Optional.empty(), bucketDb.archiveUriFor(ZoneId.defaultId(), TenantName.defaultName())); + + // Returns hardcoded name from feature flag + flagSource.withStringFlag(Flags.SYNC_HOST_LOGS_TO_S3_BUCKET.id(), "hardcoded"); + assertEquals(Optional.of(URI.create("s3://hardcoded/default/")), bucketDb.archiveUriFor(ZoneId.defaultId(), TenantName.defaultName())); + + // Finds existing bucket in db when set to "auto" + flagSource.withStringFlag(Flags.SYNC_HOST_LOGS_TO_S3_BUCKET.id(), "auto"); + assertEquals(Optional.of(URI.create("s3://existingBucket/default/")), bucketDb.archiveUriFor(ZoneId.defaultId(), TenantName.defaultName())); + + // Assigns to existing bucket while there is space + IntStream.range(0, 29).forEach(i -> + assertEquals( + Optional.of(URI.create("s3://existingBucket/tenant" + i + "/")), bucketDb + .archiveUriFor(ZoneId.defaultId(), TenantName.from("tenant" + i)))); + + // Creates new bucket when existing buckets are full + assertEquals(Optional.of(URI.create("s3://bucketName/lastDrop/")), bucketDb.archiveUriFor(ZoneId.defaultId(), TenantName.from("lastDrop"))); + + // Creates new bucket when there are no existing buckets in zone + assertEquals(Optional.of(URI.create("s3://bucketName/firstInZone/")), bucketDb.archiveUriFor(ZoneId.from("prod.us-east-3"), TenantName.from("firstInZone"))); + + // Lists all buckets by zone + Set<TenantName> existingBucketTenants = Streams.concat(Stream.of(TenantName.defaultName()), IntStream.range(0, 29).mapToObj(i -> TenantName.from("tenant" + i))).collect(Collectors.toUnmodifiableSet()); + assertEquals( + Set.of( + new ArchiveBucket("existingBucket", "keyArn").withTenants(existingBucketTenants), + new ArchiveBucket("bucketName", "keyArn").withTenant(TenantName.from("lastDrop"))), + bucketDb.buckets(ZoneId.defaultId())); + assertEquals( + Set.of(new ArchiveBucket("bucketName", "keyArn").withTenant(TenantName.from("firstInZone"))), + bucketDb.buckets(ZoneId.from("prod.us-east-3"))); + } +}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainerTest.java new file mode 100644 index 00000000000..89072519c7d --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainerTest.java @@ -0,0 +1,60 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.flags.Flags; +import com.yahoo.vespa.flags.InMemoryFlagSource; +import com.yahoo.vespa.flags.PermanentFlags; +import com.yahoo.vespa.hosted.controller.ControllerTester; +import com.yahoo.vespa.hosted.controller.LockedTenant; +import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveBucket; +import com.yahoo.vespa.hosted.controller.api.integration.archive.MockArchiveService; +import com.yahoo.vespa.hosted.controller.restapi.ContainerTester; +import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerCloudTest; +import com.yahoo.vespa.hosted.controller.tenant.Tenant; +import org.junit.Test; + +import java.time.Duration; +import java.util.Map; +import java.util.Optional; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +/** + * @author andreer + */ +public class ArchiveAccessMaintainerTest extends ControllerContainerCloudTest { + + @Test + public void grantsRoleAccess() { + var containerTester = new ContainerTester(container, ""); + ((InMemoryFlagSource) containerTester.controller().flagSource()) + .withBooleanFlag(PermanentFlags.ENABLE_PUBLIC_SIGNUP_FLOW.id(), true) + .withStringFlag(Flags.SYNC_HOST_LOGS_TO_S3_BUCKET.id(), "auto"); + var tester = new ControllerTester(containerTester); + + String tenant1role = "arn:aws:iam::123456789012:role/my-role"; + String tenant2role = "arn:aws:iam::210987654321:role/my-role"; + var tenant1 = createTenantWithAccessRole(tester, "tenant1", tenant1role); + createTenantWithAccessRole(tester, "tenant2", tenant2role); + + tester.controller().archiveBucketDb().archiveUriFor(ZoneId.from("prod.us-east-3"), tenant1); + var testBucket = new ArchiveBucket("bucketName", "keyArn").withTenant(tenant1); + + MockArchiveService archiveService = (MockArchiveService) tester.controller().serviceRegistry().archiveService(); + assertNull(archiveService.authorizedIamRoles.get(testBucket)); + new ArchiveAccessMaintainer(containerTester.controller(), Duration.ofMinutes(10)).maintain(); + assertEquals(Map.of(tenant1, tenant1role), archiveService.authorizedIamRoles.get(testBucket)); + } + + private TenantName createTenantWithAccessRole(ControllerTester tester, String tenantName, String role) { + var tenant = tester.createTenant(tenantName, Tenant.Type.cloud); + tester.controller().tenants().lockOrThrow(tenant, LockedTenant.Cloud.class, lockedTenant -> { + lockedTenant = lockedTenant.withArchiveAccessRole(Optional.of(role)); + tester.controller().tenants().store(lockedTenant); + }); + return tenant; + } +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdaterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdaterTest.java index a9666cf9113..505536558ab 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdaterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdaterTest.java @@ -5,17 +5,22 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.api.integration.archive.MockArchiveService; +import com.yahoo.vespa.flags.Flags; +import com.yahoo.vespa.flags.InMemoryFlagSource; +import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveBucket; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import org.junit.Test; import java.net.URI; import java.time.Duration; +import java.util.LinkedHashSet; import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; @@ -31,9 +36,12 @@ public class ArchiveUriUpdaterTest { public void archive_uri_test() { var updater = new ArchiveUriUpdater(tester.controller(), Duration.ofDays(1)); + ((InMemoryFlagSource) tester.controller().flagSource()) + .withStringFlag(Flags.SYNC_HOST_LOGS_TO_S3_BUCKET.id(), "auto"); + var tenant1 = TenantName.from("tenant1"); var tenant2 = TenantName.from("tenant2"); - var tenantInfra = TenantName.from("hosted-vespa"); + var tenantInfra = SystemApplication.TENANT; var application = tester.newDeploymentContext(tenant1.value(), "app1", "instance1"); ZoneId zone = ZoneId.from("prod", "ap-northeast-1"); @@ -42,19 +50,19 @@ public class ArchiveUriUpdaterTest { assertArchiveUris(Map.of(), zone); // Archive service now has URI for tenant1, but tenant1 is not deployed in zone - setArchiveUriInService(Map.of(tenant1, "uri-1"), zone); - setArchiveUriInService(Map.of(tenantInfra, "uri-3"), zone); + setBucketNameInService(Map.of(tenant1, "uri-1"), zone); + setBucketNameInService(Map.of(tenantInfra, "uri-3"), zone); updater.maintain(); assertArchiveUris(Map.of(), zone); deploy(application, zone); updater.maintain(); - assertArchiveUris(Map.of(tenant1, "uri-1", tenantInfra, "uri-3"), zone); + assertArchiveUris(Map.of(tenant1, "s3://uri-1/tenant1/", tenantInfra, "s3://uri-3/hosted-vespa/"), zone); // URI for tenant1 should be updated and removed for tenant2 setArchiveUriInNodeRepo(Map.of(tenant1, "wrong-uri", tenant2, "uri-2"), zone); updater.maintain(); - assertArchiveUris(Map.of(tenant1, "uri-1", tenantInfra, "uri-3"), zone); + assertArchiveUris(Map.of(tenant1, "s3://uri-1/tenant1/", tenantInfra, "s3://uri-3/hosted-vespa/"), zone); } private void assertArchiveUris(Map<TenantName, String> expectedUris, ZoneId zone) { @@ -64,9 +72,11 @@ public class ArchiveUriUpdaterTest { assertEquals(expectedUris, actualUris); } - private void setArchiveUriInService(Map<TenantName, String> archiveUris, ZoneId zone) { - MockArchiveService archiveService = (MockArchiveService) tester.controller().serviceRegistry().archiveService(); - archiveUris.forEach((tenant, uri) -> archiveService.setArchiveUri(zone, tenant, URI.create(uri))); + private void setBucketNameInService(Map<TenantName, String> bucketNames, ZoneId zone) { + var archiveBuckets = new LinkedHashSet<>(tester.controller().curator().readArchiveBuckets(zone)); + bucketNames.forEach((tenantName, bucketName) -> + archiveBuckets.add(new ArchiveBucket(bucketName, "keyArn").withTenant(tenantName))); + tester.controller().curator().writeArchiveBuckets(zone, archiveBuckets); } private void setArchiveUriInNodeRepo(Map<TenantName, String> archiveUris, ZoneId zone) { @@ -77,4 +87,4 @@ public class ArchiveUriUpdaterTest { private void deploy(DeploymentContext application, ZoneId zone) { application.runJob(JobType.from(SystemName.main, zone).orElseThrow(), new ApplicationPackage(new byte[0]), Version.fromString("7.1")); } -}
\ No newline at end of file +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainerTest.java index ad8d0050c73..814dc2a3f50 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainerTest.java @@ -8,6 +8,7 @@ import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.integration.aws.MockResourceTagger; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; +import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; import org.junit.Test; @@ -52,7 +53,7 @@ public class ResourceTagMaintainerTest { var hostA = new Node.Builder() .hostname(HostName.from("parentHostA." + zone.value())) .type(NodeType.host) - .owner(ApplicationId.from("hosted-vespa", "tenant-host", "default")) + .owner(ApplicationId.from(SystemApplication.TENANT.value(), "tenant-host", "default")) .exclusiveTo(ApplicationId.from("t1", "a1", "i1")) .build(); var nodeA = new Node.Builder() @@ -64,7 +65,7 @@ public class ResourceTagMaintainerTest { var hostB = new Node.Builder() .hostname(HostName.from("parentHostB." + zone.value())) .type(NodeType.host) - .owner(ApplicationId.from("hosted-vespa", "tenant-host", "default")) + .owner(ApplicationId.from(SystemApplication.TENANT.value(), "tenant-host", "default")) .build(); tester.configServer().nodeRepository().setNodes(zone, List.of(hostA, nodeA, hostB)); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ArchiveBucketsSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ArchiveBucketsSerializerTest.java new file mode 100644 index 00000000000..17814b12a09 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ArchiveBucketsSerializerTest.java @@ -0,0 +1,28 @@ +package com.yahoo.vespa.hosted.controller.persistence; + +import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveBucket; +import org.junit.Test; + +import java.util.LinkedHashSet; + +import static org.junit.Assert.assertEquals; + +public class ArchiveBucketsSerializerTest { + + @Test + public void serdes() { + var testTenants = new LinkedHashSet<TenantName>(); + testTenants.add(TenantName.from("tenant1")); + testTenants.add(TenantName.from("tenant2")); + + var testBuckets = new LinkedHashSet<ArchiveBucket>(); + testBuckets.add(new ArchiveBucket("bucket1Name", "key1Arn").withTenants(testTenants)); + testBuckets.add(new ArchiveBucket("bucket2Name", "key2Arn")); + + String zkData = "{\"buckets\":[{\"bucketName\":\"bucket1Name\",\"keyArn\":\"key1Arn\",\"tenantIds\":[\"tenant1\",\"tenant2\"]},{\"bucketName\":\"bucket2Name\",\"keyArn\":\"key2Arn\",\"tenantIds\":[]}]}"; + + assertEquals(testBuckets, ArchiveBucketsSerializer.fromJsonString(zkData)); + assertEquals(testBuckets, ArchiveBucketsSerializer.fromJsonString(ArchiveBucketsSerializer.toSlime(testBuckets).toString())); + } +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java index 4fb91639daa..7074d0d7354 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.hosted.controller.restapi.application; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.TenantName; -import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.controller.LockedTenant; @@ -107,7 +106,7 @@ public class ApplicationApiCloudTest extends ControllerContainerCloudTest { @Test public void trial_tenant_limit_reached() { - ((InMemoryFlagSource) tester.controller().flagSource()).withIntFlag(Flags.MAX_TRIAL_TENANTS.id(), 1); + ((InMemoryFlagSource) tester.controller().flagSource()).withIntFlag(PermanentFlags.MAX_TRIAL_TENANTS.id(), 1); tester.controller().serviceRegistry().billingController().setPlan(tenantName, PlanId.from("pay-as-you-go"), false); // tests that we can create the one trial tenant the flag says we can have -- and that the tenant created @@ -190,10 +189,10 @@ public class ApplicationApiCloudTest extends ControllerContainerCloudTest { lockedTenant = lockedTenant.withSecretStore(new TenantSecretStore("secret-foo", "123", "some-role")); tester.controller().tenants().store(lockedTenant); }); - var tenant = (CloudTenant) tester.controller().tenants().require(tenantName); + var tenant = tester.controller().tenants().require(tenantName, CloudTenant.class); assertEquals(1, tenant.tenantSecretStores().size()); tester.assertResponse(deleteRequest, "{\"secretStores\":[]}", 200); - tenant = (CloudTenant) tester.controller().tenants().require(tenantName); + tenant = tester.controller().tenants().require(tenantName, CloudTenant.class); assertEquals(0, tenant.tenantSecretStores().size()); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index c69d2069650..b1b1c7ffe7a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -153,6 +153,7 @@ public class ApplicationApiTest extends ControllerContainerTest { @Test public void testApplicationApi() { createAthenzDomainWithAdmin(ATHENZ_TENANT_DOMAIN, USER_ID); // (Necessary but not provided in this API) + ((InMemoryFlagSource) tester.controller().flagSource()).withStringFlag(Flags.SYNC_HOST_LOGS_TO_S3_BUCKET.id(), "my-bucket"); // GET API root tester.assertResponse(request("/application/v4/", GET).userIdentity(USER_ID), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json index 946593fca00..443e49a3896 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json @@ -51,6 +51,7 @@ "commit": "commit1" }, "status": "complete", + "archiveUri":"s3://my-bucket/tenant1/", "activity": { "lastQueried": 1527848130000, "lastWritten": 1527848130000, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json index 7ba63e1664d..7181c4ee2be 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json @@ -20,6 +20,7 @@ "revision": "(ignore)", "deployTimeEpochMs": "(ignore)", "screwdriverId": "123", + "archiveUri":"s3://my-bucket/tenant1/", "activity": { "lastQueried": 1527848130000, "lastWritten": 1527848130000, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json index 4251ba1ad95..12bd5a6efbd 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json @@ -54,6 +54,7 @@ "commit": "commit1" }, "status": "complete", + "archiveUri":"s3://my-bucket/tenant1/", "activity": { "lastQueried": 1527848130000, "lastWritten": 1527848130000, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/ChangeManagementApiHandlerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/ChangeManagementApiHandlerTest.java index 2f2e70e2cf6..cd815a2064b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/ChangeManagementApiHandlerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/ChangeManagementApiHandlerTest.java @@ -2,9 +2,11 @@ package com.yahoo.vespa.hosted.controller.restapi.changemanagement; import com.yahoo.application.container.handler.Request; +import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzUser; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeMembership; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeOwner; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryNode; @@ -32,6 +34,7 @@ public class ChangeManagementApiHandlerTest extends ControllerContainerTest { tester = new ContainerTester(container, responses); addUserToHostedOperatorRole(operator); tester.serviceRegistry().configServer().nodeRepository().addNodes(ZoneId.from("prod.us-east-3"), createNodes()); + tester.serviceRegistry().configServer().nodeRepository().putNodes(ZoneId.from("prod.us-east-3"), createNode()); } @Test @@ -49,6 +52,12 @@ public class ChangeManagementApiHandlerTest extends ControllerContainerTest { tester.assertResponse(request, new File(filename)); } + private Node createNode() { + return new Node.Builder() + .hostname(HostName.from("host1")) + .build(); + } + private List<NodeRepositoryNode> createNodes() { List<NodeRepositoryNode> nodes = new ArrayList<>(); nodes.add(createNode("node1", "host1", "default", 0 )); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json index 08741e7f38a..17c93c070fb 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json @@ -7,6 +7,9 @@ "name": "ApplicationOwnershipConfirmer" }, { + "name": "ArchiveAccessMaintainer" + }, + { "name": "ArchiveUriUpdater" }, { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java index 1ad705be0b7..03f1d75a50b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.hosted.controller.restapi.user; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; -import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.controller.ControllerTester; @@ -255,7 +254,7 @@ public class UserApiTest extends ControllerContainerCloudTest { public void maxTrialTenants() { ContainerTester tester = new ContainerTester(container, responseFiles); ((InMemoryFlagSource) tester.controller().flagSource()) - .withIntFlag(Flags.MAX_TRIAL_TENANTS.id(), 1) + .withIntFlag(PermanentFlags.MAX_TRIAL_TENANTS.id(), 1) .withBooleanFlag(PermanentFlags.ENABLE_PUBLIC_SIGNUP_FLOW.id(), true); ControllerTester controller = new ControllerTester(tester); Set<Role> operator = Set.of(Role.hostedOperator(), Role.hostedSupporter(), Role.hostedAccountant()); diff --git a/dist/vespa.spec b/dist/vespa.spec index c357784fde7..3707d4afb9c 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -68,7 +68,7 @@ BuildRequires: vespa-gtest >= 1.8.1-1 BuildRequires: vespa-icu-devel >= 65.1.0-1 BuildRequires: vespa-lz4-devel >= 1.9.2-2 BuildRequires: vespa-onnxruntime-devel = 1.4.0 -BuildRequires: vespa-openssl-devel >= 1.1.1i-1 +BuildRequires: vespa-openssl-devel >= 1.1.1k-1 BuildRequires: vespa-protobuf-devel >= 3.7.0-4 BuildRequires: vespa-libzstd-devel >= 1.4.5-2 %endif @@ -185,7 +185,7 @@ Requires: llvm7.0 Requires: vespa-icu >= 65.1.0-1 Requires: vespa-lz4 >= 1.9.2-2 Requires: vespa-onnxruntime = 1.4.0 -Requires: vespa-openssl >= 1.1.1i-1 +Requires: vespa-openssl >= 1.1.1k-1 Requires: vespa-protobuf >= 3.7.0-4 Requires: vespa-telegraf >= 1.1.1-1 Requires: vespa-valgrind >= 3.16.0-1 @@ -274,7 +274,7 @@ Summary: Vespa - The open big data serving engine - base C++ libs Requires: xxhash-libs >= 0.8.0 %if 0%{?el7} -Requires: vespa-openssl >= 1.1.1i-1 +Requires: vespa-openssl >= 1.1.1k-1 %else Requires: openssl-libs %endif diff --git a/docproc/pom.xml b/docproc/pom.xml index 36e54e689ee..dd5adcee578 100644 --- a/docproc/pom.xml +++ b/docproc/pom.xml @@ -72,12 +72,6 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> - <artifactId>jdisc_messagebus_service</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> <artifactId>messagebus</artifactId> <version>${project.version}</version> <scope>provided</scope> diff --git a/eval/CMakeLists.txt b/eval/CMakeLists.txt index 9820163725d..302b6768cea 100644 --- a/eval/CMakeLists.txt +++ b/eval/CMakeLists.txt @@ -75,6 +75,7 @@ vespa_define_module( src/tests/instruction/sum_max_dot_product_function src/tests/instruction/vector_from_doubles_function src/tests/streamed/value + src/tests/tensor/binary_format src/tests/tensor/instruction_benchmark src/tests/tensor/onnx_wrapper src/tests/tensor/tensor_conformance diff --git a/eval/src/tests/eval/value_type/value_type_test.cpp b/eval/src/tests/eval/value_type/value_type_test.cpp index 120ab2d93e9..a2b25a12b4b 100644 --- a/eval/src/tests/eval/value_type/value_type_test.cpp +++ b/eval/src/tests/eval/value_type/value_type_test.cpp @@ -572,12 +572,22 @@ TEST("require that cell array size can be calculated") { } TEST("require that all cell types can be listed") { - std::vector<CellType> expect = {CellType::FLOAT, CellType::DOUBLE}; + std::vector<CellType> expect = { CellType::DOUBLE, CellType::FLOAT, CellType::BFLOAT16, CellType::INT8 }; + std::vector<CellType> expect_stable; + std::vector<CellType> expect_unstable; auto list = CellTypeUtils::list_types(); ASSERT_EQUAL(list.size(), expect.size()); for (size_t i = 0; i < list.size(); ++i) { EXPECT_TRUE(list[i] == expect[i]); + CellMeta cm(expect[i], false); + if (cm.decay().eq(cm)) { + expect_stable.push_back(cm.cell_type); + } else { + expect_unstable.push_back(cm.cell_type); + } } + EXPECT_TRUE(expect_stable == CellTypeUtils::list_stable_types()); + EXPECT_TRUE(expect_unstable == CellTypeUtils::list_unstable_types()); } TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/eval/src/tests/instruction/dense_multi_matmul_function/dense_multi_matmul_function_test.cpp b/eval/src/tests/instruction/dense_multi_matmul_function/dense_multi_matmul_function_test.cpp index 9fa25466f4a..9da7f379246 100644 --- a/eval/src/tests/instruction/dense_multi_matmul_function/dense_multi_matmul_function_test.cpp +++ b/eval/src/tests/instruction/dense_multi_matmul_function/dense_multi_matmul_function_test.cpp @@ -39,10 +39,11 @@ struct FunInfo { void verify_optimized(const vespalib::string &expr, const FunInfo &details) { TEST_STATE(expr.c_str()); - auto same_types = CellTypeSpace(CellTypeUtils::list_types(), 2).same(); - EvalFixture::verify<FunInfo>(expr, {details}, same_types); - auto diff_types = CellTypeSpace(CellTypeUtils::list_types(), 2).different(); - EvalFixture::verify<FunInfo>(expr, {}, diff_types); + CellTypeSpace stable_types(CellTypeUtils::list_stable_types(), 2); + CellTypeSpace unstable_types(CellTypeUtils::list_unstable_types(), 2); + EvalFixture::verify<FunInfo>(expr, {details}, CellTypeSpace(stable_types).same()); + EvalFixture::verify<FunInfo>(expr, {}, CellTypeSpace(stable_types).different()); + EvalFixture::verify<FunInfo>(expr, {}, unstable_types); } void verify_not_optimized(const vespalib::string &expr) { diff --git a/eval/src/tests/instruction/inplace_map_function/inplace_map_function_test.cpp b/eval/src/tests/instruction/inplace_map_function/inplace_map_function_test.cpp index 0983f84a4af..1193060b05a 100644 --- a/eval/src/tests/instruction/inplace_map_function/inplace_map_function_test.cpp +++ b/eval/src/tests/instruction/inplace_map_function/inplace_map_function_test.cpp @@ -27,8 +27,10 @@ struct FunInfo { void verify_optimized(const vespalib::string &expr) { SCOPED_TRACE(expr.c_str()); - CellTypeSpace all_types(CellTypeUtils::list_types(), 1); - EvalFixture::verify<FunInfo>(expr, {FunInfo{false}}, all_types); + CellTypeSpace stable_types(CellTypeUtils::list_stable_types(), 1); + CellTypeSpace unstable_types(CellTypeUtils::list_unstable_types(), 1); + EvalFixture::verify<FunInfo>(expr, {FunInfo{false}}, stable_types); + EvalFixture::verify<FunInfo>(expr, {}, unstable_types); } void verify_not_optimized(const vespalib::string &expr) { diff --git a/eval/src/tests/instruction/join_with_number/join_with_number_function_test.cpp b/eval/src/tests/instruction/join_with_number/join_with_number_function_test.cpp index a2f18d7f7f7..a6486de6858 100644 --- a/eval/src/tests/instruction/join_with_number/join_with_number_function_test.cpp +++ b/eval/src/tests/instruction/join_with_number/join_with_number_function_test.cpp @@ -48,9 +48,11 @@ struct FunInfo { void verify_optimized(const vespalib::string &expr, Primary primary, bool inplace) { // fprintf(stderr, "%s\n", expr.c_str()); - const auto stable_types = CellTypeSpace({CellType::FLOAT, CellType::DOUBLE}, 2); + const CellTypeSpace stable_types(CellTypeUtils::list_stable_types(), 2); FunInfo stable_details{primary, inplace}; TEST_DO(EvalFixture::verify<FunInfo>(expr, {stable_details}, stable_types)); + const CellTypeSpace unstable_types(CellTypeUtils::list_unstable_types(), 2); + TEST_DO(EvalFixture::verify<FunInfo>(expr, {}, unstable_types)); } void verify_not_optimized(const vespalib::string &expr) { diff --git a/eval/src/tests/instruction/pow_as_map_optimizer/pow_as_map_optimizer_test.cpp b/eval/src/tests/instruction/pow_as_map_optimizer/pow_as_map_optimizer_test.cpp index e5a8cd9e92c..d123f9c89a6 100644 --- a/eval/src/tests/instruction/pow_as_map_optimizer/pow_as_map_optimizer_test.cpp +++ b/eval/src/tests/instruction/pow_as_map_optimizer/pow_as_map_optimizer_test.cpp @@ -31,15 +31,17 @@ struct InplaceInfo { void verify_optimized(const vespalib::string &expr, op1_t op1, bool inplace = false) { SCOPED_TRACE(expr.c_str()); + CellTypeSpace stable_types(CellTypeUtils::list_stable_types(), 1); if (inplace) { InplaceInfo details{op1}; - auto all_types = CellTypeSpace(CellTypeUtils::list_types(), 1); - EvalFixture::verify<InplaceInfo>(expr, {details}, all_types); + EvalFixture::verify<InplaceInfo>(expr, {details}, stable_types); } else { MapInfo details{op1}; - auto all_types = CellTypeSpace(CellTypeUtils::list_types(), 1); - EvalFixture::verify<MapInfo>(expr, {details}, all_types); + EvalFixture::verify<MapInfo>(expr, {details}, stable_types); } + MapInfo details{op1}; + CellTypeSpace unstable_types(CellTypeUtils::list_unstable_types(), 1); + EvalFixture::verify<MapInfo>(expr, {details}, unstable_types); } void verify_not_optimized(const vespalib::string &expr) { diff --git a/eval/src/tests/tensor/binary_format/.gitignore b/eval/src/tests/tensor/binary_format/.gitignore new file mode 100644 index 00000000000..4f0fdc51492 --- /dev/null +++ b/eval/src/tests/tensor/binary_format/.gitignore @@ -0,0 +1 @@ +/binary_test_spec.json diff --git a/eval/src/tests/tensor/binary_format/CMakeLists.txt b/eval/src/tests/tensor/binary_format/CMakeLists.txt new file mode 100644 index 00000000000..ac52c2b0365 --- /dev/null +++ b/eval/src/tests/tensor/binary_format/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(eval_tensor_binary_format_test_app TEST + SOURCES + binary_format_test.cpp + DEPENDS + vespaeval + GTest::GTest +) +vespa_add_test(NAME eval_tensor_binary_format_test_app COMMAND eval_tensor_binary_format_test_app) diff --git a/eval/src/tests/tensor/binary_format/binary_format_test.cpp b/eval/src/tests/tensor/binary_format/binary_format_test.cpp new file mode 100644 index 00000000000..671765d4050 --- /dev/null +++ b/eval/src/tests/tensor/binary_format/binary_format_test.cpp @@ -0,0 +1,142 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/eval/eval/test/test_io.h> +#include <vespa/eval/eval/test/gen_spec.h> +#include <vespa/eval/eval/cell_type.h> +#include <vespa/eval/eval/tensor_spec.h> +#include <vespa/eval/eval/simple_value.h> +#include <vespa/eval/streamed/streamed_value_builder_factory.h> +#include <vespa/eval/eval/fast_value.h> +#include <vespa/eval/eval/value_codec.h> +#include <vespa/vespalib/io/mapped_file_input.h> +#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/objects/nbostream.h> +#include <vespa/vespalib/gtest/gtest.h> + +using namespace vespalib; +using namespace vespalib::eval; +using namespace vespalib::eval::test; +using namespace vespalib::slime::convenience; + +using vespalib::make_string_short::fmt; + +vespalib::string get_source_dir() { + const char *dir = getenv("SOURCE_DIRECTORY"); + return (dir ? dir : "."); +} +vespalib::string source_dir = get_source_dir(); +vespalib::string module_src_path = source_dir + "/../../../../"; +vespalib::string module_build_path = "../../../../"; + +const ValueBuilderFactory &simple = SimpleValueBuilderFactory::get(); +const ValueBuilderFactory &streamed = StreamedValueBuilderFactory::get(); +const ValueBuilderFactory &fast = FastValueBuilderFactory::get(); + +TEST(TensorBinaryFormatTest, tensor_binary_format_test_spec_can_be_generated) { + vespalib::string spec = module_src_path + "src/apps/make_tensor_binary_format_test_spec/test_spec.json"; + vespalib::string binary = module_build_path + "src/apps/make_tensor_binary_format_test_spec/eval_make_tensor_binary_format_test_spec_app"; + EXPECT_EQ(system(fmt("%s > binary_test_spec.json", binary.c_str()).c_str()), 0); + EXPECT_EQ(system(fmt("diff -u %s binary_test_spec.json", spec.c_str()).c_str()), 0); +} + +void verify_encode_decode(const TensorSpec &spec, + const ValueBuilderFactory &encode_factory, + const ValueBuilderFactory &decode_factory) +{ + nbostream data; + auto value = value_from_spec(spec, encode_factory); + encode_value(*value, data); + auto value2 = decode_value(data, decode_factory); + TensorSpec spec2 = spec_from_value(*value2); + EXPECT_EQ(spec2, spec); +} + +void verify_encode_decode(const GenSpec &spec) { + for (CellType ct : CellTypeUtils::list_types()) { + auto my_spec = spec.cpy().cells(ct); + if (my_spec.bad_scalar()) continue; + auto my_tspec = my_spec.gen(); + verify_encode_decode(my_tspec, simple, fast); + verify_encode_decode(my_tspec, fast, simple); + verify_encode_decode(my_tspec, simple, streamed); + verify_encode_decode(my_tspec, streamed, simple); + } +} + +TEST(TensorBinaryFormatTest, encode_decode) { + verify_encode_decode(GenSpec(42)); + verify_encode_decode(GenSpec().idx("x", 3)); + verify_encode_decode(GenSpec().idx("x", 3).idx("y", 5)); + verify_encode_decode(GenSpec().idx("x", 3).idx("y", 5).idx("z", 7)); + verify_encode_decode(GenSpec().map("x", 3)); + verify_encode_decode(GenSpec().map("x", 3).map("y", 2)); + verify_encode_decode(GenSpec().map("x", 3).map("y", 2).map("z", 4)); + verify_encode_decode(GenSpec().idx("x", 3).map("y", 2).idx("z", 7)); + verify_encode_decode(GenSpec().map("x", 3).idx("y", 5).map("z", 4)); +} + +uint8_t unhex(char c) { + if (c >= '0' && c <= '9') { + return (c - '0'); + } + if (c >= 'A' && c <= 'F') { + return ((c - 'A') + 10); + } + EXPECT_TRUE(false) << "bad hex char"; + return 0; +} + +nbostream extract_data(const Memory &hex_dump) { + nbostream data; + if ((hex_dump.size > 2) && (hex_dump.data[0] == '0') && (hex_dump.data[1] == 'x')) { + for (size_t i = 2; i < (hex_dump.size - 1); i += 2) { + data << uint8_t((unhex(hex_dump.data[i]) << 4) | unhex(hex_dump.data[i + 1])); + } + } + return data; +} + +bool is_same(const nbostream &a, const nbostream &b) { + return (Memory(a.peek(), a.size()) == Memory(b.peek(), b.size())); +} + +void test_binary_format_spec(const Inspector &test, const ValueBuilderFactory &factory) { + Stash stash; + TensorSpec spec = TensorSpec::from_slime(test["tensor"]); + const Inspector &binary = test["binary"]; + EXPECT_GT(binary.entries(), 0u); + nbostream encoded; + encode_value(*value_from_spec(spec, factory), encoded); + bool matched_encode = false; + for (size_t i = 0; i < binary.entries(); ++i) { + nbostream data = extract_data(binary[i].asString()); + matched_encode = (matched_encode || is_same(encoded, data)); + EXPECT_EQ(spec_from_value(*decode_value(data, factory)), spec); + EXPECT_EQ(data.size(), 0u); + } + EXPECT_TRUE(matched_encode); +} + +void test_binary_format_spec(Cursor &test) { + test_binary_format_spec(test, simple); + test_binary_format_spec(test, streamed); + test_binary_format_spec(test, fast); +} + +TEST(TensorBinaryFormatTest, tensor_binary_format_test_spec) { + vespalib::string path = module_src_path; + path.append("src/apps/make_tensor_binary_format_test_spec/test_spec.json"); + MappedFileInput file(path); + EXPECT_TRUE(file.valid()); + auto handle_test = [this](Slime &slime) + { + test_binary_format_spec(slime.get()); + }; + auto handle_summary = [](Slime &slime) + { + EXPECT_GT(slime["num_tests"].asLong(), 0); + }; + for_each_test(file, handle_test, handle_summary); +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/eval/src/tests/tensor/tensor_conformance/tensor_conformance_test.cpp b/eval/src/tests/tensor/tensor_conformance/tensor_conformance_test.cpp index 15aa7212a94..3d21f9b4113 100644 --- a/eval/src/tests/tensor/tensor_conformance/tensor_conformance_test.cpp +++ b/eval/src/tests/tensor/tensor_conformance/tensor_conformance_test.cpp @@ -32,13 +32,6 @@ TEST("require that FastValue implementation passes all conformance tests") { TEST_DO(TensorConformance::run_tests(module_src_path, FastValueBuilderFactory::get())); } -TEST("require that tensor serialization test spec can be generated") { - vespalib::string spec = module_src_path + "src/apps/make_tensor_binary_format_test_spec/test_spec.json"; - vespalib::string binary = module_build_path + "src/apps/make_tensor_binary_format_test_spec/eval_make_tensor_binary_format_test_spec_app"; - EXPECT_EQUAL(system(fmt("%s > binary_test_spec.json", binary.c_str()).c_str()), 0); - EXPECT_EQUAL(system(fmt("diff -u %s binary_test_spec.json", spec.c_str()).c_str()), 0); -} - TEST("require that cross-language tensor conformance tests pass with C++ expression evaluation") { vespalib::string result_file = "conformance_result.json"; vespalib::string binary = module_build_path + "src/apps/tensor_conformance/vespa-tensor-conformance"; diff --git a/eval/src/vespa/eval/eval/binary_format.txt b/eval/src/vespa/eval/eval/binary_format.txt index 4d74c21005a..955f821fe00 100644 --- a/eval/src/vespa/eval/eval/binary_format.txt +++ b/eval/src/vespa/eval/eval/binary_format.txt @@ -23,7 +23,7 @@ mixed_with_cell_type[7]. (mixed tensors are tagged as both 'sparse' and 'dense') if ('with_cell_type') - 1_4_int -> cell_type (0:double, 1:float) + 1_4_int -> cell_type (0:double, 1:float, 2:bfloat16, 3:int8) if ('sparse'): 1_4_int: number of mapped dimensions -> 'n_mapped' diff --git a/eval/src/vespa/eval/eval/cell_type.cpp b/eval/src/vespa/eval/eval/cell_type.cpp index 753d888c24c..94bd0b14573 100644 --- a/eval/src/vespa/eval/eval/cell_type.cpp +++ b/eval/src/vespa/eval/eval/cell_type.cpp @@ -33,7 +33,19 @@ CellTypeUtils::mem_size(CellType cell_type, size_t sz) std::vector<CellType> CellTypeUtils::list_types() { - return {CellType::FLOAT, CellType::DOUBLE}; + return {CellType::DOUBLE, CellType::FLOAT, CellType::BFLOAT16, CellType::INT8 }; +} + +std::vector<CellType> +CellTypeUtils::list_stable_types() +{ + return {CellType::DOUBLE, CellType::FLOAT}; +} + +std::vector<CellType> +CellTypeUtils::list_unstable_types() +{ + return {CellType::BFLOAT16, CellType::INT8 }; } } diff --git a/eval/src/vespa/eval/eval/cell_type.h b/eval/src/vespa/eval/eval/cell_type.h index 57f707c2aa8..79750c5c875 100644 --- a/eval/src/vespa/eval/eval/cell_type.h +++ b/eval/src/vespa/eval/eval/cell_type.h @@ -6,15 +6,19 @@ #include <vector> #include <cstdint> #include <cassert> +#include "int8float.h" +#include <vespa/vespalib/util/bfloat16.h> namespace vespalib::eval { -enum class CellType : char { FLOAT, DOUBLE }; +enum class CellType : char { DOUBLE, FLOAT, BFLOAT16, INT8 }; // converts actual cell type to CellType enum value template <typename CT> constexpr CellType get_cell_type(); template <> constexpr CellType get_cell_type<double>() { return CellType::DOUBLE; } template <> constexpr CellType get_cell_type<float>() { return CellType::FLOAT; } +template <> constexpr CellType get_cell_type<BFloat16>() { return CellType::BFLOAT16; } +template <> constexpr CellType get_cell_type<Int8Float>() { return CellType::INT8; } // check if the given CellType enum value and actual cell type match template <typename CT> constexpr bool check_cell_type(CellType type) { @@ -29,6 +33,10 @@ template <CellType cell_type> constexpr auto get_cell_value() { return double(); } else if constexpr (cell_type == CellType::FLOAT) { return float(); + } else if constexpr (cell_type == CellType::BFLOAT16) { + return BFloat16(); + } else if constexpr (cell_type == CellType::INT8) { + return Int8Float(); } else { static_assert((cell_type == CellType::DOUBLE), "unknown cell type"); } @@ -136,9 +144,11 @@ struct CellMeta { struct TypifyCellType { template <typename T> using Result = TypifyResultType<T>; template <typename F> static decltype(auto) resolve(CellType value, F &&f) { - switch (value) { - case CellType::DOUBLE: return f(Result<double>()); - case CellType::FLOAT: return f(Result<float>()); + switch(value) { + case CellType::DOUBLE: return f(Result<double>()); + case CellType::FLOAT: return f(Result<float>()); + case CellType::BFLOAT16: return f(Result<BFloat16>()); + case CellType::INT8: return f(Result<Int8Float>()); } abort(); } @@ -158,8 +168,10 @@ struct TypifyCellMeta { } template <typename F> static decltype(auto) resolve(CellMetaNotScalar value, F &&f) { switch (value.cell_type) { - case CellType::DOUBLE: return f(Result<CellMeta(CellType::DOUBLE, false)>()); - case CellType::FLOAT: return f(Result<CellMeta(CellType::FLOAT, false)>()); + case CellType::DOUBLE: return f(Result<CellMeta(CellType::DOUBLE, false)>()); + case CellType::FLOAT: return f(Result<CellMeta(CellType::FLOAT, false)>()); + case CellType::BFLOAT16: return f(Result<CellMeta(CellType::BFLOAT16, false)>()); + case CellType::INT8: return f(Result<CellMeta(CellType::INT8, false)>()); } abort(); } @@ -175,8 +187,8 @@ struct TypifyCellMeta { } template <typename F> static decltype(auto) resolve(LimitedCellMetaNotScalar value, F &&f) { switch (value.cell_type) { - case CellType::DOUBLE: return f(Result<CellMeta(CellType::DOUBLE, false)>()); - case CellType::FLOAT: return f(Result<CellMeta(CellType::FLOAT, false)>()); + case CellType::DOUBLE: return f(Result<CellMeta(CellType::DOUBLE, false)>()); + case CellType::FLOAT: return f(Result<CellMeta(CellType::FLOAT, false)>()); default: break; } abort(); @@ -187,6 +199,8 @@ struct CellTypeUtils { static uint32_t alignment(CellType cell_type); static size_t mem_size(CellType cell_type, size_t sz); static std::vector<CellType> list_types(); + static std::vector<CellType> list_stable_types(); + static std::vector<CellType> list_unstable_types(); }; } // namespace diff --git a/eval/src/vespa/eval/eval/dense_cells_value.cpp b/eval/src/vespa/eval/eval/dense_cells_value.cpp index 126ef806668..a699153242d 100644 --- a/eval/src/vespa/eval/eval/dense_cells_value.cpp +++ b/eval/src/vespa/eval/eval/dense_cells_value.cpp @@ -15,5 +15,7 @@ DenseCellsValue<T>::get_memory_usage() const { template class DenseCellsValue<double>; template class DenseCellsValue<float>; +template class DenseCellsValue<BFloat16>; +template class DenseCellsValue<Int8Float>; } diff --git a/eval/src/vespa/eval/eval/test/gen_spec.cpp b/eval/src/vespa/eval/eval/test/gen_spec.cpp index 0b624a457d7..fd2c1f39382 100644 --- a/eval/src/vespa/eval/eval/test/gen_spec.cpp +++ b/eval/src/vespa/eval/eval/test/gen_spec.cpp @@ -4,6 +4,7 @@ #include <vespa/eval/eval/string_stuff.h> #include <vespa/vespalib/util/require.h> #include <vespa/vespalib/util/stringfmt.h> +#include <ostream> using vespalib::make_string_short::fmt; @@ -162,4 +163,10 @@ GenSpec::gen() const return result.normalize(); } +std::ostream &operator<<(std::ostream &out, const GenSpec &spec) +{ + out << spec.gen(); + return out; +} + } // namespace diff --git a/eval/src/vespa/eval/eval/test/gen_spec.h b/eval/src/vespa/eval/eval/test/gen_spec.h index f0eca6074dc..3f7550ba644 100644 --- a/eval/src/vespa/eval/eval/test/gen_spec.h +++ b/eval/src/vespa/eval/eval/test/gen_spec.h @@ -153,4 +153,6 @@ public: operator TensorSpec() const { return gen(); } }; +std::ostream &operator<<(std::ostream &out, const GenSpec &spec); + } // namespace diff --git a/eval/src/vespa/eval/eval/test/tensor_conformance.cpp b/eval/src/vespa/eval/eval/test/tensor_conformance.cpp index 30933c78325..c58f8312cbf 100644 --- a/eval/src/vespa/eval/eval/test/tensor_conformance.cpp +++ b/eval/src/vespa/eval/eval/test/tensor_conformance.cpp @@ -15,7 +15,6 @@ #include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/io/mapped_file_input.h> #include "tensor_model.h" -#include "test_io.h" #include "reference_evaluation.h" using vespalib::make_string_short::fmt; @@ -70,31 +69,6 @@ void verify_result(const ValueBuilderFactory &factory, const vespalib::string &e // NaN value const double my_nan = std::numeric_limits<double>::quiet_NaN(); -uint8_t unhex(char c) { - if (c >= '0' && c <= '9') { - return (c - '0'); - } - if (c >= 'A' && c <= 'F') { - return ((c - 'A') + 10); - } - TEST_ERROR("bad hex char"); - return 0; -} - -nbostream extract_data(const Memory &hex_dump) { - nbostream data; - if ((hex_dump.size > 2) && (hex_dump.data[0] == '0') && (hex_dump.data[1] == 'x')) { - for (size_t i = 2; i < (hex_dump.size - 1); i += 2) { - data << uint8_t((unhex(hex_dump.data[i]) << 4) | unhex(hex_dump.data[i + 1])); - } - } - return data; -} - -bool is_same(const nbostream &a, const nbostream &b) { - return (Memory(a.peek(), a.size()) == Memory(b.peek(), b.size())); -} - // Test wrapper to avoid passing global test parameters around struct TestContext { @@ -678,82 +652,6 @@ struct TestContext { //------------------------------------------------------------------------- - void verify_encode_decode(const TensorSpec &spec, - const ValueBuilderFactory &encode_factory, - const ValueBuilderFactory &decode_factory) - { - nbostream data; - auto value = value_from_spec(spec, encode_factory); - encode_value(*value, data); - auto value2 = decode_value(data, decode_factory); - TensorSpec spec2 = spec_from_value(*value2); - EXPECT_EQUAL(spec2, spec); - } - - void verify_encode_decode(const TensorSpec &spec) { - const ValueBuilderFactory &simple = SimpleValueBuilderFactory::get(); - TEST_DO(verify_encode_decode(spec, factory, simple)); - if (&factory != &simple) { - TEST_DO(verify_encode_decode(spec, simple, factory)); - } - } - - void test_binary_format_spec(Cursor &test) { - Stash stash; - TensorSpec spec = TensorSpec::from_slime(test["tensor"]); - const Inspector &binary = test["binary"]; - EXPECT_GREATER(binary.entries(), 0u); - nbostream encoded; - encode_value(*value_from_spec(spec, factory), encoded); - test.setData("encoded", Memory(encoded.peek(), encoded.size())); - bool matched_encode = false; - for (size_t i = 0; i < binary.entries(); ++i) { - nbostream data = extract_data(binary[i].asString()); - matched_encode = (matched_encode || is_same(encoded, data)); - EXPECT_EQUAL(spec_from_value(*decode_value(data, factory)), spec); - EXPECT_EQUAL(data.size(), 0u); - } - EXPECT_TRUE(matched_encode); - } - - void test_binary_format_spec() { - vespalib::string path = module_path; - path.append("src/apps/make_tensor_binary_format_test_spec/test_spec.json"); - MappedFileInput file(path); - EXPECT_TRUE(file.valid()); - auto handle_test = [this](Slime &slime) - { - size_t fail_cnt = TEST_MASTER.getProgress().failCnt; - TEST_DO(test_binary_format_spec(slime.get())); - if (TEST_MASTER.getProgress().failCnt > fail_cnt) { - fprintf(stderr, "failed:\n%s", slime.get().toString().c_str()); - } - }; - auto handle_summary = [](Slime &slime) - { - EXPECT_GREATER(slime["num_tests"].asLong(), 0); - }; - for_each_test(file, handle_test, handle_summary); - } - - void test_binary_format() { - TEST_DO(test_binary_format_spec()); - TEST_DO(verify_encode_decode(spec(42))); - TEST_DO(verify_encode_decode(spec({x(3)}, N()))); - TEST_DO(verify_encode_decode(spec({x(3),y(5)}, N()))); - TEST_DO(verify_encode_decode(spec({x(3),y(5),z(7)}, N()))); - TEST_DO(verify_encode_decode(spec(float_cells({x(3),y(5),z(7)}), N()))); - TEST_DO(verify_encode_decode(spec({x({"a","b","c"})}, N()))); - TEST_DO(verify_encode_decode(spec({x({"a","b","c"}),y({"foo","bar"})}, N()))); - TEST_DO(verify_encode_decode(spec({x({"a","b","c"}),y({"foo","bar"}),z({"i","j","k","l"})}, N()))); - TEST_DO(verify_encode_decode(spec(float_cells({x({"a","b","c"}),y({"foo","bar"}),z({"i","j","k","l"})}), N()))); - TEST_DO(verify_encode_decode(spec({x(3),y({"foo", "bar"}),z(7)}, N()))); - TEST_DO(verify_encode_decode(spec({x({"a","b","c"}),y(5),z({"i","j","k","l"})}, N()))); - TEST_DO(verify_encode_decode(spec(float_cells({x({"a","b","c"}),y(5),z({"i","j","k","l"})}), N()))); - } - - //------------------------------------------------------------------------- - void run_tests() { TEST_DO(test_tensor_create_type()); TEST_DO(test_tensor_reduce()); @@ -767,7 +665,6 @@ struct TestContext { TEST_DO(test_tensor_create()); TEST_DO(test_tensor_peek()); TEST_DO(test_tensor_merge()); - TEST_DO(test_binary_format()); } }; diff --git a/eval/src/vespa/eval/eval/typed_cells.h b/eval/src/vespa/eval/eval/typed_cells.h index b65fa2b40e4..581de355694 100644 --- a/eval/src/vespa/eval/eval/typed_cells.h +++ b/eval/src/vespa/eval/eval/typed_cells.h @@ -17,6 +17,8 @@ struct TypedCells { explicit TypedCells(ConstArrayRef<double> cells) : data(cells.begin()), type(CellType::DOUBLE), size(cells.size()) {} explicit TypedCells(ConstArrayRef<float> cells) : data(cells.begin()), type(CellType::FLOAT), size(cells.size()) {} + explicit TypedCells(ConstArrayRef<BFloat16> cells) : data(cells.begin()), type(CellType::BFLOAT16), size(cells.size()) {} + explicit TypedCells(ConstArrayRef<Int8Float> cells) : data(cells.begin()), type(CellType::INT8), size(cells.size()) {} TypedCells() : data(nullptr), type(CellType::DOUBLE), size(0) {} TypedCells(const void *dp, CellType ct, size_t sz) : data(dp), type(ct), size(sz) {} diff --git a/eval/src/vespa/eval/eval/value_codec.cpp b/eval/src/vespa/eval/eval/value_codec.cpp index 85feadca85e..bd9d36bed2f 100644 --- a/eval/src/vespa/eval/eval/value_codec.cpp +++ b/eval/src/vespa/eval/eval/value_codec.cpp @@ -20,11 +20,15 @@ namespace { constexpr uint32_t DOUBLE_CELL_TYPE = 0; constexpr uint32_t FLOAT_CELL_TYPE = 1; +constexpr uint32_t BFLOAT16_CELL_TYPE = 2; +constexpr uint32_t INT8_CELL_TYPE = 3; inline uint32_t cell_type_to_id(CellType cell_type) { switch (cell_type) { case CellType::DOUBLE: return DOUBLE_CELL_TYPE; case CellType::FLOAT: return FLOAT_CELL_TYPE; + case CellType::BFLOAT16: return BFLOAT16_CELL_TYPE; + case CellType::INT8: return INT8_CELL_TYPE; } throw IllegalArgumentException(fmt("Unknown CellType=%u", (uint32_t)cell_type)); } @@ -33,6 +37,8 @@ inline CellType id_to_cell_type(uint32_t id) { switch (id) { case DOUBLE_CELL_TYPE: return CellType::DOUBLE; case FLOAT_CELL_TYPE: return CellType::FLOAT; + case BFLOAT16_CELL_TYPE: return CellType::BFLOAT16; + case INT8_CELL_TYPE: return CellType::INT8; } throw IllegalArgumentException(fmt("Unknown CellType id=%u", id)); } diff --git a/eval/src/vespa/eval/eval/value_type_spec.cpp b/eval/src/vespa/eval/eval/value_type_spec.cpp index b518ccd1b30..92646ed1f67 100644 --- a/eval/src/vespa/eval/eval/value_type_spec.cpp +++ b/eval/src/vespa/eval/eval/value_type_spec.cpp @@ -10,8 +10,10 @@ namespace vespalib::eval::value_type { vespalib::string cell_type_to_name(CellType cell_type) { switch (cell_type) { - case CellType::DOUBLE: return "double"; - case CellType::FLOAT: return "float"; + case CellType::DOUBLE: return "double"; + case CellType::FLOAT: return "float"; + case CellType::BFLOAT16: return "bfloat16"; + case CellType::INT8: return "int8"; } abort(); } diff --git a/eval/src/vespa/eval/onnx/onnx_wrapper.cpp b/eval/src/vespa/eval/onnx/onnx_wrapper.cpp index 2891b37ebe8..e9758f2ddc8 100644 --- a/eval/src/vespa/eval/onnx/onnx_wrapper.cpp +++ b/eval/src/vespa/eval/onnx/onnx_wrapper.cpp @@ -1,6 +1,7 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "onnx_wrapper.h" +#include <vespa/eval/eval/cell_type.h> #include <vespa/eval/eval/dense_cells_value.h> #include <vespa/eval/eval/value_type.h> #include <vespa/vespalib/util/arrayref.h> @@ -21,6 +22,14 @@ using vespalib::ConstArrayRef; using vespalib::make_string_short::fmt; +// as documented in onnxruntime_cxx_api.h : +namespace Ort { +template <> +struct TypeToTensorType<vespalib::BFloat16> { static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_BFLOAT16; }; +template <> +struct TypeToTensorType<vespalib::eval::Int8Float> { static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8; }; +} + namespace vespalib::eval { namespace { diff --git a/eval/src/vespa/eval/streamed/streamed_value.cpp b/eval/src/vespa/eval/streamed/streamed_value.cpp index c09e433b9b9..63765d9b1da 100644 --- a/eval/src/vespa/eval/streamed/streamed_value.cpp +++ b/eval/src/vespa/eval/streamed/streamed_value.cpp @@ -22,6 +22,8 @@ StreamedValue<T>::get_memory_usage() const template class StreamedValue<double>; template class StreamedValue<float>; +template class StreamedValue<BFloat16>; +template class StreamedValue<Int8Float>; } // namespace diff --git a/eval/src/vespa/eval/streamed/streamed_value_builder.cpp b/eval/src/vespa/eval/streamed/streamed_value_builder.cpp index 957121c42b7..ba5d92bf5d2 100644 --- a/eval/src/vespa/eval/streamed/streamed_value_builder.cpp +++ b/eval/src/vespa/eval/streamed/streamed_value_builder.cpp @@ -9,5 +9,7 @@ StreamedValueBuilder<T>::~StreamedValueBuilder() = default; template class StreamedValueBuilder<double>; template class StreamedValueBuilder<float>; +template class StreamedValueBuilder<BFloat16>; +template class StreamedValueBuilder<Int8Float>; } // namespace diff --git a/fat-model-dependencies/pom.xml b/fat-model-dependencies/pom.xml index d3228cf3933..a934024b4b9 100644 --- a/fat-model-dependencies/pom.xml +++ b/fat-model-dependencies/pom.xml @@ -141,11 +141,6 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> - <artifactId>messagebus-disc</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> <artifactId>container-messagebus</artifactId> <version>${project.version}</version> </dependency> diff --git a/flags/src/main/java/com/yahoo/vespa/flags/BooleanFlag.java b/flags/src/main/java/com/yahoo/vespa/flags/BooleanFlag.java index 1245aca65b8..0a4b8a1c5c2 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/BooleanFlag.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/BooleanFlag.java @@ -12,6 +12,11 @@ public class BooleanFlag extends FlagImpl<Boolean, BooleanFlag> { super(id, defaultValue, vector, serializer, source, BooleanFlag::new); } + @Override + public BooleanFlag self() { + return this; + } + public boolean value() { return boxedValue(); } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/DoubleFlag.java b/flags/src/main/java/com/yahoo/vespa/flags/DoubleFlag.java index 8c3974727c6..9d606887715 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/DoubleFlag.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/DoubleFlag.java @@ -12,6 +12,11 @@ public class DoubleFlag extends FlagImpl<Double, DoubleFlag> { super(id, defaultValue, vector, serializer, source, DoubleFlag::new); } + @Override + public DoubleFlag self() { + return this; + } + public double value() { return boxedValue(); } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/FetchVector.java b/flags/src/main/java/com/yahoo/vespa/flags/FetchVector.java index 37849b65adf..ede7bd6a109 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/FetchVector.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/FetchVector.java @@ -33,9 +33,12 @@ public class FetchVector { /** Node type from com.yahoo.config.provision.NodeType::name, e.g. tenant, host, confighost, controller, etc. */ NODE_TYPE, - /** Cluster type from com.yahoo.config.provision.ClusterSpec.Type::name, e.g. content, container, admin */ + /** Cluster type from com.yahoo.config.provision.ClusterSpec.Type::value, e.g. content, container, admin */ CLUSTER_TYPE, + /** Cluster ID from com.yahoo.config.provision.ClusterSpec.Id::value, e.g. cluster-controllers, logserver. */ + CLUSTER_ID, + /** * Fully qualified hostname. * diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flag.java b/flags/src/main/java/com/yahoo/vespa/flags/Flag.java index 7bf491a39c7..718478ab426 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flag.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flag.java @@ -1,6 +1,8 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.flags; +import java.util.Optional; + /** * Interface for flag. * @@ -12,12 +14,20 @@ public interface Flag<T, F> { /** The flag ID. */ FlagId id(); + /** A generic type-safe method for getting {@code this}. */ + F self(); + /** Returns the flag serializer. */ FlagSerializer<T> serializer(); /** Returns an immutable clone of the current object, except with the dimension set accordingly. */ F with(FetchVector.Dimension dimension, String dimensionValue); + /** Same as {@link #with(FetchVector.Dimension, String)} if value is present, and otherwise returns {@code this}. */ + default F with(FetchVector.Dimension dimension, Optional<String> dimensionValue) { + return dimensionValue.map(value -> with(dimension, value)).orElse(self()); + } + /** Returns the value, boxed if the flag wraps a primitive type. */ T boxedValue(); } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 4576b546d2e..460741a6b8e 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -13,6 +13,7 @@ import java.util.Optional; import java.util.TreeMap; import static com.yahoo.vespa.flags.FetchVector.Dimension.APPLICATION_ID; +import static com.yahoo.vespa.flags.FetchVector.Dimension.CLUSTER_ID; import static com.yahoo.vespa.flags.FetchVector.Dimension.CLUSTER_TYPE; import static com.yahoo.vespa.flags.FetchVector.Dimension.HOSTNAME; import static com.yahoo.vespa.flags.FetchVector.Dimension.NODE_TYPE; @@ -103,46 +104,39 @@ public class Flags { public static final UnboundBooleanFlag USE_THREE_PHASE_UPDATES = defineFeatureFlag( "use-three-phase-updates", false, - List.of("vekterli"), "2020-12-02", "2021-04-01", + List.of("vekterli"), "2020-12-02", "2021-06-01", "Whether to enable the use of three-phase updates when bucket replicas are out of sync.", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); public static final UnboundStringFlag TLS_FOR_ZOOKEEPER_CLIENT_SERVER_COMMUNICATION = defineStringFlag( - "tls-for-zookeeper-client-server-communication", "OFF", - List.of("hmusum"), "2020-12-02", "2021-04-01", + "tls-for-zookeeper-client-server-communication", "TLS_WITH_PORT_UNIFICATION", + List.of("hmusum"), "2020-12-02", "2021-06-01", "How to setup TLS for ZooKeeper client/server communication. Valid values are OFF, PORT_UNIFICATION, TLS_WITH_PORT_UNIFICATION, TLS_ONLY", "Takes effect on restart of config server", NODE_TYPE, HOSTNAME); public static final UnboundBooleanFlag USE_TLS_FOR_ZOOKEEPER_CLIENT = defineFeatureFlag( - "use-tls-for-zookeeper-client", false, - List.of("hmusum"), "2020-12-02", "2021-04-01", + "use-tls-for-zookeeper-client", true, + List.of("hmusum"), "2020-12-02", "2021-05-01", "Whether to use TLS for ZooKeeper clients", "Takes effect on restart of process", NODE_TYPE, HOSTNAME); public static final UnboundBooleanFlag PROVISION_TENANT_ROLES = defineFeatureFlag( "provision-tenant-roles", false, - List.of("tokle"), "2020-12-02", "2021-04-01", + List.of("tokle"), "2020-12-02", "2021-06-01", "Whether tenant roles should be provisioned", "Takes effect on next deployment (controller)", TENANT_ID); public static final UnboundBooleanFlag TENANT_IAM_ROLE = defineFeatureFlag( "application-iam-roles", false, - List.of("tokle"), "2020-12-02", "2021-04-01", + List.of("tokle"), "2020-12-02", "2021-06-01", "Allow separate iam roles when provisioning/assigning hosts", "Takes effect immediately on new hosts, on next redeploy for applications", TENANT_ID); - public static final UnboundIntFlag MAX_TRIAL_TENANTS = defineIntFlag( - "max-trial-tenants", -1, - List.of("ogronnesby"), "2020-12-03", "2021-04-01", - "The maximum nr. of tenants with trial plan, -1 is unlimited", - "Takes effect immediately" - ); - public static final UnboundBooleanFlag HIDE_SHARED_ROUTING_ENDPOINT = defineFeatureFlag( "hide-shared-routing-endpoint", false, List.of("tokle"), "2020-12-02", "2021-06-01", @@ -151,13 +145,6 @@ public class Flags { APPLICATION_ID ); - public static final UnboundBooleanFlag USE_ACCESS_CONTROL_CLIENT_AUTHENTICATION = defineFeatureFlag( - "use-access-control-client-authentication", false, - List.of("tokle"), "2020-12-02", "2021-04-01", - "Whether application container should set up client authentication on default port based on access control element", - "Takes effect on next internal redeployment", - APPLICATION_ID); - public static final UnboundBooleanFlag USE_ASYNC_MESSAGE_HANDLING_ON_SCHEDULE = defineFeatureFlag( "async-message-handling-on-schedule", false, List.of("baldersheim"), "2020-12-02", "2022-01-01", @@ -166,8 +153,8 @@ public class Flags { ZONE_ID, APPLICATION_ID); public static final UnboundIntFlag MAX_PENDING_MOVE_OPS = defineIntFlag( - "max-pending-move-ops", 10, - List.of("baldersheim"), "2021-02-15", "2021-04-01", + "max-pending-move-ops", 100, + List.of("baldersheim"), "2021-02-15", "2021-05-01", "Max number of move operations inflight", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); @@ -180,26 +167,19 @@ public class Flags { ZONE_ID, APPLICATION_ID); public static final UnboundBooleanFlag USE_BUCKET_EXECUTOR_FOR_LID_SPACE_COMPACT = defineFeatureFlag( - "use-bucket-executor-for-lid-space-compact", false, - List.of("baldersheim"), "2021-01-24", "2021-04-01", + "use-bucket-executor-for-lid-space-compact", true, + List.of("baldersheim"), "2021-01-24", "2021-05-01", "Wheter to use content-level bucket executor or legacy frozen buckets", "Takes effect on next internal redeployment", APPLICATION_ID); public static final UnboundBooleanFlag USE_BUCKET_EXECUTOR_FOR_BUCKET_MOVE = defineFeatureFlag( - "use-bucket-executor-for-bucket-move", false, - List.of("baldersheim"), "2021-02-15", "2021-04-01", + "use-bucket-executor-for-bucket-move", true, + List.of("baldersheim"), "2021-02-15", "2021-05-01", "Wheter to use content-level bucket executor or legacy frozen buckets", "Takes effect on next internal redeployment", APPLICATION_ID); - public static final UnboundBooleanFlag USE_POWER_OF_TWO_CHOICES_LOAD_BALANCING = defineFeatureFlag( - "use-power-of-two-choices-load-balancing", false, - List.of("tokle"), "2020-12-02", "2021-04-01", - "Whether to use Power of two load balancing algorithm for application", - "Takes effect on next internal redeployment", - APPLICATION_ID); - public static final UnboundBooleanFlag GROUP_SUSPENSION = defineFeatureFlag( "group-suspension", true, List.of("hakon"), "2021-01-22", "2021-04-22", @@ -209,20 +189,20 @@ public class Flags { public static final UnboundBooleanFlag ENABLE_FEED_BLOCK_IN_DISTRIBUTOR = defineFeatureFlag( "enable-feed-block-in-distributor", true, - List.of("geirst"), "2021-01-27", "2021-04-01", + List.of("geirst"), "2021-01-27", "2021-05-01", "Enables blocking of feed in the distributor if resource usage is above limit on at least one content node", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); public static final UnboundDoubleFlag MAX_DEAD_BYTES_RATIO = defineDoubleFlag( "max-dead-bytes-ratio", 0.2, - List.of("baldersheim", "geirst","toregge"), "2021-02-03", "2021-04-01", + List.of("baldersheim", "geirst","toregge"), "2021-02-03", "2021-05-01", "max ratio of dead to used memory bytes in large data structures before compaction is attempted", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); public static final UnboundStringFlag SYNC_HOST_LOGS_TO_S3_BUCKET = defineStringFlag( - "sync-host-logs-to-s3-bucket", "", List.of("andreer", "valerijf"), "2021-02-10", "2021-04-01", + "sync-host-logs-to-s3-bucket", "", List.of("andreer", "valerijf"), "2021-02-10", "2021-05-01", "Host-admin should sync host logs to an S3 bucket named by this flag. If left empty, sync is disabled", "Takes effect on next run of S3 log sync task in host-admin", TENANT_ID, ZONE_ID); @@ -245,7 +225,7 @@ public class Flags { List.of("hmusum"), "2021-03-15", "2021-05-15", "JVM max heap size for config proxy in Mb", "Takes effect on restart of Docker container", - CLUSTER_TYPE); + CLUSTER_TYPE, CLUSTER_ID); public static final UnboundStringFlag DEDICATED_CLUSTER_CONTROLLER_FLAVOR = defineStringFlag( "dedicated-cluster-controller-flavor", "", List.of("jonmv"), "2021-02-25", "2021-04-25", diff --git a/flags/src/main/java/com/yahoo/vespa/flags/IntFlag.java b/flags/src/main/java/com/yahoo/vespa/flags/IntFlag.java index 4a59c34e8d0..94a6e9b6e15 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/IntFlag.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/IntFlag.java @@ -12,6 +12,11 @@ public class IntFlag extends FlagImpl<Integer, IntFlag> { super(id, defaultValue, vector, serializer, source, IntFlag::new); } + @Override + public IntFlag self() { + return this; + } + public int value() { return boxedValue(); } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/JacksonFlag.java b/flags/src/main/java/com/yahoo/vespa/flags/JacksonFlag.java index e4afbd231a1..007110f077d 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/JacksonFlag.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/JacksonFlag.java @@ -12,6 +12,11 @@ public class JacksonFlag<T> extends FlagImpl<T, JacksonFlag<T>> { super(id, defaultValue, vector, serializer, source, JacksonFlag::new); } + @Override + public JacksonFlag<T> self() { + return this; + } + public T value() { return boxedValue(); } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/ListFlag.java b/flags/src/main/java/com/yahoo/vespa/flags/ListFlag.java index 6fefc5e22e3..4c7e9683833 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/ListFlag.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/ListFlag.java @@ -13,6 +13,11 @@ public class ListFlag<T> extends FlagImpl<List<T>, ListFlag<T>> { super(id, defaultValue, vector, serializer, source, ListFlag::new); } + @Override + public ListFlag<T> self() { + return this; + } + public List<T> value() { return boxedValue(); } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/LongFlag.java b/flags/src/main/java/com/yahoo/vespa/flags/LongFlag.java index 8a1707c9e6a..1242d05c290 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/LongFlag.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/LongFlag.java @@ -12,6 +12,11 @@ public class LongFlag extends FlagImpl<Long, LongFlag> { super(id, defaultValue, vector, serializer, source, LongFlag::new); } + @Override + public LongFlag self() { + return this; + } + public long value() { return boxedValue(); } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java b/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java index 7099124024b..ad096a20786 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java @@ -11,6 +11,8 @@ import java.time.format.DateTimeFormatter; import java.util.List; import static com.yahoo.vespa.flags.FetchVector.Dimension.APPLICATION_ID; +import static com.yahoo.vespa.flags.FetchVector.Dimension.CLUSTER_ID; +import static com.yahoo.vespa.flags.FetchVector.Dimension.CLUSTER_TYPE; import static com.yahoo.vespa.flags.FetchVector.Dimension.CONSOLE_USER_EMAIL; import static com.yahoo.vespa.flags.FetchVector.Dimension.HOSTNAME; import static com.yahoo.vespa.flags.FetchVector.Dimension.NODE_TYPE; @@ -102,9 +104,9 @@ public class PermanentFlags { public static final UnboundDoubleFlag CONTAINER_CPU_CAP = defineDoubleFlag( "container-cpu-cap", 0, "Hard limit on how many CPUs a container may use. This value is multiplied by CPU allocated to node, so " + - "to cap CPU at 200%, set this to 2, etc.", + "to cap CPU at 200%, set this to 2, etc. 0 disables the cap to allow unlimited CPU.", "Takes effect on next node agent tick. Change is orchestrated, but does NOT require container restart", - HOSTNAME, APPLICATION_ID); + HOSTNAME, APPLICATION_ID, CLUSTER_ID, CLUSTER_TYPE); public static final UnboundListFlag<String> DISABLED_HOST_ADMIN_TASKS = defineListFlag( "disabled-host-admin-tasks", List.of(), String.class, @@ -143,6 +145,12 @@ public class PermanentFlags { "takes effect on JVM restart", NODE_TYPE, APPLICATION_ID); + public static final UnboundIntFlag MAX_TRIAL_TENANTS = defineIntFlag( + "max-trial-tenants", -1, + "The maximum nr. of tenants with trial plan, -1 is unlimited", + "Takes effect immediately" + ); + private PermanentFlags() {} private static UnboundBooleanFlag defineFeatureFlag( diff --git a/flags/src/main/java/com/yahoo/vespa/flags/StringFlag.java b/flags/src/main/java/com/yahoo/vespa/flags/StringFlag.java index 925af130b95..33391f43d80 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/StringFlag.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/StringFlag.java @@ -12,6 +12,11 @@ public class StringFlag extends FlagImpl<String, StringFlag> { super(id, defaultValue, vector, serializer, source, StringFlag::new); } + @Override + public StringFlag self() { + return this; + } + public String value() { return boxedValue(); } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/json/DimensionHelper.java b/flags/src/main/java/com/yahoo/vespa/flags/json/DimensionHelper.java index f109d3c950b..b18c0c43bbc 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/json/DimensionHelper.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/json/DimensionHelper.java @@ -17,6 +17,7 @@ public class DimensionHelper { serializedDimensions.put(FetchVector.Dimension.HOSTNAME, "hostname"); serializedDimensions.put(FetchVector.Dimension.APPLICATION_ID, "application"); serializedDimensions.put(FetchVector.Dimension.NODE_TYPE, "node-type"); + serializedDimensions.put(FetchVector.Dimension.CLUSTER_ID, "cluster-id"); serializedDimensions.put(FetchVector.Dimension.CLUSTER_TYPE, "cluster-type"); serializedDimensions.put(FetchVector.Dimension.VESPA_VERSION, "vespa-version"); serializedDimensions.put(FetchVector.Dimension.CONSOLE_USER_EMAIL, "console-user-email"); diff --git a/jdisc_messagebus_service/.gitignore b/jdisc_messagebus_service/.gitignore deleted file mode 100644 index 12251442258..00000000000 --- a/jdisc_messagebus_service/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -/target -/pom.xml.build diff --git a/jdisc_messagebus_service/OWNERS b/jdisc_messagebus_service/OWNERS deleted file mode 100644 index 78b92e411b4..00000000000 --- a/jdisc_messagebus_service/OWNERS +++ /dev/null @@ -1,2 +0,0 @@ -gjoranv -bjorncs diff --git a/jdisc_messagebus_service/README.md b/jdisc_messagebus_service/README.md deleted file mode 100644 index cecb21de952..00000000000 --- a/jdisc_messagebus_service/README.md +++ /dev/null @@ -1,4 +0,0 @@ -<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -# JDisc messagebus service - -Messagebus protocol implementation for JDisc. diff --git a/jdisc_messagebus_service/pom.xml b/jdisc_messagebus_service/pom.xml deleted file mode 100644 index 55f8392a5df..00000000000 --- a/jdisc_messagebus_service/pom.xml +++ /dev/null @@ -1,54 +0,0 @@ -<?xml version="1.0"?> -<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<project xmlns="http://maven.apache.org/POM/4.0.0" - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 - http://maven.apache.org/xsd/maven-4.0.0.xsd"> - <modelVersion>4.0.0</modelVersion> - <parent> - <groupId>com.yahoo.vespa</groupId> - <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> - </parent> - <artifactId>jdisc_messagebus_service</artifactId> - <version>7-SNAPSHOT</version> - <packaging>container-plugin</packaging> - <dependencies> - <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>jdisc_core</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>component</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>messagebus</artifactId> - <version>${project.version}</version> - </dependency> - </dependencies> - <build> - <plugins> - <plugin> - <groupId>com.yahoo.vespa</groupId> - <artifactId>bundle-plugin</artifactId> - <extensions>true</extensions> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-compiler-plugin</artifactId> - </plugin> - </plugins> - </build> -</project> diff --git a/jrt/src/com/yahoo/jrt/Worker.java b/jrt/src/com/yahoo/jrt/Worker.java index d20ebadf856..5f0407e4f5a 100644 --- a/jrt/src/com/yahoo/jrt/Worker.java +++ b/jrt/src/com/yahoo/jrt/Worker.java @@ -4,7 +4,7 @@ package com.yahoo.jrt; class Worker { - private static int WORK_LIMIT = 1024; + private static final int WORK_LIMIT = 1024; private class Run implements Runnable { public void run() { @@ -27,7 +27,7 @@ class Worker { } private static class DoHandshakeWork implements Runnable { - private Connection connection; + private final Connection connection; DoHandshakeWork(Connection c) { connection = c; } diff --git a/juniper/src/vespa/juniper/query.h b/juniper/src/vespa/juniper/query.h index 63a6a9e1a4a..2d8c6b4bbe0 100644 --- a/juniper/src/vespa/juniper/query.h +++ b/juniper/src/vespa/juniper/query.h @@ -155,17 +155,6 @@ public: */ virtual bool VisitANDNOT(const QueryItem* item, int arity) = 0; - /** To be called upon by IQuery::Traverse visiting a THRESHOLD query item - * @param item The (opaque to IQueryVisitor) item that is visited - * @param arity The number of children of this item - * @param threshold The threshold value for which the sum of the individual - * subexpressions' weights (as obtained by the IQueryVisitor::Weight function - * should result in a hit for the THRESHOLD expression. - * @return if false, caller should skip calling this element's children visitors, - * otherwise caller should proceed as normal - */ - virtual bool VisitTHRESHOLD(const QueryItem* item, int arity, int threshold) = 0; - /** To be called upon by IQuery::Traverse visiting any other query item * than the ones handled by Juniper (to avoid inconsistency in the * traversal wrt. arities) diff --git a/juniper/src/vespa/juniper/queryvisitor.cpp b/juniper/src/vespa/juniper/queryvisitor.cpp index f8f8d2d3da7..96a3f4ce771 100644 --- a/juniper/src/vespa/juniper/queryvisitor.cpp +++ b/juniper/src/vespa/juniper/queryvisitor.cpp @@ -189,21 +189,11 @@ bool QueryVisitor::VisitANDNOT(const QueryItem*, int arity) return true; } -bool QueryVisitor::VisitTHRESHOLD(const QueryItem*, int arity, int threshold) -{ - LOG(debug, "juniper: VisitTHRESHOLD[%d,%d]", arity, threshold); - QueryNode* node = new QueryNode(arity, threshold); - node->_options = _qhandle->_options; - - insert(node); - return true; -} - bool QueryVisitor::VisitOther(const QueryItem*, int arity) { LOG(debug, "juniper: VisitOther[%d]", arity); - insert(NULL); + insert(nullptr); return false; } diff --git a/juniper/src/vespa/juniper/queryvisitor.h b/juniper/src/vespa/juniper/queryvisitor.h index 2557e91021c..abdd3d2b28c 100644 --- a/juniper/src/vespa/juniper/queryvisitor.h +++ b/juniper/src/vespa/juniper/queryvisitor.h @@ -22,6 +22,8 @@ namespace juniper { class QueryVisitor : public juniper::IQueryVisitor { public: + QueryVisitor(QueryVisitor &) = delete; + QueryVisitor &operator=(QueryVisitor &) = delete; QueryVisitor(const IQuery& query, QueryHandle* qhandle, juniper::QueryModifier & queryModifier); ~QueryVisitor(); @@ -33,7 +35,6 @@ public: bool VisitRANK(const QueryItem* item, int arity) override; bool VisitPHRASE(const QueryItem* item, int arity) override; bool VisitANDNOT(const QueryItem* item, int arity) override; - bool VisitTHRESHOLD(const QueryItem* item, int arity, int threshold) override; bool VisitOther(const QueryItem* item, int arity) override; void VisitKeyword(const QueryItem* item, const char* keyword, const size_t length = 0, bool prefix = false, bool specialToken = false) override; @@ -49,7 +50,6 @@ protected: std::string get_index(const QueryItem* item); private: // Helper functions/members for use during construction only. - void update_parameters(const char* options); void insert(QueryExpr* expr); void postprocess_query(); juniper::QueryModifier & _queryModifier; @@ -62,9 +62,6 @@ private: QueryHandle* _qhandle; int _term_index; bool _got_stack; // Set when we have created a stack root - - QueryVisitor(QueryVisitor &); - QueryVisitor &operator=(QueryVisitor &); }; diff --git a/logserver/bin/logserver-start.sh b/logserver/bin/logserver-start.sh index fe4b5ebbb64..6af8c076246 100755 --- a/logserver/bin/logserver-start.sh +++ b/logserver/bin/logserver-start.sh @@ -78,7 +78,7 @@ ROOT=${VESPA_HOME%/} export ROOT cd $ROOT || { echo "Cannot cd to $ROOT" 1>&2; exit 1; } -addopts="-server -Xms32m -Xmx256m -XX:CompressedClassSpaceSize=32m -XX:MaxDirectMemorySize=32m -XX:ThreadStackSize=256 -XX:MaxJavaStackTraceDepth=1000 -XX:ActiveProcessorCount=2 -Djava.io.tmpdir=${VESPA_HOME}/tmp" +addopts="-server -Xms32m -Xmx256m -XX:CompressedClassSpaceSize=32m -XX:MaxDirectMemorySize=32m -XX:ThreadStackSize=256 -XX:MaxJavaStackTraceDepth=1000 -XX:ActiveProcessorCount=2 -XX:-OmitStackTraceInFastThrow -Djava.io.tmpdir=${VESPA_HOME}/tmp" oomopt="-XX:+ExitOnOutOfMemoryError" diff --git a/messagebus-disc/.gitignore b/messagebus-disc/.gitignore deleted file mode 100644 index 3cc25b51fc4..00000000000 --- a/messagebus-disc/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -/pom.xml.build -/target diff --git a/messagebus-disc/OWNERS b/messagebus-disc/OWNERS deleted file mode 100644 index 12b533ec610..00000000000 --- a/messagebus-disc/OWNERS +++ /dev/null @@ -1 +0,0 @@ -havardpe diff --git a/messagebus-disc/README b/messagebus-disc/README deleted file mode 100644 index 256990ecfb1..00000000000 --- a/messagebus-disc/README +++ /dev/null @@ -1,2 +0,0 @@ -This module provides jDISC ClientProvider and ServerProvider implementations for -Vespa Message Bus. diff --git a/messagebus-disc/pom.xml b/messagebus-disc/pom.xml deleted file mode 100644 index 5df6cc0a03f..00000000000 --- a/messagebus-disc/pom.xml +++ /dev/null @@ -1,117 +0,0 @@ -<?xml version="1.0"?> -<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<project xmlns="http://maven.apache.org/POM/4.0.0" - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 - http://maven.apache.org/xsd/maven-4.0.0.xsd"> - <modelVersion>4.0.0</modelVersion> - <parent> - <groupId>com.yahoo.vespa</groupId> - <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> - </parent> - <artifactId>messagebus-disc</artifactId> - <version>7-SNAPSHOT</version> - <packaging>container-plugin</packaging> - <name>${project.artifactId}</name> - <dependencies> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>jdisc_messagebus_service</artifactId> - <version>${project.version}</version> - </dependency> - </dependencies> - <build> - <plugins> - <plugin> - <groupId>com.yahoo.vespa</groupId> - <artifactId>bundle-plugin</artifactId> - <extensions>true</extensions> - </plugin> - <plugin> - <!-- Only added to make IntelliJ use correct language level --> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-compiler-plugin</artifactId> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-javadoc-plugin</artifactId> - <executions> - <execution> - <id>attach-javadocs</id> - <goals> - <goal>jar</goal> - </goals> - <configuration> - <finalName>${project.artifactId}</finalName> - <excludePackageNames>com.yahoo.jdisc.core</excludePackageNames> - <doclint>${doclint},-missing</doclint> - </configuration> - </execution> - </executions> - </plugin> - <plugin> - <artifactId>maven-source-plugin</artifactId> - <executions> - <execution> - <id>attach-sources</id> - <goals> - <goal>jar-no-fork</goal> - </goals> - <configuration> - <finalName>${project.artifactId}</finalName> - </configuration> - </execution> - </executions> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-shade-plugin</artifactId> - <configuration> - <finalName>${project.artifactId}-jar-with-dependencies</finalName> - <filters> - <filter> - <!-- Don't include signature files from bouncycastle in uber jar. --> - <artifact>*:*</artifact> - <excludes> - <exclude>META-INF/*.SF</exclude> - <exclude>META-INF/*.DSA</exclude> - <exclude>META-INF/*.RSA</exclude> - </excludes> - </filter> - </filters> - <transformers> - <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"> - <manifestEntries> - <Bundle-SymbolicName>${project.groupId}.${project.artifactId}</Bundle-SymbolicName> - <Export-Package>com.yahoo.messagebus,com.yahoo.messagebus.jdisc,com.yahoo.messagebus.network, - com.yahoo.messagebus.network.rpc,com.yahoo.messagebus.routing,com.yahoo.messagebus.shared,com.yahoo.messagebus.test</Export-Package> - <Import-Package> - com.google.inject;version="1.3", - com.yahoo.jdisc, - com.yahoo.jdisc.application, - com.yahoo.jdisc.handler, - com.yahoo.jdisc.service, - javax.xml.parsers, - org.w3c.dom, - org.xml.sax, - org.xml.sax.ext, - org.xml.sax.helpers - </Import-Package> - </manifestEntries> - </transformer> - </transformers> - </configuration> - <executions> - <execution> - <phase>package</phase> - <goals> - <goal>shade</goal> - </goals> - </execution> - </executions> - </plugin> - </plugins> - </build> -</project> diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsRetriever.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsRetriever.java index bef4864fb6d..619c62fae43 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsRetriever.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsRetriever.java @@ -3,14 +3,15 @@ package ai.vespa.metricsproxy.http.application; import ai.vespa.metricsproxy.metric.model.ConsumerId; import ai.vespa.metricsproxy.metric.model.MetricsPacket; -import ai.vespa.util.http.hc4.VespaHttpClientBuilder; +import ai.vespa.util.http.hc5.VespaHttpClientBuilder; import com.google.inject.Inject; import com.yahoo.component.AbstractComponent; +import org.apache.hc.client5.http.classic.HttpClient; +import org.apache.hc.client5.http.config.RequestConfig; +import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; +import org.apache.hc.core5.util.Timeout; + import java.util.logging.Level; -import org.apache.http.client.HttpClient; -import org.apache.http.client.config.RequestConfig; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import java.time.Clock; import java.time.Duration; @@ -103,11 +104,11 @@ public class ApplicationMetricsRetriever extends AbstractComponent { } private static CloseableHttpClient createHttpClient() { - return VespaHttpClientBuilder.create(PoolingHttpClientConnectionManager::new) + return VespaHttpClientBuilder.create() .setUserAgent("application-metrics-retriever") .setDefaultRequestConfig(RequestConfig.custom() - .setConnectTimeout(HTTP_CONNECT_TIMEOUT) - .setSocketTimeout(HTTP_SOCKET_TIMEOUT) + .setConnectTimeout(Timeout.ofMilliseconds(HTTP_CONNECT_TIMEOUT)) + .setResponseTimeout(Timeout.ofMilliseconds(HTTP_SOCKET_TIMEOUT)) .build()) .build(); } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/NodeMetricsClient.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/NodeMetricsClient.java index 78bc3e96322..2e17443e821 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/NodeMetricsClient.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/NodeMetricsClient.java @@ -6,9 +6,9 @@ import ai.vespa.metricsproxy.metric.model.MetricsPacket; import ai.vespa.metricsproxy.metric.model.json.GenericJsonUtil; import ai.vespa.metricsproxy.metric.model.processing.MetricsProcessor; import com.yahoo.yolean.Exceptions; -import org.apache.http.client.HttpClient; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.impl.client.BasicResponseHandler; +import org.apache.hc.client5.http.classic.HttpClient; +import org.apache.hc.client5.http.classic.methods.HttpGet; +import org.apache.hc.client5.http.impl.classic.BasicHttpClientResponseHandler; import java.io.IOException; import java.time.Clock; @@ -70,7 +70,7 @@ public class NodeMetricsClient { log.log(FINE, () -> "Retrieving metrics from host " + metricsUri); try { - String metricsJson = httpClient.execute(new HttpGet(metricsUri), new BasicResponseHandler()); + String metricsJson = httpClient.execute(new HttpGet(metricsUri), new BasicHttpClientResponseHandler()); var metricsBuilders = GenericJsonUtil.toMetricsPackets(metricsJson); var metrics = processAndBuild(metricsBuilders, new ServiceIdDimensionProcessor(), diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/dimensions/PublicDimensions.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/dimensions/PublicDimensions.java index f6bda2f9ef5..84c95f8e39e 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/dimensions/PublicDimensions.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/dimensions/PublicDimensions.java @@ -25,6 +25,10 @@ public final class PublicDimensions { public static final String INTERNAL_CLUSTER_ID = "clusterid"; public static final String CLUSTER_ID = "clusterId"; + // This dimension is not currently (March 2021) added to the 'commonDimensions' allow-list below, due to the + // limit of 10 total dimensions in public http apis. See e.g. MetricsV2Handler#MAX_DIMENSIONS. + public static final String GROUP_ID = "groupId"; + // Internal name (instance) is confusing, so renamed to 'serviceId' for public use. // This is added by the metrics-proxy. public static final String INTERNAL_SERVICE_ID = "instance"; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java index 05a2b85af68..bfbadc00628 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java @@ -1,13 +1,15 @@ // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.service; -import ai.vespa.util.http.hc4.VespaHttpClientBuilder; +import ai.vespa.util.http.hc5.VespaHttpClientBuilder; + import java.util.logging.Level; import com.yahoo.yolean.Exceptions; -import org.apache.http.client.config.RequestConfig; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.impl.client.BasicResponseHandler; -import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.hc.client5.http.classic.methods.HttpGet; +import org.apache.hc.client5.http.config.RequestConfig; +import org.apache.hc.client5.http.impl.classic.BasicHttpClientResponseHandler; +import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; +import org.apache.hc.core5.util.Timeout; import java.io.IOException; import java.net.URI; @@ -45,7 +47,7 @@ public abstract class HttpMetricFetcher { String getJson() throws IOException { log.log(Level.FINE, "Connecting to url " + url + " for service '" + service + "'"); - return httpClient.execute(new HttpGet(url), new BasicResponseHandler()); + return httpClient.execute(new HttpGet(url), new BasicHttpClientResponseHandler()); } public String toString() { @@ -82,8 +84,8 @@ public abstract class HttpMetricFetcher { return VespaHttpClientBuilder.create() .setUserAgent("metrics-proxy-http-client") .setDefaultRequestConfig(RequestConfig.custom() - .setConnectTimeout(CONNECTION_TIMEOUT) - .setSocketTimeout(SOCKET_TIMEOUT) + .setConnectTimeout(Timeout.ofMilliseconds(CONNECTION_TIMEOUT)) + .setResponseTimeout(Timeout.ofMilliseconds(SOCKET_TIMEOUT)) .build()) .build(); } diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/NodeMetricsClientTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/NodeMetricsClientTest.java index eba32941620..ab84a4edcde 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/NodeMetricsClientTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/NodeMetricsClientTest.java @@ -5,8 +5,9 @@ import ai.vespa.metricsproxy.http.metrics.MetricsV1Handler; import ai.vespa.metricsproxy.metric.model.MetricsPacket; import com.github.tomakehurst.wiremock.junit.WireMockClassRule; import com.yahoo.test.ManualClock; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; + +import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; +import org.apache.hc.client5.http.impl.classic.HttpClients; import org.junit.Before; import org.junit.BeforeClass; import org.junit.ClassRule; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java index f401b614637..e335e66c1c1 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.nodeagent; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.NodeType; @@ -381,11 +382,12 @@ public class NodeAgentImpl implements NodeAgent { private ContainerResources getContainerResources(NodeAgentContext context) { double cpuCap = noCpuCap(context.zone()) ? 0 : - context.node().owner() - .map(appId -> containerCpuCap.with(FetchVector.Dimension.APPLICATION_ID, appId.serializedForm())) - .orElse(containerCpuCap) + context.vcpuOnThisHost() * containerCpuCap + .with(FetchVector.Dimension.APPLICATION_ID, context.node().owner().map(ApplicationId::serializedForm)) + .with(FetchVector.Dimension.CLUSTER_ID, context.node().membership().map(NodeMembership::clusterId)) + .with(FetchVector.Dimension.CLUSTER_TYPE, context.node().membership().map(membership -> membership.type().value())) .with(FetchVector.Dimension.HOSTNAME, context.node().hostname()) - .value() * context.vcpuOnThisHost(); + .value(); return ContainerResources.from(cpuCap, context.vcpuOnThisHost(), context.node().memoryGb()); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java index 5135ae73ea1..1f4bd55525d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java @@ -269,8 +269,10 @@ public final class Node implements Nodelike { } /** Returns a node with the flavor assigned to the given value */ - public Node with(Flavor flavor) { - return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, + public Node with(Flavor flavor, Agent agent, Instant instant) { + if (flavor.equals(this.flavor)) return this; + History updateHistory = history.with(new History.Event(History.Event.Type.resized, agent, instant)); + return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, allocation, updateHistory, type, reports, modelName, reservedTo, exclusiveTo, switchHostname); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/History.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/History.java index 755cd72be93..0c5a8ea1d9f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/History.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/History.java @@ -158,6 +158,8 @@ public class History { down, // The node made a config request, indicating it is live requested, + // The node resources/flavor were changed + resized(false), // The node was rebooted rebooted(false), // The node upgraded its OS (implies a reboot) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java index f08d7aa07e1..9d36be67431 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java @@ -420,6 +420,7 @@ public class NodeSerializer { case "deallocated" : return History.Event.Type.deallocated; case "down" : return History.Event.Type.down; case "requested" : return History.Event.Type.requested; + case "resized" : return History.Event.Type.resized; case "rebooted" : return History.Event.Type.rebooted; case "osUpgraded" : return History.Event.Type.osUpgraded; case "firmwareVerified" : return History.Event.Type.firmwareVerified; @@ -444,6 +445,7 @@ public class NodeSerializer { case deallocated : return "deallocated"; case down : return "down"; case requested: return "requested"; + case resized: return "resized"; case rebooted: return "rebooted"; case osUpgraded: return "osUpgraded"; case firmwareVerified: return "firmwareVerified"; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java index 49eb44a4ec0..03db7e0e7e5 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java @@ -14,6 +14,7 @@ import com.yahoo.vespa.hosted.provision.NodeMutex; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.applications.Application; import com.yahoo.vespa.hosted.provision.applications.ScalingEvent; +import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.Allocation; import java.time.Instant; @@ -216,7 +217,7 @@ class Activator { HostSpec hostSpec = getHost(node.hostname(), hosts); node = hostSpec.membership().get().retired() ? node.retire(at) : node.unretire(); if (! hostSpec.advertisedResources().equals(node.resources())) // A resized node - node = node.with(new Flavor(hostSpec.advertisedResources())); + node = node.with(new Flavor(hostSpec.advertisedResources()), Agent.application, at); Allocation allocation = node.allocation().get() .with(hostSpec.membership().get()) .withRequestedResources(hostSpec.requestedResources() diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java index e014b50a25f..3d3a54774e4 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java @@ -272,7 +272,8 @@ class NodeAllocation { NodeResources hostResources = allNodes.parentOf(node).get().flavor().resources(); return node.with(new Flavor(requestedNodes.resources().get() .with(hostResources.diskSpeed()) - .with(hostResources.storageType()))); + .with(hostResources.storageType())), + Agent.application, nodeRepository.clock().instant()); } private Node setCluster(ClusterSpec cluster, Node node) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java index ea951798da0..e8fa4f04eed 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java @@ -132,7 +132,7 @@ public class NodePatcher implements AutoCloseable { case "failCount" : return node.with(node.status().withFailCount(asLong(value).intValue())); case "flavor" : - return node.with(nodeFlavors.getFlavorOrThrow(asString(value))); + return node.with(nodeFlavors.getFlavorOrThrow(asString(value)), Agent.operator, clock.instant()); case "parentHostname" : return node.withParentHostname(asString(value)); case "ipAddresses" : @@ -152,19 +152,19 @@ public class NodePatcher implements AutoCloseable { return node.withOpenStackId(asString(value)); case "diskGb": case "minDiskAvailableGb": - return node.with(node.flavor().with(node.flavor().resources().withDiskGb(value.asDouble()))); + return node.with(node.flavor().with(node.flavor().resources().withDiskGb(value.asDouble())), Agent.operator, clock.instant()); case "memoryGb": case "minMainMemoryAvailableGb": - return node.with(node.flavor().with(node.flavor().resources().withMemoryGb(value.asDouble()))); + return node.with(node.flavor().with(node.flavor().resources().withMemoryGb(value.asDouble())), Agent.operator, clock.instant()); case "vcpu": case "minCpuCores": - return node.with(node.flavor().with(node.flavor().resources().withVcpu(value.asDouble()))); + return node.with(node.flavor().with(node.flavor().resources().withVcpu(value.asDouble())), Agent.operator, clock.instant()); case "fastDisk": - return node.with(node.flavor().with(node.flavor().resources().with(value.asBool() ? fast : slow))); + return node.with(node.flavor().with(node.flavor().resources().with(value.asBool() ? fast : slow)), Agent.operator, clock.instant()); case "remoteStorage": - return node.with(node.flavor().with(node.flavor().resources().with(value.asBool() ? remote : local))); + return node.with(node.flavor().with(node.flavor().resources().with(value.asBool() ? remote : local)), Agent.operator, clock.instant()); case "bandwidthGbps": - return node.with(node.flavor().with(node.flavor().resources().withBandwidthGbps(value.asDouble()))); + return node.with(node.flavor().with(node.flavor().resources().withBandwidthGbps(value.asDouble())), Agent.operator, clock.instant()); case "modelName": return value.type() == Type.NIX ? node.withoutModelName() : node.withModelName(asString(value)); case "requiredDiskSpeed": diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java index 0a852bbab56..881646fa546 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java @@ -85,7 +85,7 @@ public class NodeSerializerTest { assertEquals(1, node.history().events().size()); node = node.withRestart(new Generation(1, 2)); node = node.withReboot(new Generation(3, 4)); - node = node.with(FlavorConfigBuilder.createDummies("large").getFlavorOrThrow("large")); + node = node.with(FlavorConfigBuilder.createDummies("large").getFlavorOrThrow("large"), Agent.system, clock.instant()); node = node.with(node.status().withVespaVersion(Version.fromString("1.2.3"))); node = node.with(node.status().withIncreasedFailCount().withIncreasedFailCount()); node = node.with(NodeType.tenant); @@ -105,7 +105,7 @@ public class NodeSerializerTest { assertEquals(node.allocation().get().membership(), copy.allocation().get().membership()); assertEquals(node.allocation().get().requestedResources(), copy.allocation().get().requestedResources()); assertEquals(node.allocation().get().isRemovable(), copy.allocation().get().isRemovable()); - assertEquals(1, copy.history().events().size()); + assertEquals(2, copy.history().events().size()); assertEquals(clock.instant().truncatedTo(MILLIS), copy.history().event(History.Event.Type.reserved).get().at()); assertEquals(NodeType.tenant, copy.type()); } @@ -283,12 +283,13 @@ public class NodeSerializerTest { public void flavor_overrides_serialization() { Node node = createNode(); assertEquals(20, node.flavor().resources().diskGb(), 0); - node = node.with(node.flavor().with(FlavorOverrides.ofDisk(1234))); + node = node.with(node.flavor().with(FlavorOverrides.ofDisk(1234)), Agent.system, clock.instant()); assertEquals(1234, node.flavor().resources().diskGb(), 0); Node copy = nodeSerializer.fromJson(Node.State.provisioned, nodeSerializer.toJson(node)); assertEquals(1234, copy.flavor().resources().diskGb(), 0); assertEquals(node, copy); + assertTrue(node.history().event(History.Event.Type.resized).isPresent()); } @Test diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java index 0db5453c963..889fd537740 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java @@ -760,7 +760,8 @@ public class ProvisioningTest { // Pick out a random application node and make it's parent larger, this will make it the spare host NodeList nodes = tester.nodeRepository().nodes().list(); Node randomNode = nodes.owner(application).shuffle(new Random()).first().get(); - tester.nodeRepository().nodes().write(nodes.parentOf(randomNode).get().with(new Flavor(new NodeResources(2, 10, 20, 8))), () -> {}); + tester.nodeRepository().nodes().write(nodes.parentOf(randomNode).get() + .with(new Flavor(new NodeResources(2, 10, 20, 8)), Agent.system, tester.nodeRepository().clock().instant()), () -> {}); // Re-deploy application with 1 node less, the retired node should be on the spare host tester.deploy(application, spec, Capacity.from(new ClusterResources(5, 1, defaultResources))); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4-after-changes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4-after-changes.json index 77e1fb1e16d..a2f43480605 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4-after-changes.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4-after-changes.json @@ -70,6 +70,11 @@ "agent": "system" }, { + "event": "resized", + "at": 123, + "agent": "operator" + }, + { "event": "wantToRetire", "at": 123, "agent": "operator" diff --git a/orchestrator/pom.xml b/orchestrator/pom.xml index fed0e617c79..da4a7dc4efc 100644 --- a/orchestrator/pom.xml +++ b/orchestrator/pom.xml @@ -134,6 +134,16 @@ <artifactId>hamcrest-all</artifactId> <scope>test</scope> </dependency> + <dependency> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.junit.vintage</groupId> + <artifactId>junit-vintage-engine</artifactId> + <scope>test</scope> + </dependency> </dependencies> <build> <plugins> diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostSuspensionResource.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostSuspensionHandler.java index 3d60d40dfcf..ca720ec8b68 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostSuspensionResource.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostSuspensionHandler.java @@ -1,72 +1,81 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.orchestrator.resources; import com.google.common.util.concurrent.UncheckedTimeoutException; -import com.yahoo.container.jaxrs.annotation.Component; -import java.util.logging.Level; +import com.google.inject.Inject; +import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.jdisc.Response; +import com.yahoo.restapi.JacksonJsonResponse; +import com.yahoo.restapi.RestApi; +import com.yahoo.restapi.RestApiException; +import com.yahoo.restapi.RestApiRequestHandler; import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.orchestrator.BatchHostNameNotFoundException; import com.yahoo.vespa.orchestrator.BatchInternalErrorException; import com.yahoo.vespa.orchestrator.Orchestrator; import com.yahoo.vespa.orchestrator.policy.BatchHostStateChangeDeniedException; -import com.yahoo.vespa.orchestrator.restapi.HostSuspensionApi; import com.yahoo.vespa.orchestrator.restapi.wire.BatchOperationResult; -import javax.inject.Inject; -import javax.ws.rs.Path; -import javax.ws.rs.WebApplicationException; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; import java.util.List; +import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; /** * @author hakonhall + * @author bjorncs */ -@Path(HostSuspensionApi.PATH_PREFIX) -public class HostSuspensionResource implements HostSuspensionApi { +public class HostSuspensionHandler extends RestApiRequestHandler<HostSuspensionHandler> { - private static final Logger log = Logger.getLogger(HostSuspensionResource.class.getName()); + private static final Logger log = Logger.getLogger(HostSuspensionHandler.class.getName()); private final Orchestrator orchestrator; @Inject - public HostSuspensionResource(@Component Orchestrator orchestrator) { + public HostSuspensionHandler(LoggingRequestHandler.Context context, Orchestrator orchestrator) { + super(context, HostSuspensionHandler::createRestApiDefinition); this.orchestrator = orchestrator; } - @Override - public BatchOperationResult suspendAll(String parentHostnameString, List<String> hostnamesAsStrings) { + private static RestApi createRestApiDefinition(HostSuspensionHandler self) { + return RestApi.builder() + .addRoute(RestApi.route("/orchestrator/v1/suspensions/hosts/{hostname}") + .put(self::suspendAll)) + .registerJacksonResponseEntity(BatchOperationResult.class) + .build(); + } + + private BatchOperationResult suspendAll(RestApi.RequestContext context) { + String parentHostnameString = context.pathParameters().getStringOrThrow("hostname"); + List<String> hostnamesAsStrings = context.queryParameters().getStringList("hostname"); + HostName parentHostname = new HostName(parentHostnameString); List<HostName> hostnames = hostnamesAsStrings.stream().map(HostName::new).collect(Collectors.toList()); try { orchestrator.suspendAll(parentHostname, hostnames); } catch (BatchHostStateChangeDeniedException e) { log.log(Level.FINE, "Failed to suspend nodes " + hostnames + " with parent host " + parentHostname, e); - throw createWebApplicationException(e.getMessage(), Response.Status.CONFLICT); + throw createRestApiException(e.getMessage(), Response.Status.CONFLICT, e); } catch (UncheckedTimeoutException e) { log.log(Level.FINE, "Failed to suspend nodes " + hostnames + " with parent host " + parentHostname, e); - throw createWebApplicationException(e.getMessage(), Response.Status.CONFLICT); + throw createRestApiException(e.getMessage(), Response.Status.CONFLICT, e); } catch (BatchHostNameNotFoundException e) { log.log(Level.FINE, "Failed to suspend nodes " + hostnames + " with parent host " + parentHostname, e); // Note that we're returning BAD_REQUEST instead of NOT_FOUND because the resource identified // by the URL path was found. It's one of the hostnames in the request it failed to find. - throw createWebApplicationException(e.getMessage(), Response.Status.BAD_REQUEST); + throw createRestApiException(e.getMessage(), Response.Status.BAD_REQUEST, e); } catch (BatchInternalErrorException e) { log.log(Level.FINE, "Failed to suspend nodes " + hostnames + " with parent host " + parentHostname, e); - throw createWebApplicationException(e.getMessage(), Response.Status.INTERNAL_SERVER_ERROR); + throw createRestApiException(e.getMessage(), Response.Status.INTERNAL_SERVER_ERROR, e); } log.log(Level.FINE, "Suspended " + hostnames + " with parent " + parentHostname); return BatchOperationResult.successResult(); } - private WebApplicationException createWebApplicationException(String errorMessage, Response.Status status) { - return new WebApplicationException( - Response.status(status) - .entity(new BatchOperationResult(errorMessage)) - .type(MediaType.APPLICATION_JSON_TYPE) - .build()); + private RestApiException createRestApiException(String errorMessage, int statusCode, Throwable cause) { + return new RestApiException( + new JacksonJsonResponse<>(statusCode, new BatchOperationResult(errorMessage), true), + errorMessage, + cause); } - } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/ApplicationSuspensionResource.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/appsuspension/ApplicationSuspensionResource.java index 6a118f9d606..361b1f5e361 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/ApplicationSuspensionResource.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/appsuspension/ApplicationSuspensionResource.java @@ -1,9 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.orchestrator.resources; +package com.yahoo.vespa.orchestrator.resources.appsuspension; import com.yahoo.config.provision.ApplicationId; import com.yahoo.container.jaxrs.annotation.Component; -import java.util.logging.Level; import com.yahoo.vespa.orchestrator.ApplicationIdNotFoundException; import com.yahoo.vespa.orchestrator.ApplicationStateChangeDeniedException; import com.yahoo.vespa.orchestrator.OrchestratorImpl; @@ -18,13 +17,14 @@ import javax.ws.rs.Path; import javax.ws.rs.WebApplicationException; import javax.ws.rs.core.Response; import java.util.Set; +import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; /** * @author smorgrav */ -@Path(ApplicationSuspensionApi.PATH_PREFIX) +@Path("") public class ApplicationSuspensionResource implements ApplicationSuspensionApi { private static final Logger log = Logger.getLogger(ApplicationSuspensionResource.class.getName()); diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HealthResource.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/health/HealthResource.java index 47f2f063540..3265ac75642 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HealthResource.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/health/HealthResource.java @@ -1,10 +1,12 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.orchestrator.resources; +package com.yahoo.vespa.orchestrator.resources.health; import com.google.inject.Inject; import com.yahoo.config.provision.ApplicationId; import com.yahoo.container.jaxrs.annotation.Component; import com.yahoo.vespa.applicationmodel.ServiceStatusInfo; +import com.yahoo.vespa.orchestrator.resources.ApplicationServices; +import com.yahoo.vespa.orchestrator.resources.ServiceResource; import com.yahoo.vespa.orchestrator.restapi.wire.ApplicationReferenceList; import com.yahoo.vespa.orchestrator.restapi.wire.UrlReference; import com.yahoo.vespa.service.manager.HealthMonitorApi; @@ -26,7 +28,7 @@ import java.util.stream.Collectors; /** * @author hakonhall */ -@Path("/v1/health") +@Path("") public class HealthResource { private final UriInfo uriInfo; private final HealthMonitorApi healthMonitorApi; diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostResource.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/host/HostResource.java index 9ed9e1b9063..c55eeeef069 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostResource.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/host/HostResource.java @@ -1,9 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.orchestrator.resources; +package com.yahoo.vespa.orchestrator.resources.host; import com.google.common.util.concurrent.UncheckedTimeoutException; import com.yahoo.container.jaxrs.annotation.Component; -import java.util.logging.Level; import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.orchestrator.Host; import com.yahoo.vespa.orchestrator.HostNameNotFoundException; @@ -11,6 +10,7 @@ import com.yahoo.vespa.orchestrator.OrchestrationException; import com.yahoo.vespa.orchestrator.Orchestrator; import com.yahoo.vespa.orchestrator.policy.HostStateChangeDeniedException; import com.yahoo.vespa.orchestrator.policy.HostedVespaPolicy; +import com.yahoo.vespa.orchestrator.resources.instance.InstanceResource; import com.yahoo.vespa.orchestrator.restapi.HostApi; import com.yahoo.vespa.orchestrator.restapi.wire.GetHostResponse; import com.yahoo.vespa.orchestrator.restapi.wire.HostService; @@ -33,13 +33,14 @@ import javax.ws.rs.core.UriInfo; import java.net.URI; import java.time.Instant; import java.util.List; +import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; /** * @author oyving */ -@Path(HostApi.PATH_PREFIX) +@Path("") public class HostResource implements HostApi { private static final Logger log = Logger.getLogger(HostResource.class.getName()); diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceResource.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/instance/InstanceResource.java index 1cf2a2a4965..742f7d6bbd7 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceResource.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/instance/InstanceResource.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.orchestrator.resources; +package com.yahoo.vespa.orchestrator.resources.instance; import com.yahoo.config.provision.ApplicationId; import com.yahoo.container.jaxrs.annotation.Component; @@ -12,6 +12,7 @@ import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.applicationmodel.ServiceStatusInfo; import com.yahoo.vespa.applicationmodel.ServiceType; import com.yahoo.vespa.orchestrator.OrchestratorUtil; +import com.yahoo.vespa.orchestrator.resources.InstanceStatusResponse; import com.yahoo.vespa.orchestrator.restapi.wire.SlobrokEntryResponse; import com.yahoo.vespa.orchestrator.restapi.wire.WireHostInfo; import com.yahoo.vespa.orchestrator.status.HostInfo; @@ -46,7 +47,7 @@ import static com.yahoo.vespa.orchestrator.OrchestratorUtil.parseApplicationInst * @author andreer * @author bakksjo */ -@Path("/v1/instances") +@Path("") public class InstanceResource { public static final String DEFAULT_SLOBROK_PATTERN = "**"; diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostSuspensionHandlerTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostSuspensionHandlerTest.java new file mode 100644 index 00000000000..9d413526037 --- /dev/null +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostSuspensionHandlerTest.java @@ -0,0 +1,113 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.orchestrator.resources; + +import com.google.common.util.concurrent.UncheckedTimeoutException; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.jdisc.test.MockMetric; +import com.yahoo.test.json.JsonTestHelper; +import com.yahoo.vespa.orchestrator.BatchHostNameNotFoundException; +import com.yahoo.vespa.orchestrator.BatchInternalErrorException; +import com.yahoo.vespa.orchestrator.Orchestrator; +import com.yahoo.vespa.orchestrator.OrchestratorImpl; +import com.yahoo.vespa.orchestrator.policy.BatchHostStateChangeDeniedException; +import com.yahoo.vespa.orchestrator.resources.host.HostResourceTest; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.time.Clock; +import java.time.Instant; +import java.util.List; +import java.util.concurrent.Executors; +import java.util.stream.Collectors; + +import static com.yahoo.jdisc.http.HttpRequest.Method; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * @author hakonhall + * @author bjorncs + */ +class HostSuspensionHandlerTest { + + private final Clock clock = mock(Clock.class); + + @BeforeEach + public void setUp() { + when(clock.instant()).thenReturn(Instant.now()); + } + + @Test + void returns_200_on_success_batch() throws IOException { + HostSuspensionHandler handler = createHandler(HostResourceTest.createAlwaysAllowOrchestrator(clock)); + HttpResponse response = executeSuspendAllRequest(handler, "parentHostname", List.of("hostname1", "hostname2")); + assertSuccess(response); + } + + @Test + void returns_200_empty_batch() throws IOException { + HostSuspensionHandler handler = createHandler(HostResourceTest.createAlwaysAllowOrchestrator(clock)); + HttpResponse response = executeSuspendAllRequest(handler, "parentHostname", List.of()); + assertSuccess(response); + } + + // Note: Missing host is 404 for a single-host, but 400 for multi-host (batch). + // This is so because the hostname is part of the URL path for single-host, while the + // hostnames are part of the request body for multi-host. + @Test + void returns_400_when_host_unknown_for_batch() { + HostSuspensionHandler handler = createHandler(HostResourceTest.createHostNotFoundOrchestrator(clock)); + HttpResponse response = executeSuspendAllRequest(handler, "parentHostname", List.of("hostname1", "hostname2")); + assertEquals(400, response.getStatus()); + } + + @Test + void returns_409_when_request_rejected_by_policies_for_batch() { + OrchestratorImpl alwaysRejectResolver = HostResourceTest.createAlwaysRejectResolver(clock); + HostSuspensionHandler handler = createHandler(alwaysRejectResolver); + HttpResponse response = executeSuspendAllRequest(handler, "parentHostname", List.of("hostname1", "hostname2")); + assertEquals(409, response.getStatus()); + } + + + @Test + void throws_409_on_suspendAll_timeout() throws BatchHostStateChangeDeniedException, BatchHostNameNotFoundException, BatchInternalErrorException { + Orchestrator orchestrator = mock(Orchestrator.class); + doThrow(new UncheckedTimeoutException("Timeout Message")).when(orchestrator).suspendAll(any(), any()); + HostSuspensionHandler handler = createHandler(orchestrator); + HttpResponse response = executeSuspendAllRequest(handler, "parenthost", List.of("h1", "h2", "h3")); + assertEquals(409, response.getStatus()); + } + + private static HostSuspensionHandler createHandler(Orchestrator orchestrator) { + return new HostSuspensionHandler( + new LoggingRequestHandler.Context(Executors.newSingleThreadExecutor(), new MockMetric()), + orchestrator); + } + + private static HttpResponse executeSuspendAllRequest(HostSuspensionHandler handler, String parentHostname, List<String> hostnames) { + StringBuilder uriBuilder = new StringBuilder("/orchestrator/v1/suspensions/hosts/").append(parentHostname); + if (!hostnames.isEmpty()) { + uriBuilder.append(hostnames.stream() + .map(hostname -> "hostname=" + hostname) + .collect(Collectors.joining("&", "?", ""))); + } + HttpRequest request = HttpRequest.createTestRequest(uriBuilder.toString(), Method.PUT); + return handler.handle(request); + } + + private static void assertSuccess(HttpResponse response) throws IOException { + assertEquals(200, response.getStatus()); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + response.render(out); + JsonTestHelper.assertJsonEquals(out.toString(), "{\"failure-reason\": null}"); + } + +} diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/ApplicationSuspensionResourceTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/appsuspension/ApplicationSuspensionResourceTest.java index 4035e29d91a..a7514de5acd 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/ApplicationSuspensionResourceTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/appsuspension/ApplicationSuspensionResourceTest.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.orchestrator.resources; +package com.yahoo.vespa.orchestrator.resources.appsuspension; import com.yahoo.application.Application; import com.yahoo.application.Networking; @@ -152,8 +152,10 @@ public class ApplicationSuspensionResourceTest { " <component id=\"com.yahoo.vespa.orchestrator.OrchestratorImpl\" bundle=\"orchestrator\" />\n" + " <component id=\"com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactoryMock\" bundle=\"orchestrator\" />\n" + "\n" + - " <rest-api path=\"orchestrator\">\n" + - " <components bundle=\"orchestrator\" />\n" + + " <rest-api path=\"orchestrator/v1/suspensions/applications\" jersey2=\"true\">\n" + + " <components bundle=\"orchestrator\">\n" + + " <package>com.yahoo.vespa.orchestrator.resources.appsuspension</package>\n" + + " </components>\n" + " </rest-api>\n" + "\n" + " <http>\n" + diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/host/HostResourceTest.java index 7f4ef1a336c..d056c3730fd 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/host/HostResourceTest.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.orchestrator.resources; +package com.yahoo.vespa.orchestrator.resources.host; import com.google.common.util.concurrent.UncheckedTimeoutException; import com.yahoo.jdisc.Metric; @@ -17,8 +17,6 @@ import com.yahoo.vespa.applicationmodel.ServiceType; import com.yahoo.vespa.applicationmodel.TenantId; import com.yahoo.vespa.curator.mock.MockCurator; import com.yahoo.vespa.flags.InMemoryFlagSource; -import com.yahoo.vespa.orchestrator.BatchHostNameNotFoundException; -import com.yahoo.vespa.orchestrator.BatchInternalErrorException; import com.yahoo.vespa.orchestrator.DummyAntiServiceMonitor; import com.yahoo.vespa.orchestrator.Host; import com.yahoo.vespa.orchestrator.HostNameNotFoundException; @@ -29,11 +27,9 @@ import com.yahoo.vespa.orchestrator.OrchestratorImpl; import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactoryMock; import com.yahoo.vespa.orchestrator.model.ApplicationApi; import com.yahoo.vespa.orchestrator.model.ApplicationApiFactory; -import com.yahoo.vespa.orchestrator.policy.BatchHostStateChangeDeniedException; import com.yahoo.vespa.orchestrator.policy.HostStateChangeDeniedException; import com.yahoo.vespa.orchestrator.policy.Policy; import com.yahoo.vespa.orchestrator.policy.SuspensionReasons; -import com.yahoo.vespa.orchestrator.restapi.wire.BatchOperationResult; import com.yahoo.vespa.orchestrator.restapi.wire.GetHostResponse; import com.yahoo.vespa.orchestrator.restapi.wire.PatchHostRequest; import com.yahoo.vespa.orchestrator.restapi.wire.PatchHostResponse; @@ -56,15 +52,12 @@ import javax.ws.rs.core.UriInfo; import java.net.URI; import java.time.Clock; import java.time.Instant; -import java.util.Arrays; import java.util.Collections; -import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doThrow; @@ -95,8 +88,6 @@ public class HostResourceTest { Set.of()))); } - private final InMemoryFlagSource flagSource = new InMemoryFlagSource(); - private static final ServiceMonitor alwaysEmptyServiceMonitor = new ServiceMonitor() { private final ServiceModel emptyServiceModel = new ServiceModel(Map.of()); @@ -128,26 +119,8 @@ public class HostResourceTest { } } - private final OrchestratorImpl alwaysAllowOrchestrator = new OrchestratorImpl( - new AlwaysAllowPolicy(), - new ClusterControllerClientFactoryMock(), - EVERY_HOST_IS_UP_HOST_STATUS_SERVICE, - serviceMonitor, - SERVICE_MONITOR_CONVERGENCE_LATENCY_SECONDS, - clock, - applicationApiFactory, - flagSource); - - private final OrchestratorImpl hostNotFoundOrchestrator = new OrchestratorImpl( - new AlwaysAllowPolicy(), - new ClusterControllerClientFactoryMock(), - EVERY_HOST_IS_UP_HOST_STATUS_SERVICE, - alwaysEmptyServiceMonitor, - SERVICE_MONITOR_CONVERGENCE_LATENCY_SECONDS, - clock, - applicationApiFactory, - flagSource); - + private final OrchestratorImpl alwaysAllowOrchestrator = createAlwaysAllowOrchestrator(clock); + private final OrchestratorImpl hostNotFoundOrchestrator = createHostNotFoundOrchestrator(clock); private final UriInfo uriInfo = mock(UriInfo.class); @Before @@ -155,6 +128,42 @@ public class HostResourceTest { when(clock.instant()).thenReturn(Instant.now()); } + public static OrchestratorImpl createAlwaysAllowOrchestrator(Clock clock) { + return new OrchestratorImpl( + new AlwaysAllowPolicy(), + new ClusterControllerClientFactoryMock(), + EVERY_HOST_IS_UP_HOST_STATUS_SERVICE, + serviceMonitor, + SERVICE_MONITOR_CONVERGENCE_LATENCY_SECONDS, + clock, + applicationApiFactory, + new InMemoryFlagSource()); + } + + public static OrchestratorImpl createHostNotFoundOrchestrator(Clock clock) { + return new OrchestratorImpl( + new AlwaysAllowPolicy(), + new ClusterControllerClientFactoryMock(), + EVERY_HOST_IS_UP_HOST_STATUS_SERVICE, + alwaysEmptyServiceMonitor, + SERVICE_MONITOR_CONVERGENCE_LATENCY_SECONDS, + clock, + applicationApiFactory, + new InMemoryFlagSource()); + } + + public static OrchestratorImpl createAlwaysRejectResolver(Clock clock) { + return new OrchestratorImpl( + new HostResourceTest.AlwaysFailPolicy(), + new ClusterControllerClientFactoryMock(), + EVERY_HOST_IS_UP_HOST_STATUS_SERVICE, + serviceMonitor, + SERVICE_MONITOR_CONVERGENCE_LATENCY_SECONDS, + clock, + applicationApiFactory, + new InMemoryFlagSource()); + } + @Test public void returns_200_on_success() { HostResource hostResource = @@ -168,21 +177,6 @@ public class HostResourceTest { } @Test - public void returns_200_on_success_batch() { - HostSuspensionResource hostSuspensionResource = new HostSuspensionResource(alwaysAllowOrchestrator); - BatchOperationResult response = hostSuspensionResource.suspendAll("parentHostname", - Arrays.asList("hostname1", "hostname2")); - assertTrue(response.success()); - } - - @Test - public void returns_200_empty_batch() { - HostSuspensionResource hostSuspensionResource = new HostSuspensionResource(alwaysAllowOrchestrator); - BatchOperationResult response = hostSuspensionResource.suspendAll("parentHostname", List.of()); - assertTrue(response.success()); - } - - @Test public void throws_404_when_host_unknown() { try { HostResource hostResource = @@ -194,20 +188,6 @@ public class HostResourceTest { } } - // Note: Missing host is 404 for a single-host, but 400 for multi-host (batch). - // This is so because the hostname is part of the URL path for single-host, while the - // hostnames are part of the request body for multi-host. - @Test - public void throws_400_when_host_unknown_for_batch() { - try { - HostSuspensionResource hostSuspensionResource = new HostSuspensionResource(hostNotFoundOrchestrator); - hostSuspensionResource.suspendAll("parentHostname", Arrays.asList("hostname1", "hostname2")); - fail(); - } catch (WebApplicationException w) { - assertEquals(400, w.getResponse().getStatus()); - } - } - private static class AlwaysFailPolicy implements Policy { @Override public SuspensionReasons grantSuspensionRequest(OrchestratorContext context, ApplicationApi applicationApi) throws HostStateChangeDeniedException { @@ -250,7 +230,7 @@ public class HostResourceTest { SERVICE_MONITOR_CONVERGENCE_LATENCY_SECONDS, clock, applicationApiFactory, - flagSource); + new InMemoryFlagSource()); try { HostResource hostResource = new HostResource(alwaysRejectResolver, uriInfo); @@ -261,27 +241,6 @@ public class HostResourceTest { } } - @Test - public void throws_409_when_request_rejected_by_policies_for_batch() { - final OrchestratorImpl alwaysRejectResolver = new OrchestratorImpl( - new AlwaysFailPolicy(), - new ClusterControllerClientFactoryMock(), - EVERY_HOST_IS_UP_HOST_STATUS_SERVICE, - serviceMonitor, - SERVICE_MONITOR_CONVERGENCE_LATENCY_SECONDS, - clock, - applicationApiFactory, - flagSource); - - try { - HostSuspensionResource hostSuspensionResource = new HostSuspensionResource(alwaysRejectResolver); - hostSuspensionResource.suspendAll("parentHostname", Arrays.asList("hostname1", "hostname2")); - fail(); - } catch (WebApplicationException w) { - assertEquals(409, w.getResponse().getStatus()); - } - } - @Test(expected = BadRequestException.class) public void patch_state_may_throw_bad_request() { Orchestrator orchestrator = mock(Orchestrator.class); @@ -380,17 +339,4 @@ public class HostResourceTest { } } - @Test - public void throws_409_on_suspendAll_timeout() throws BatchHostStateChangeDeniedException, BatchHostNameNotFoundException, BatchInternalErrorException { - Orchestrator orchestrator = mock(Orchestrator.class); - doThrow(new UncheckedTimeoutException("Timeout Message")).when(orchestrator).suspendAll(any(), any()); - - try { - HostSuspensionResource resource = new HostSuspensionResource(orchestrator); - resource.suspendAll("parenthost", Arrays.asList("h1", "h2", "h3")); - fail(); - } catch (WebApplicationException w) { - assertEquals(409, w.getResponse().getStatus()); - } - } } diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/InstanceResourceTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/instance/InstanceResourceTest.java index ef6e26d2e99..8e2eeb7410d 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/InstanceResourceTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/instance/InstanceResourceTest.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.orchestrator.resources; +package com.yahoo.vespa.orchestrator.resources.instance; import com.fasterxml.jackson.databind.ObjectMapper; import com.yahoo.config.provision.ApplicationId; @@ -9,6 +9,7 @@ import com.yahoo.vespa.applicationmodel.ConfigId; import com.yahoo.vespa.applicationmodel.ServiceStatus; import com.yahoo.vespa.applicationmodel.ServiceStatusInfo; import com.yahoo.vespa.applicationmodel.ServiceType; +import com.yahoo.vespa.orchestrator.resources.instance.InstanceResource; import com.yahoo.vespa.orchestrator.restapi.wire.SlobrokEntryResponse; import com.yahoo.vespa.service.manager.UnionMonitorManager; import com.yahoo.vespa.service.monitor.SlobrokApi; @@ -94,12 +94,10 @@ <module>jdisc_core</module> <module>jdisc_core_test</module> <module>jdisc_jetty</module> - <module>jdisc_messagebus_service</module> <module>jrt</module> <module>linguistics</module> <module>logd</module> <module>logserver</module> - <module>messagebus-disc</module> <module>messagebus</module> <module>metrics</module> <module>metrics-proxy</module> diff --git a/searchcore/src/tests/proton/common/attribute_updater/attribute_updater_test.cpp b/searchcore/src/tests/proton/common/attribute_updater/attribute_updater_test.cpp index c74c93a376a..e92f0363ec2 100644 --- a/searchcore/src/tests/proton/common/attribute_updater/attribute_updater_test.cpp +++ b/searchcore/src/tests/proton/common/attribute_updater/attribute_updater_test.cpp @@ -30,7 +30,7 @@ #include <vespa/searchlib/attribute/attributefactory.h> #include <vespa/searchlib/attribute/reference_attribute.h> #include <vespa/searchlib/tensor/dense_tensor_attribute.h> -#include <vespa/searchlib/tensor/serialized_tensor_attribute.h> +#include <vespa/searchlib/tensor/serialized_fast_value_attribute.h> #include <vespa/searchlib/test/weighted_type_test_utils.h> #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/vespalib/testkit/testapp.h> @@ -50,7 +50,7 @@ using search::attribute::Reference; using search::attribute::ReferenceAttribute; using search::tensor::ITensorAttribute; using search::tensor::DenseTensorAttribute; -using search::tensor::SerializedTensorAttribute; +using search::tensor::SerializedFastValueAttribute; using search::tensor::TensorAttribute; using vespalib::eval::SimpleValue; using vespalib::eval::TensorSpec; @@ -459,7 +459,7 @@ TEST_F("require that tensor modify update is applied", } TEST_F("require that tensor add update is applied", - TensorFixture<SerializedTensorAttribute>("tensor(x{})", "sparse_tensor")) + TensorFixture<SerializedFastValueAttribute>("tensor(x{})", "sparse_tensor")) { f.setTensor(TensorSpec(f.type).add({{"x", "a"}}, 2)); f.applyValueUpdate(*f.attribute, 1, @@ -468,7 +468,7 @@ TEST_F("require that tensor add update is applied", } TEST_F("require that tensor add update to non-existing tensor creates empty tensor first", - TensorFixture<SerializedTensorAttribute>("tensor(x{})", "sparse_tensor")) + TensorFixture<SerializedFastValueAttribute>("tensor(x{})", "sparse_tensor")) { f.applyValueUpdate(*f.attribute, 1, TensorAddUpdate(makeTensorFieldValue(TensorSpec(f.type).add({{"x", "a"}}, 3)))); @@ -476,7 +476,7 @@ TEST_F("require that tensor add update to non-existing tensor creates empty tens } TEST_F("require that tensor remove update is applied", - TensorFixture<SerializedTensorAttribute>("tensor(x{})", "sparse_tensor")) + TensorFixture<SerializedFastValueAttribute>("tensor(x{})", "sparse_tensor")) { f.setTensor(TensorSpec(f.type).add({{"x", "a"}}, 2).add({{"x", "b"}}, 3)); f.applyValueUpdate(*f.attribute, 1, diff --git a/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp b/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp index 4e04677f972..5dd6f0ff10f 100644 --- a/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp +++ b/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp @@ -2091,6 +2091,63 @@ TEST(DocumentMetaStoreTest, call_to_remove_is_notified) EXPECT_EQ(1, listener->remove_cnt); } +namespace { + +void try_compact_document_meta_store(DocumentMetaStore &dms) +{ + dms.removeAllOldGenerations(); + dms.commit(true); +} + +} + +TEST(DocumentMetaStoreTest, gid_to_lid_map_can_be_compacted) +{ + auto dms = std::make_shared<DocumentMetaStore>(createBucketDB()); + dms->constructFreeList(); + static constexpr uint32_t full_size = 1000; + for (uint32_t i = 1; i < full_size; ++i) { + addLid(*dms, i); + } + dms->commit(true); + AttributeGuard guard(dms); + remove(full_size - 1, 100, *dms); + dms->commit(true); + auto status_before = dms->getStatus(); + EXPECT_LT(0, status_before.getOnHold()); + guard = AttributeGuard(); + try_compact_document_meta_store(*dms); + auto status_early = dms->getStatus(); + EXPECT_LT(status_before.getDead(), status_early.getDead()); + EXPECT_EQ(0, status_early.getOnHold()); + bool compaction_done = false; + for (uint32_t i = 0; i < 15 && !compaction_done; ++i) { + AttributeGuard guard2(dms); + auto status_loop_iteration_start = dms->getStatus(); + try_compact_document_meta_store(*dms); + try_compact_document_meta_store(*dms); + auto status_second = dms->getStatus(); + if (i > 0) { + EXPECT_GT(status_before.getUsed(), status_second.getUsed()); + } + EXPECT_GT(status_early.getDead(), status_second.getDead()); + try_compact_document_meta_store(*dms); + auto status_third = dms->getStatus(); + EXPECT_EQ(status_second.getDead(), status_third.getDead()); + EXPECT_EQ(status_second.getUsed(), status_third.getUsed()); + EXPECT_EQ(status_second.getOnHold(), status_third.getOnHold()); + EXPECT_GE(status_loop_iteration_start.getDead(), status_third.getDead()); + if (status_loop_iteration_start.getDead() == status_third.getDead()) { + compaction_done = true; + } + } + EXPECT_TRUE(compaction_done); + auto status_after = dms->getStatus(); + EXPECT_GT(status_before.getUsed(), status_after.getUsed()); + EXPECT_GT(status_early.getDead(), status_after.getDead()); + EXPECT_EQ(0, status_after.getOnHold()); +} + } GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchcore/src/tests/proton/matching/matching_test.cpp b/searchcore/src/tests/proton/matching/matching_test.cpp index f68e4956e15..8057dbf2da6 100644 --- a/searchcore/src/tests/proton/matching/matching_test.cpp +++ b/searchcore/src/tests/proton/matching/matching_test.cpp @@ -299,7 +299,7 @@ struct MyWorld { void verify_diversity_filter(SearchRequest::SP req, bool expectDiverse) { Matcher::SP matcher = createMatcher(); search::fef::Properties overrides; - auto mtf = matcher->create_match_tools_factory(*req, searchContext, attributeContext, metaStore, overrides); + auto mtf = matcher->create_match_tools_factory(*req, searchContext, attributeContext, metaStore, overrides, true); auto diversity = mtf->createDiversifier(HeapSize::lookup(config)); EXPECT_EQUAL(expectDiverse, static_cast<bool>(diversity)); } @@ -309,7 +309,7 @@ struct MyWorld { SearchRequest::SP request = createSimpleRequest("f1", "spread"); search::fef::Properties overrides; MatchToolsFactory::UP match_tools_factory = matcher->create_match_tools_factory( - *request, searchContext, attributeContext, metaStore, overrides); + *request, searchContext, attributeContext, metaStore, overrides, true); MatchTools::UP match_tools = match_tools_factory->createMatchTools(); match_tools->setup_first_phase(); return match_tools->match_data().get_termwise_limit(); diff --git a/searchcore/src/tests/proton/matching/query_test.cpp b/searchcore/src/tests/proton/matching/query_test.cpp index 7cc11b0dc0c..02e6b05622e 100644 --- a/searchcore/src/tests/proton/matching/query_test.cpp +++ b/searchcore/src/tests/proton/matching/query_test.cpp @@ -164,6 +164,17 @@ fef_test::IndexEnvironment plain_index_env; fef_test::IndexEnvironment resolved_index_env; fef_test::IndexEnvironment attribute_index_env; +vespalib::string +termAsString(const search::query::Range &term) { + vespalib::asciistream os; + return (os << term).str(); +} + +const vespalib::string & +termAsString(const vespalib::string & term) { + return term; +} + void setupIndexEnvironments() { FieldInfo field_info(FieldType::INDEX, CollectionType::SINGLE, field, field_id); @@ -898,9 +909,9 @@ void Test::requireThatWeakAndBlueprintsAreCreatedCorrectly() { void Test::requireThatParallelWandBlueprintsAreCreatedCorrectly() { using search::queryeval::WeakAndBlueprint; - ProtonWandTerm wand(field, 42, Weight(100), 123, 9000, 1.25); - wand.append(Node::UP(new ProtonStringTerm("foo", field, 0, Weight(3)))); - wand.append(Node::UP(new ProtonStringTerm("bar", field, 0, Weight(7)))); + ProtonWandTerm wand(2, field, 42, Weight(100), 123, 9000, 1.25); + wand.addTerm("foo", Weight(3)); + wand.addTerm("bar", Weight(7)); ViewResolver viewResolver; ResolveViewVisitor resolve_visitor(viewResolver, attribute_index_env); diff --git a/searchcore/src/tests/proton/matching/querynodes_test.cpp b/searchcore/src/tests/proton/matching/querynodes_test.cpp index 07f4e53bf6d..7795d1563d2 100644 --- a/searchcore/src/tests/proton/matching/querynodes_test.cpp +++ b/searchcore/src/tests/proton/matching/querynodes_test.cpp @@ -518,6 +518,13 @@ TEST("requireThatSimpleIntermediatesGetProperBlending") { TEST_DO(checkProperBlendingWithParent<Rank>()); } +TEST("control query nodes size") { + EXPECT_EQUAL(160u, sizeof(search::query::NumberTerm)); + EXPECT_EQUAL(192u, sizeof(ProtonNodeTypes::NumberTerm)); + EXPECT_EQUAL(160u, sizeof(search::query::StringTerm)); + EXPECT_EQUAL(192u, sizeof(ProtonNodeTypes::StringTerm)); +} + } // namespace TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchcore/src/tests/proton/matching/termdataextractor_test.cpp b/searchcore/src/tests/proton/matching/termdataextractor_test.cpp index fb338d409e7..c5da26d9847 100644 --- a/searchcore/src/tests/proton/matching/termdataextractor_test.cpp +++ b/searchcore/src/tests/proton/matching/termdataextractor_test.cpp @@ -39,31 +39,6 @@ namespace search { class AttributeManager; } namespace { -class Test : public vespalib::TestApp { - void requireThatTermsAreAdded(); - void requireThatAViewWithTwoFieldsGivesOneTermDataPerTerm(); - void requireThatUnrankedTermsAreSkipped(); - void requireThatNegativeTermsAreSkipped(); - void requireThatSameElementIsSkipped(); - -public: - int Main() override; -}; - -int -Test::Main() -{ - TEST_INIT("termdataextractor_test"); - - TEST_DO(requireThatTermsAreAdded()); - TEST_DO(requireThatAViewWithTwoFieldsGivesOneTermDataPerTerm()); - TEST_DO(requireThatUnrankedTermsAreSkipped()); - TEST_DO(requireThatNegativeTermsAreSkipped()); - TEST_DO(requireThatSameElementIsSkipped()); - - TEST_DONE(); -} - const string field = "field"; const uint32_t id[] = { 10, 11, 12, 13, 14, 15, 16, 17, 18 }; @@ -77,14 +52,10 @@ Node::UP getQuery(const ViewResolver &resolver) query_builder.addSubstringTerm("baz", field, id[3], Weight(0)); query_builder.addSuffixTerm("qux", field, id[4], Weight(0)); query_builder.addRangeTerm(Range(), field, id[5], Weight(0)); - query_builder.addWeightedSetTerm(1, field, id[6], Weight(0)); - { - // weighted token - query_builder.addStringTerm("bar", field, id[3], Weight(0)); - } + query_builder.addWeightedSetTerm(1, field, id[6], Weight(0)) + .addTerm("bar", Weight(0)); - query_builder.addLocationTerm(Location(Point{10, 10}, 3, 0), - field, id[7], Weight(0)); + query_builder.addLocationTerm(Location(Point{10, 10}, 3, 0), field, id[7], Weight(0)); Node::UP node = query_builder.build(); fef_test::IndexEnvironment index_environment; @@ -98,7 +69,7 @@ Node::UP getQuery(const ViewResolver &resolver) return node; } -void Test::requireThatTermsAreAdded() { +TEST("requireThatTermsAreAdded") { Node::UP node = getQuery(ViewResolver()); vector<const ITermData *> term_data; @@ -110,7 +81,7 @@ void Test::requireThatTermsAreAdded() { } } -void Test::requireThatAViewWithTwoFieldsGivesOneTermDataPerTerm() { +TEST("requireThatAViewWithTwoFieldsGivesOneTermDataPerTerm") { ViewResolver resolver; resolver.add(field, "foo"); resolver.add(field, "bar"); @@ -125,9 +96,7 @@ void Test::requireThatAViewWithTwoFieldsGivesOneTermDataPerTerm() { } } -void -Test::requireThatUnrankedTermsAreSkipped() -{ +TEST("requireThatUnrankedTermsAreSkipped") { QueryBuilder<ProtonNodeTypes> query_builder; query_builder.addAnd(2); query_builder.addStringTerm("term1", field, id[0], Weight(0)); @@ -142,9 +111,7 @@ Test::requireThatUnrankedTermsAreSkipped() EXPECT_EQUAL(id[0], term_data[0]->getUniqueId()); } -void -Test::requireThatNegativeTermsAreSkipped() -{ +TEST("requireThatNegativeTermsAreSkipped") { QueryBuilder<ProtonNodeTypes> query_builder; query_builder.addAnd(2); query_builder.addStringTerm("term1", field, id[0], Weight(0)); @@ -163,8 +130,7 @@ Test::requireThatNegativeTermsAreSkipped() EXPECT_EQUAL(id[1], term_data[1]->getUniqueId()); } -void -Test::requireThatSameElementIsSkipped() +TEST("requireThatSameElementIsSkipped") { QueryBuilder<ProtonNodeTypes> query_builder; query_builder.addAnd(2); @@ -183,4 +149,4 @@ Test::requireThatSameElementIsSkipped() } // namespace -TEST_APPHOOK(Test); +TEST_MAIN() { TEST_RUN_ALL(); }
\ No newline at end of file diff --git a/searchcore/src/tests/proton/server/disk_mem_usage_sampler/disk_mem_usage_sampler_test.cpp b/searchcore/src/tests/proton/server/disk_mem_usage_sampler/disk_mem_usage_sampler_test.cpp index 666b4b50885..b7f217290d6 100644 --- a/searchcore/src/tests/proton/server/disk_mem_usage_sampler/disk_mem_usage_sampler_test.cpp +++ b/searchcore/src/tests/proton/server/disk_mem_usage_sampler/disk_mem_usage_sampler_test.cpp @@ -55,7 +55,7 @@ TEST_F(DiskMemUsageSamplerTest, resource_usage_is_sampled) { // Poll for up to 20 seconds to get a sample. size_t i = 0; - for (; i < (20s / 50ms); ++i) { + for (; i < static_cast<size_t>(20s / 50ms); ++i) { if (filter().get_transient_memory_usage() > 0) { break; } diff --git a/searchcore/src/vespa/searchcore/config/proton.def b/searchcore/src/vespa/searchcore/config/proton.def index 9ae15bd919e..55dc036df63 100644 --- a/searchcore/src/vespa/searchcore/config/proton.def +++ b/searchcore/src/vespa/searchcore/config/proton.def @@ -19,6 +19,16 @@ partition int default=0 restart ## Distribution key distributionkey int default=-1 +## Number of threads used for rpc transport threads +## A zero value will make the backend smart about the number. +rpc.transportthreads int default=0 restart + +## Dispatch search requests to threadpool +search.async bool default=true + +## Dispatch docsum requests to threadpool +docsum.async bool default=true + ## Num searcher threads numsearcherthreads int default=64 restart diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp index 8152761c64d..d0cada2224f 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp @@ -199,6 +199,39 @@ DocumentMetaStore::insert(GidToLidMapKey key, const RawDocumentMetaData &metaDat updateCommittedDocIdLimit(); } +namespace { + +// minimum dead bytes in gid to lid map before consider compaction +constexpr size_t DEAD_BYTES_SLACK = 0x10000u; + +} + +bool +DocumentMetaStore::consider_compact_gid_to_lid_map() +{ + if (_gidToLidMap.getAllocator().getNodeStore().has_held_buffers()) { + return false; + } + auto &compaction_strategy = getConfig().getCompactionStrategy(); + size_t used_bytes = _cached_gid_to_lid_map_memory_usage.usedBytes(); + size_t dead_bytes = _cached_gid_to_lid_map_memory_usage.deadBytes(); + bool compact_memory = ((dead_bytes >= DEAD_BYTES_SLACK) && + (used_bytes * compaction_strategy.getMaxDeadBytesRatio() < dead_bytes)); + return compact_memory; +} + +void +DocumentMetaStore::onCommit() +{ + if (consider_compact_gid_to_lid_map()) { + _gidToLidMap.compact_worst(); + _gid_to_lid_map_write_itr_prepare_serial_num = 0u; + _gid_to_lid_map_write_itr.begin(_gidToLidMap.getRoot()); + this->incGeneration(); + this->updateStat(true); + } +} + void DocumentMetaStore::onUpdateStat() { @@ -207,7 +240,8 @@ DocumentMetaStore::onUpdateStat() size_t bvSize = _lidAlloc.getUsedLidsSize(); usage.incAllocatedBytes(bvSize); usage.incUsedBytes(bvSize); - usage.merge(_gidToLidMap.getMemoryUsage()); + _cached_gid_to_lid_map_memory_usage = _gidToLidMap.getMemoryUsage(); + usage.merge(_cached_gid_to_lid_map_memory_usage); // the free lists are not taken into account here updateStatistics(_metaDataStore.size(), _metaDataStore.size(), @@ -400,7 +434,8 @@ DocumentMetaStore::DocumentMetaStore(BucketDBOwnerSP bucketDB, _shrinkLidSpaceBlockers(0), _subDbType(subDbType), _trackDocumentSizes(true), - _op_listener() + _op_listener(), + _cached_gid_to_lid_map_memory_usage() { ensureSpace(0); // lid 0 is reserved setCommittedDocIdLimit(1u); // lid 0 is reserved diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.h index e5a009a6bd2..9b8fa26f9c7 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.h +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.h @@ -75,6 +75,7 @@ private: const SubDbType _subDbType; bool _trackDocumentSizes; OperationListenerSP _op_listener; + vespalib::MemoryUsage _cached_gid_to_lid_map_memory_usage; DocId getFreeLid(); DocId peekFreeLid(); @@ -83,6 +84,8 @@ private: const GlobalId & getRawGid(DocId lid) const { return getRawMetaData(lid).getGid(); } + bool consider_compact_gid_to_lid_map(); + void onCommit() override; void onUpdateStat() override; // Implements AttributeVector diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreattribute.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreattribute.h index f4e936e663a..01b259615d8 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreattribute.h +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreattribute.h @@ -23,8 +23,6 @@ public: size_t getFixedWidth() const override { return document::GlobalId::LENGTH; } - - void onCommit() override {} }; } diff --git a/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.cpp b/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.cpp index 16946c398b6..513d2630854 100644 --- a/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.cpp +++ b/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.cpp @@ -26,12 +26,10 @@ public: : _engine(engine), _request(std::move(request)), _client(client) - { - // empty - } + { } void run() override { - _engine.performSearch(std::move(_request), _client); + _client.searchDone(_engine.performSearch(std::move(_request))); } }; @@ -43,9 +41,10 @@ namespace proton { using namespace vespalib::slime; -MatchEngine::MatchEngine(size_t numThreads, size_t threadsPerSearch, uint32_t distributionKey) +MatchEngine::MatchEngine(size_t numThreads, size_t threadsPerSearch, uint32_t distributionKey, bool async) : _lock(), _distributionKey(distributionKey), + _async(async), _closed(false), _handlers(), _executor(std::max(size_t(1), numThreads / threadsPerSearch), 256_Ki, match_engine_executor), @@ -106,13 +105,15 @@ MatchEngine::search(search::engine::SearchRequest::Source request, return ret; } - _executor.execute(std::make_unique<SearchTask>(*this, std::move(request), client)); - return search::engine::SearchReply::UP(); + if (_async) { + _executor.execute(std::make_unique<SearchTask>(*this, std::move(request), client)); + return search::engine::SearchReply::UP(); + } + return performSearch(std::move(request)); } -void -MatchEngine::performSearch(search::engine::SearchRequest::Source req, - search::engine::SearchClient &client) +std::unique_ptr<search::engine::SearchReply> +MatchEngine::performSearch(search::engine::SearchRequest::Source req) { auto ret = std::make_unique<search::engine::SearchReply>(); @@ -151,7 +152,7 @@ MatchEngine::performSearch(search::engine::SearchRequest::Source req, vespalib::slime::BinaryFormat::encode(ret->request->trace().getSlime(), output); trace.add("slime", output.obtain().make_stringref()); } - client.searchDone(std::move(ret)); + return ret; } bool MatchEngine::isOnline() const { diff --git a/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.h b/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.h index 8adc198bb40..39ab95c4bc2 100644 --- a/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.h +++ b/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.h @@ -19,6 +19,7 @@ class MatchEngine : public search::engine::SearchServer, private: std::mutex _lock; const uint32_t _distributionKey; + bool _async; bool _closed; HandlerMap<ISearchHandler> _handlers; vespalib::ThreadStackExecutor _executor; @@ -43,8 +44,12 @@ public: * @param numThreads Number of threads allocated for handling search requests. * @param threadsPerSearch number of threads used for each search * @param distributionKey distributionkey of this node. + * @param async if query is dispatched to threadpool */ - MatchEngine(size_t numThreads, size_t threadsPerSearch, uint32_t distributionKey); + MatchEngine(size_t numThreads, size_t threadsPerSearch, uint32_t distributionKey, bool async); + MatchEngine(size_t numThreads, size_t threadsPerSearch, uint32_t distributionKey) + : MatchEngine(numThreads, threadsPerSearch, distributionKey, true) + {} /** * Frees any allocated resources. this will also stop all internal threads @@ -114,8 +119,8 @@ public: * @param req The search request to perform. * @param client The client to pass the results to. */ - void performSearch(search::engine::SearchRequest::Source req, - search::engine::SearchClient &client); + std::unique_ptr<search::engine::SearchReply> + performSearch(search::engine::SearchRequest::Source req); /** obtain current online status */ bool isOnline() const; @@ -123,8 +128,7 @@ public: /** * Set node up/down, based on info from cluster controller. */ - void - setNodeUp(bool nodeUp); + void setNodeUp(bool nodeUp); StatusReport::UP reportStatus() const; diff --git a/searchcore/src/vespa/searchcore/proton/matching/handlerecorder.h b/searchcore/src/vespa/searchcore/proton/matching/handlerecorder.h index a114291b6bd..49399f307dc 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/handlerecorder.h +++ b/searchcore/src/vespa/searchcore/proton/matching/handlerecorder.h @@ -36,6 +36,7 @@ public: HandleRecorder(); ~HandleRecorder(); const HandleMap& get_handles() const { return _handles; } + HandleMap steal_handles() && { return std::move(_handles); } static void register_handle(search::fef::TermFieldHandle handle, search::fef::MatchDataDetails requested_details); vespalib::string to_string() const; diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp b/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp index 19bece5ae9c..55194e51048 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp @@ -92,7 +92,7 @@ MatchTools::setup(search::fef::RankProgram::UP rank_program, double termwise_lim recorder.tag_match_data(*_match_data); _match_data->set_termwise_limit(termwise_limit); _search = _query.createSearch(*_match_data); - _used_handles = recorder.get_handles(); + _used_handles = std::move(recorder).steal_handles(); _search_has_changed = false; } } @@ -167,7 +167,8 @@ MatchToolsFactory(QueryLimiter & queryLimiter, const IIndexEnvironment & indexEnv, const RankSetup & rankSetup, const Properties & rankProperties, - const Properties & featureOverrides) + const Properties & featureOverrides, + bool is_search) : _queryLimiter(queryLimiter), _requestContext(doom, attributeContext, rankProperties, extractAttributeBlueprintParams(rankSetup, rankProperties)), _query(), @@ -193,9 +194,11 @@ MatchToolsFactory(QueryLimiter & queryLimiter, _query.optimize(); trace.addEvent(4, "MTF: Fetch Postings"); _query.fetchPostings(); - trace.addEvent(5, "MTF: Handle Global Filters"); - double global_filter_limit = GlobalFilterLimit::lookup(rankProperties, rankSetup.get_global_filter_limit()); - _query.handle_global_filters(searchContext.getDocIdLimit(), global_filter_limit); + if (is_search) { + trace.addEvent(5, "MTF: Handle Global Filters"); + double global_filter_limit = GlobalFilterLimit::lookup(rankProperties, rankSetup.get_global_filter_limit()); + _query.handle_global_filters(searchContext.getDocIdLimit(), global_filter_limit); + } _query.freeze(); trace.addEvent(5, "MTF: prepareSharedState"); _rankSetup.prepareSharedState(_queryEnv, _queryEnv.getObjectStore()); diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_tools.h b/searchcore/src/vespa/searchcore/proton/matching/match_tools.h index 43288cb18c6..56bf5d19f3a 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_tools.h +++ b/searchcore/src/vespa/searchcore/proton/matching/match_tools.h @@ -119,7 +119,8 @@ public: const search::fef::IIndexEnvironment &indexEnv, const search::fef::RankSetup &rankSetup, const search::fef::Properties &rankProperties, - const search::fef::Properties &featureOverrides); + const search::fef::Properties &featureOverrides, + bool is_search); ~MatchToolsFactory(); bool valid() const { return _valid; } const MaybeMatchPhaseLimiter &match_limiter() const { return *_match_limiter; } diff --git a/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp b/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp index 98c4fdaa89a..e459a45040b 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp @@ -136,7 +136,7 @@ using search::fef::indexproperties::softtimeout::Factor; std::unique_ptr<MatchToolsFactory> Matcher::create_match_tools_factory(const search::engine::Request &request, ISearchContext &searchContext, IAttributeContext &attrContext, const search::IDocumentMetaStore &metaStore, - const Properties &feature_overrides) const + const Properties &feature_overrides, bool is_search) const { const Properties & rankProperties = request.propertiesMap.rankProperties(); bool softTimeoutEnabled = Enabled::lookup(rankProperties, _rankSetup->getSoftTimeoutEnabled()); @@ -156,7 +156,7 @@ Matcher::create_match_tools_factory(const search::engine::Request &request, ISea return std::make_unique<MatchToolsFactory>(_queryLimiter, doom, searchContext, attrContext, request.trace(), request.getStackRef(), request.location, _viewResolver, metaStore, _indexEnv, *_rankSetup, - rankProperties, feature_overrides); + rankProperties, feature_overrides, is_search); } size_t @@ -216,7 +216,7 @@ Matcher::match(const SearchRequest &request, vespalib::ThreadBundle &threadBundl } MatchToolsFactory::UP mtf = create_match_tools_factory(request, searchContext, attrContext, - metaStore, *feature_overrides); + metaStore, *feature_overrides, true); isDoomExplicit = mtf->getRequestContext().getDoom().isExplicitSoftDoom(); traceQuery(6, request.trace(), mtf->query()); if (!mtf->valid()) { @@ -368,7 +368,7 @@ Matcher::create_docsum_matcher(const DocsumRequest &req, ISearchContext &search_ } StupidMetaStore meta; MatchToolsFactory::UP mtf = create_match_tools_factory(req, search_ctx, attr_ctx, meta, - req.propertiesMap.featureOverrides()); + req.propertiesMap.featureOverrides(), false); if (!mtf->valid()) { LOG(warning, "could not initialize docsum matching: %s", (expectedSessionCached) ? "session has expired" : "invalid query"); diff --git a/searchcore/src/vespa/searchcore/proton/matching/matcher.h b/searchcore/src/vespa/searchcore/proton/matching/matcher.h index 1a04940b2f4..dcd1bbb2b46 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/matcher.h +++ b/searchcore/src/vespa/searchcore/proton/matching/matcher.h @@ -107,7 +107,7 @@ public: std::unique_ptr<MatchToolsFactory> create_match_tools_factory(const search::engine::Request &request, ISearchContext &searchContext, IAttributeContext &attrContext, const search::IDocumentMetaStore &metaStore, - const Properties &feature_overrides) const; + const Properties &feature_overrides, bool is_search) const; /** * Perform a search against this matcher. diff --git a/searchcore/src/vespa/searchcore/proton/matching/query.cpp b/searchcore/src/vespa/searchcore/proton/matching/query.cpp index d8597bd133c..07024950779 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/query.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/query.cpp @@ -60,25 +60,23 @@ inject(Node::UP query, Node::UP to_inject) { return query; } -std::vector<LocationTerm *> -find_location_terms(Node *tree) { - std::vector<LocationTerm *> retval; - std::vector<Node *> nodes; - nodes.push_back(tree); - // Note the nodes vector being iterated over is appended in the loop - for (size_t i = 0; i < nodes.size(); ++i) { - Node * node = nodes[i]; - if (node->isLocationTerm() ) { - retval.push_back(static_cast<LocationTerm *>(node)); - } - if (node->isIntermediate()) { - auto parent = static_cast<const search::query::Intermediate *>(node); - for (Node * child : parent->getChildren()) { - nodes.push_back(child); - } +void +find_location_terms(Node *node, std::vector<LocationTerm *> & locations) { + if (node->isLocationTerm() ) { + locations.push_back(static_cast<LocationTerm *>(node)); + } else if (node->isIntermediate()) { + auto parent = static_cast<const search::query::Intermediate *>(node); + for (Node * child : parent->getChildren()) { + find_location_terms(child, locations); } } - return retval; +} + +std::vector<LocationTerm *> +find_location_terms(Node *tree) { + std::vector<LocationTerm *> locations; + find_location_terms(tree, locations); + return locations; } GeoLocationSpec parse_location_string(string str) { @@ -139,11 +137,9 @@ void exchange_location_nodes(const string &location_str, IntermediateBlueprint * asRankOrAndNot(Blueprint * blueprint) { - IntermediateBlueprint * rankOrAndNot = dynamic_cast<RankBlueprint*>(blueprint); - if (rankOrAndNot == nullptr) { - rankOrAndNot = dynamic_cast<AndNotBlueprint*>(blueprint); - } - return rankOrAndNot; + return ((blueprint->isAndNot() || blueprint->isRank())) + ? static_cast<IntermediateBlueprint *>(blueprint) + : nullptr; } IntermediateBlueprint * diff --git a/searchcore/src/vespa/searchcore/proton/matching/querynodes.cpp b/searchcore/src/vespa/searchcore/proton/matching/querynodes.cpp index bb8a669f91a..5f70d1f638b 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/querynodes.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/querynodes.cpp @@ -54,7 +54,7 @@ ProtonTermData::resolve(const ViewResolver &resolver, for (size_t i = 0; i < fields.size(); ++i) { const FieldInfo *info = idxEnv.getFieldByName(fields[i]); if (info != 0) { - _fields.push_back(FieldEntry(fields[i], info->id())); + _fields.emplace_back(fields[i], info->id()); _fields.back().attribute_field = (info->type() == FieldType::ATTRIBUTE) || (info->type() == FieldType::HIDDEN_ATTRIBUTE); @@ -79,7 +79,7 @@ ProtonTermData::resolveFromChildren(const std::vector<Node *> &subterms) if (lookupField(subSpec.getFieldId()) == 0) { // this must happen before handles are reserved LOG_ASSERT(subSpec.getHandle() == search::fef::IllegalHandle); - _fields.push_back(FieldEntry(subSpec.getName(), subSpec.getFieldId())); + _fields.emplace_back(subSpec.getName(), subSpec.getFieldId()); } } } diff --git a/searchcore/src/vespa/searchcore/proton/server/bucketmovejobv2.cpp b/searchcore/src/vespa/searchcore/proton/server/bucketmovejobv2.cpp index 1c1a77475fc..24a4d6d5dee 100644 --- a/searchcore/src/vespa/searchcore/proton/server/bucketmovejobv2.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/bucketmovejobv2.cpp @@ -25,7 +25,6 @@ LOG_SETUP(".proton.server.bucketmovejob"); using document::BucketId; using storage::spi::BucketInfo; using storage::spi::Bucket; -using storage::spi::makeBucketTask; using proton::bucketdb::BucketMover; using vespalib::makeLambdaTask; using vespalib::Trinary; diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp index 103805a41cd..1beb0a04de0 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp @@ -109,8 +109,7 @@ diskMemUsageSamplerConfig(const ProtonConfig &proton, const HwInfo &hwInfo) } size_t -derive_shared_threads(const ProtonConfig &proton, - const HwInfo::Cpu &cpuInfo) { +derive_shared_threads(const ProtonConfig &proton, const HwInfo::Cpu &cpuInfo) { size_t scaledCores = (size_t)std::ceil(cpuInfo.cores() * proton.feeding.concurrency); // We need at least 1 guaranteed free worker in order to ensure progress so #documentsdbs + 1 should suffice, @@ -118,6 +117,16 @@ derive_shared_threads(const ProtonConfig &proton, return std::max(scaledCores, proton.documentdb.size() + proton.flush.maxconcurrent + 1); } +uint32_t +computeRpcTransportThreads(const ProtonConfig & cfg, const HwInfo::Cpu &cpuInfo) { + bool areSearchAndDocsumAsync = cfg.docsum.async && cfg.search.async; + return (cfg.rpc.transportthreads > 0) + ? cfg.rpc.transportthreads + : areSearchAndDocsumAsync + ? cpuInfo.cores()/8 + : cpuInfo.cores(); +} + struct MetricsUpdateHook : metrics::UpdateHook { Proton &self; @@ -281,9 +290,10 @@ Proton::init(const BootstrapConfig::SP & configSnapshot) _fileHeaderContext.setClusterName(protonConfig.clustername, protonConfig.basedir); _matchEngine = std::make_unique<MatchEngine>(protonConfig.numsearcherthreads, protonConfig.numthreadspersearch, - protonConfig.distributionkey); + protonConfig.distributionkey, + protonConfig.search.async); _distributionKey = protonConfig.distributionkey; - _summaryEngine= std::make_unique<SummaryEngine>(protonConfig.numsummarythreads); + _summaryEngine= std::make_unique<SummaryEngine>(protonConfig.numsummarythreads, protonConfig.docsum.async); _docsumBySlime = std::make_unique<DocsumBySlime>(*_summaryEngine); IFlushStrategy::SP strategy; @@ -335,6 +345,7 @@ Proton::init(const BootstrapConfig::SP & configSnapshot) _prepareRestartHandler = std::make_unique<PrepareRestartHandler>(*_flushEngine); RPCHooks::Params rpcParams(*this, protonConfig.rpcport, _configUri.getConfigId(), + std::max(2u, computeRpcTransportThreads(protonConfig, hwInfo.cpu())), std::max(2u, hwInfo.cpu().cores()/4)); rpcParams.slobrok_config = _configUri.createWithNewId(protonConfig.slobrokconfigid); _rpcHooks = std::make_unique<RPCHooks>(rpcParams); diff --git a/searchcore/src/vespa/searchcore/proton/server/rpc_hooks.cpp b/searchcore/src/vespa/searchcore/proton/server/rpc_hooks.cpp index bc8cc7c55c9..62109eda98d 100644 --- a/searchcore/src/vespa/searchcore/proton/server/rpc_hooks.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/rpc_hooks.cpp @@ -177,12 +177,14 @@ RPCHooksBase::initRPC() } -RPCHooksBase::Params::Params(Proton &parent, uint32_t port, const vespalib::string &ident, uint32_t numThreads) +RPCHooksBase::Params::Params(Proton &parent, uint32_t port, const vespalib::string &ident, + uint32_t transportThreads, uint32_t executorThreads) : proton(parent), slobrok_config(config::ConfigUri("admin/slobrok.0")), identity(ident), rtcPort(port), - numRpcThreads(numThreads) + numTranportThreads(transportThreads), + numDocsumRpcThreads(executorThreads) { } RPCHooksBase::Params::~Params() = default; @@ -190,7 +192,7 @@ RPCHooksBase::Params::~Params() = default; RPCHooksBase::RPCHooksBase(Params ¶ms) : _proton(params.proton), _docsumByRPC(std::make_unique<DocsumByRPC>(_proton.getDocsumBySlime())), - _transport(std::make_unique<FNET_Transport>(2)), + _transport(std::make_unique<FNET_Transport>(params.numTranportThreads)), _orb(std::make_unique<FRT_Supervisor>(_transport.get())), _proto_rpc_adapter(std::make_unique<ProtoRpcAdapter>( _proton.get_search_server(), @@ -199,7 +201,7 @@ RPCHooksBase::RPCHooksBase(Params ¶ms) _regAPI(*_orb, slobrok::ConfiguratorFactory(params.slobrok_config)), _stateLock(), _stateCond(), - _executor(params.numRpcThreads, 128_Ki, proton_rpc_executor) + _executor(params.numDocsumRpcThreads, 128_Ki, proton_rpc_executor) { } void diff --git a/searchcore/src/vespa/searchcore/proton/server/rpc_hooks.h b/searchcore/src/vespa/searchcore/proton/server/rpc_hooks.h index 8e3679d8fa5..4bba2752867 100644 --- a/searchcore/src/vespa/searchcore/proton/server/rpc_hooks.h +++ b/searchcore/src/vespa/searchcore/proton/server/rpc_hooks.h @@ -86,9 +86,13 @@ public: config::ConfigUri slobrok_config; vespalib::string identity; uint32_t rtcPort; - uint32_t numRpcThreads; + uint32_t numTranportThreads; + // TODO: This can be eliminated and reduced to a fixed low number once old rpc has been removed from the qrs. + // Or even use the shared executor + uint32_t numDocsumRpcThreads; - Params(Proton &parent, uint32_t port, const vespalib::string &ident, uint32_t numRpcThreads); + Params(Proton &parent, uint32_t port, const vespalib::string &ident, + uint32_t numTransportThreads, uint32_t numDocsumRpcThreads); ~Params(); }; RPCHooksBase(const RPCHooksBase &) = delete; @@ -96,7 +100,7 @@ public: RPCHooksBase(Params ¶ms); auto &proto_rpc_adapter_metrics() { return _proto_rpc_adapter->metrics(); } void set_online(); - virtual ~RPCHooksBase(); + ~RPCHooksBase() override; void close(); void rpc_GetState(FRT_RPCRequest *req); diff --git a/searchcore/src/vespa/searchcore/proton/server/searchcontext.h b/searchcore/src/vespa/searchcore/proton/server/searchcontext.h index b9ea6b334b3..b21eef46733 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchcontext.h +++ b/searchcore/src/vespa/searchcore/proton/server/searchcontext.h @@ -12,7 +12,7 @@ namespace proton { * the documenttype. First create, search and rank, then group/sort, * collect hits. */ -class SearchContext : public matching::ISearchContext +class SearchContext final : public matching::ISearchContext { private: /// Snapshot of the indexes used. diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp index 945702d07bc..552d2b84179 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp @@ -783,9 +783,7 @@ StoreOnlyFeedView::heartBeat(SerialNum serialNum) { assert(_writeService.master().isCurrentThread()); _metaStore.removeAllOldGenerations(); - if (serialNum > _metaStore.getLastSerialNum()) { - _metaStore.commit(CommitParam(serialNum)); - } + _metaStore.commit(CommitParam(serialNum)); heartBeatSummary(serialNum); heartBeatIndexedFields(serialNum); heartBeatAttributes(serialNum); diff --git a/searchcore/src/vespa/searchcore/proton/summaryengine/summaryengine.cpp b/searchcore/src/vespa/searchcore/proton/summaryengine/summaryengine.cpp index 934e6445bd5..1867fb06bf4 100644 --- a/searchcore/src/vespa/searchcore/proton/summaryengine/summaryengine.cpp +++ b/searchcore/src/vespa/searchcore/proton/summaryengine/summaryengine.cpp @@ -59,8 +59,9 @@ SummaryEngine::DocsumMetrics::DocsumMetrics() SummaryEngine::DocsumMetrics::~DocsumMetrics() = default; -SummaryEngine::SummaryEngine(size_t numThreads) +SummaryEngine::SummaryEngine(size_t numThreads, bool async) : _lock(), + _async(async), _closed(false), _handlers(), _executor(numThreads, 128_Ki, summary_engine_executor), @@ -116,9 +117,12 @@ SummaryEngine::getDocsums(DocsumRequest::Source request, DocsumClient & client) return ret; } - auto task =std::make_unique<DocsumTask>(*this, std::move(request), client); - _executor.execute(std::move(task)); - return DocsumReply::UP(); + if (_async) { + auto task = std::make_unique<DocsumTask>(*this, std::move(request), client); + _executor.execute(std::move(task)); + return DocsumReply::UP(); + } + return getDocsums(request.release()); } DocsumReply::UP diff --git a/searchcore/src/vespa/searchcore/proton/summaryengine/summaryengine.h b/searchcore/src/vespa/searchcore/proton/summaryengine/summaryengine.h index c1cb1f91a2a..fa3bcc805a4 100644 --- a/searchcore/src/vespa/searchcore/proton/summaryengine/summaryengine.h +++ b/searchcore/src/vespa/searchcore/proton/summaryengine/summaryengine.h @@ -32,6 +32,7 @@ private: }; std::mutex _lock; + bool _async; bool _closed; HandlerMap<ISearchHandler> _handlers; vespalib::ThreadStackExecutor _executor; @@ -49,7 +50,10 @@ public: * * @param numThreads Number of threads allocated for handling summary requests. */ - SummaryEngine(size_t numThreads); + SummaryEngine(size_t numThreads, bool async); + SummaryEngine(size_t numThreads) + : SummaryEngine(numThreads, true) + { } /** * Frees any allocated resources. This will also stop all internal threads diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/LambdaFunctionNode.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/LambdaFunctionNode.java index 2a6e6793bcd..bbf1a1a251e 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/LambdaFunctionNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/LambdaFunctionNode.java @@ -112,6 +112,11 @@ public class LambdaFunctionNode extends CompositeNode { if ( ! (node.children().get(0) instanceof ReferenceNode) || ! (node.children().get(1) instanceof ReferenceNode)) { return Optional.empty(); } + var lhs = (ReferenceNode) node.children().get(0); + var rhs = (ReferenceNode) node.children().get(1); + if (! lhs.getName().equals(arguments.get(0)) || ! rhs.getName().equals(arguments.get(1))) { + return Optional.empty(); + } if (node.operators().size() != 1) { return Optional.empty(); } diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTestCase.java index 5c4840a555d..e1daf8c8fe2 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTestCase.java @@ -237,6 +237,8 @@ public class EvaluationTestCase { // tensor join tester.assertEvaluates("{ {x:0,y:0}:15, {x:1,y:0}:35 }", "join(tensor0, tensor1, f(x,y) (x*y))", "{ {x:0}:3, {x:1}:7 }", "{ {y:0}:5 }"); + tester.assertEvaluates("{ {x:0,y:0}:6, {x:1,y:0}:14 }", "join(tensor0, tensor1, f(x,y) (x+x))", "{ {x:0}:3, {x:1}:7 }", "{ {y:0}:5 }"); + tester.assertEvaluates("{ {x:0}:2, {x:1}:-3 }", "join(tensor0, tensor1, f(x,y) (y-x))", "{ {x:0}:3, {x:1}:7 }", "{ {x:0}:5, {x:1}:4 }"); // -- join composites tester.assertEvaluates("{ }", "tensor0 * tensor0", "{}"); tester.assertEvaluates("{{x:0,y:0,z:0}:0.0}", "( tensor0 * tensor1 ) * ( tensor2 * tensor1 )", diff --git a/searchlib/src/tests/attribute/bitvector/bitvector_test.cpp b/searchlib/src/tests/attribute/bitvector/bitvector_test.cpp index bba266ca06e..d965f555b79 100644 --- a/searchlib/src/tests/attribute/bitvector/bitvector_test.cpp +++ b/searchlib/src/tests/attribute/bitvector/bitvector_test.cpp @@ -474,21 +474,17 @@ BitVectorTest::test(BasicType bt, bool filter) { Config cfg(bt, ct); - AttributePtr v = make(cfg, pref, fastSearch, - enableBitVectors, enableOnlyBitVector, filter); + AttributePtr v = make(cfg, pref, fastSearch, enableBitVectors, enableOnlyBitVector, filter); addDocs(v, 1024); VectorType &tv = as<VectorType>(v); populate(tv, 2, 1023, true); SearchContextPtr sc = getSearch<VectorType>(tv, true); - checkSearch(v, std::move(sc), 2, 1022, 205, !enableBitVectors && !filter, - true); + checkSearch(v, std::move(sc), 2, 1022, 205, !enableBitVectors && !filter, true); sc = getSearch<VectorType>(tv, false); - checkSearch(v, std::move(sc), 2, 1022, 205, !enableOnlyBitVector && - !filter, true); - const search::IDocumentWeightAttribute *dwa = - v->asDocumentWeightAttribute(); - if (dwa != NULL) { + checkSearch(v, std::move(sc), 2, 1022, 205, !enableOnlyBitVector && !filter, true); + const search::IDocumentWeightAttribute *dwa = v->asDocumentWeightAttribute(); + if (dwa != nullptr) { search::IDocumentWeightAttribute::LookupResult lres = dwa->lookup(getSearchStr<VectorType>(), dwa->get_dictionary_snapshot()); typedef search::queryeval::DocumentWeightSearchIterator DWSI; @@ -504,21 +500,16 @@ BitVectorTest::test(BasicType bt, } populate(tv, 2, 973, false); sc = getSearch<VectorType>(tv, true); - checkSearch(v, std::move(sc), 977, 1022, 10, !enableOnlyBitVector && - !filter, true); + checkSearch(v, std::move(sc), 977, 1022, 10, !enableOnlyBitVector &&!filter, true); populate(tv, 2, 973, true); sc = getSearch<VectorType>(tv, true); - checkSearch(v, std::move(sc), 2, 1022, 205, !enableBitVectors && !filter, - true); + checkSearch(v, std::move(sc), 2, 1022, 205, !enableBitVectors && !filter, true); addDocs(v, 15000); sc = getSearch<VectorType>(tv, true); - checkSearch(v, std::move(sc), 2, 1022, 205, !enableOnlyBitVector && - !filter, true); + checkSearch(v, std::move(sc), 2, 1022, 205, !enableOnlyBitVector && !filter, true); populateAll(tv, 10, 15000, true); sc = getSearch<VectorType>(tv, true); - checkSearch(v, std::move(sc), 2, 14999, 14992, - !enableBitVectors && !filter, - false); + checkSearch(v, std::move(sc), 2, 14999, 14992, !enableBitVectors && !filter, false); } diff --git a/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp b/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp index 6eb13ea8f7d..61d5b3795e4 100644 --- a/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp +++ b/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp @@ -164,7 +164,7 @@ private: void testReload(AttributePtr v0, AttributePtr v1, AttributePtr v2, MemAttr::SP mv0, MemAttr::SP mv1, MemAttr::SP mv2, MemAttr::SP emv0, MemAttr::SP emv1, MemAttr::SP emv2, - Config cfg, const vespalib::string &pref, bool fastSearch); + Config cfg, const vespalib::string &pref, bool fastSearch, search::DictionaryConfig dictionary_config); public: template <typename VectorType, typename BufferType> @@ -596,7 +596,8 @@ EnumeratedSaveTest::testReload(AttributePtr v0, MemAttr::SP emv2, Config cfg, const vespalib::string &pref, - bool fastSearch) + bool fastSearch, + search::DictionaryConfig dictionary_config) { // typedef AttributePtr AVP; @@ -611,6 +612,7 @@ EnumeratedSaveTest::testReload(AttributePtr v0, Config check_cfg(cfg); check_cfg.setFastSearch(fastSearch); + check_cfg.set_dictionary_config(dictionary_config); TEST_DO((checkLoad<VectorType, BufferType>(check_cfg, pref + "0", v0))); TEST_DO((checkLoad<VectorType, BufferType>(check_cfg, pref + "1", v1))); TEST_DO((checkLoad<VectorType, BufferType>(check_cfg, pref + "2", v2))); @@ -695,11 +697,27 @@ EnumeratedSaveTest::test(BasicType bt, CollectionType ct, TEST_DO((testReload<VectorType, BufferType>(v0, v1, v2, mv0, mv1, mv2, emv0, emv1, emv2, - cfg, pref, false))); + cfg, pref, false, search::DictionaryConfig(search::DictionaryConfig::Type::BTREE)))); TEST_DO((testReload<VectorType, BufferType>(v0, v1, v2, mv0, mv1, mv2, emv0, emv1, emv2, - cfg, pref, true))); + cfg, pref, true, search::DictionaryConfig(search::DictionaryConfig::Type::BTREE)))); + TEST_DO((testReload<VectorType, BufferType>(v0, v1, v2, + mv0, mv1, mv2, + emv0, emv1, emv2, + cfg, pref, false, search::DictionaryConfig(search::DictionaryConfig::Type::BTREE_AND_HASH)))); + TEST_DO((testReload<VectorType, BufferType>(v0, v1, v2, + mv0, mv1, mv2, + emv0, emv1, emv2, + cfg, pref, true, search::DictionaryConfig(search::DictionaryConfig::Type::BTREE_AND_HASH)))); + TEST_DO((testReload<VectorType, BufferType>(v0, v1, v2, + mv0, mv1, mv2, + emv0, emv1, emv2, + cfg, pref, false, search::DictionaryConfig(search::DictionaryConfig::Type::HASH)))); + TEST_DO((testReload<VectorType, BufferType>(v0, v1, v2, + mv0, mv1, mv2, + emv0, emv1, emv2, + cfg, pref, true, search::DictionaryConfig(search::DictionaryConfig::Type::HASH)))); } TEST_F("Test enumerated save with single value int8", EnumeratedSaveTest) diff --git a/searchlib/src/tests/attribute/enumstore/enumstore_test.cpp b/searchlib/src/tests/attribute/enumstore/enumstore_test.cpp index ca5e6e8a721..0f15c288f63 100644 --- a/searchlib/src/tests/attribute/enumstore/enumstore_test.cpp +++ b/searchlib/src/tests/attribute/enumstore/enumstore_test.cpp @@ -17,46 +17,66 @@ using FloatEnumStore = EnumStoreT<float>; using NumericEnumStore = EnumStoreT<int32_t>; using StringEnumStore = EnumStoreT<const char*>; -struct OrderedDoubleEnumStore { +struct BTreeDoubleEnumStore { using EnumStoreType = DoubleEnumStore; static constexpr Type type = Type::BTREE; }; -struct UnorderedDoubleEnumStore { +struct HybridDoubleEnumStore { using EnumStoreType = DoubleEnumStore; static constexpr Type type = Type::BTREE_AND_HASH; }; -struct OrderedFloatEnumStore { +struct HashDoubleEnumStore { + using EnumStoreType = DoubleEnumStore; + static constexpr Type type = Type::HASH; +}; + +struct BTreeFloatEnumStore { using EnumStoreType = FloatEnumStore; static constexpr Type type = Type::BTREE; }; -struct UnorderedFloatEnumStore { +struct HybridFloatEnumStore { using EnumStoreType = FloatEnumStore; static constexpr Type type = Type::BTREE_AND_HASH; }; -struct OrderedNumericEnumStore { +struct HashFloatEnumStore { + using EnumStoreType = FloatEnumStore; + static constexpr Type type = Type::HASH; +}; + +struct BTreeNumericEnumStore { using EnumStoreType = NumericEnumStore; static constexpr Type type = Type::BTREE; }; -struct UnorderedNumericEnumStore { +struct HybridNumericEnumStore { using EnumStoreType = NumericEnumStore; static constexpr Type type = Type::BTREE_AND_HASH; }; -struct OrderedStringEnumStore { +struct HashNumericEnumStore { + using EnumStoreType = NumericEnumStore; + static constexpr Type type = Type::HASH; +}; + +struct BTreeStringEnumStore { using EnumStoreType = StringEnumStore; static constexpr Type type = Type::BTREE; }; -struct UnorderedStringEnumStore { +struct HybridStringEnumStore { using EnumStoreType = StringEnumStore; static constexpr Type type = Type::BTREE_AND_HASH; }; +struct HashStringEnumStore { + using EnumStoreType = StringEnumStore; + static constexpr Type type = Type::HASH; +}; + using StringVector = std::vector<std::string>; using generation_t = vespalib::GenerationHandler::generation_t; @@ -100,13 +120,13 @@ checkReaders(const StringEnumStore& ses, } } -template <typename EnumStoreTypeAndOrdering> +template <typename EnumStoreTypeAndDictionaryType> class FloatEnumStoreTest : public ::testing::Test { public: - using EnumStoreType = typename EnumStoreTypeAndOrdering::EnumStoreType; + using EnumStoreType = typename EnumStoreTypeAndDictionaryType::EnumStoreType; EnumStoreType es; FloatEnumStoreTest() - : es(false, EnumStoreTypeAndOrdering::type) + : es(false, EnumStoreTypeAndDictionaryType::type) {} }; @@ -116,7 +136,7 @@ public: #pragma GCC diagnostic ignored "-Wsuggest-override" #endif -using FloatEnumStoreTestTypes = ::testing::Types<OrderedFloatEnumStore, OrderedDoubleEnumStore, UnorderedFloatEnumStore, UnorderedDoubleEnumStore>; +using FloatEnumStoreTestTypes = ::testing::Types<BTreeFloatEnumStore, BTreeDoubleEnumStore, HybridFloatEnumStore, HybridDoubleEnumStore, HashFloatEnumStore, HashDoubleEnumStore>; VESPA_GTEST_TYPED_TEST_SUITE(FloatEnumStoreTest, FloatEnumStoreTestTypes); TYPED_TEST(FloatEnumStoreTest, numbers_can_be_inserted_and_retrieved) @@ -180,6 +200,8 @@ void testUniques(const StringEnumStore& ses, const std::vector<std::string>& unique) { auto read_snapshot = ses.get_dictionary().get_read_snapshot(); + read_snapshot->fill(); + read_snapshot->sort(); std::vector<EnumIndex> saved_indexes; read_snapshot->foreach_key([&saved_indexes](EntryRef idx) { saved_indexes.push_back(idx); }); uint32_t i = 0; @@ -442,16 +464,16 @@ LoaderTestValues<StringEnumStore>::load_values(enumstore::EnumeratedLoaderBase& loader.load_unique_values(raw_values.data(), raw_values.size()); } -template <typename EnumStoreTypeAndOrdering> +template <typename EnumStoreTypeAndDictionaryType> class LoaderTest : public ::testing::Test { public: - using EnumStoreType = typename EnumStoreTypeAndOrdering::EnumStoreType; + using EnumStoreType = typename EnumStoreTypeAndDictionaryType::EnumStoreType; using EntryType = typename EnumStoreType::EntryType; EnumStoreType store; using Values = LoaderTestValues<EnumStoreType>; LoaderTest() - : store(true, EnumStoreTypeAndOrdering::type) + : store(true, EnumStoreTypeAndDictionaryType::type) {} void load_values(enumstore::EnumeratedLoaderBase& loader) const { @@ -503,7 +525,7 @@ public: #pragma GCC diagnostic ignored "-Wsuggest-override" #endif -using LoaderTestTypes = ::testing::Types<OrderedNumericEnumStore, OrderedFloatEnumStore, OrderedStringEnumStore, UnorderedNumericEnumStore, UnorderedFloatEnumStore, UnorderedStringEnumStore>; +using LoaderTestTypes = ::testing::Types<BTreeNumericEnumStore, BTreeFloatEnumStore, BTreeStringEnumStore, HybridNumericEnumStore, HybridFloatEnumStore, HybridStringEnumStore, HashNumericEnumStore, HashFloatEnumStore, HashStringEnumStore>; VESPA_GTEST_TYPED_TEST_SUITE(LoaderTest, LoaderTestTypes); TYPED_TEST(LoaderTest, store_is_instantiated_with_enumerated_loader) @@ -557,15 +579,15 @@ TYPED_TEST(LoaderTest, store_is_instantiated_with_non_enumerated_loader) #pragma GCC diagnostic pop -template <typename EnumStoreTypeAndOrdering> +template <typename EnumStoreTypeAndDictionaryType> class EnumStoreDictionaryTest : public ::testing::Test { public: - using EnumStoreType = typename EnumStoreTypeAndOrdering::EnumStoreType; + using EnumStoreType = typename EnumStoreTypeAndDictionaryType::EnumStoreType; using EntryType = typename EnumStoreType::EntryType; EnumStoreType store; EnumStoreDictionaryTest() - : store(true, EnumStoreTypeAndOrdering::type) + : store(true, EnumStoreTypeAndDictionaryType::type) {} // Reuse test values from LoaderTest @@ -578,9 +600,9 @@ public: static EntryRef fake_pidx() { return EntryRef(42); } }; -template <typename EnumStoreTypeAndOrdering> +template <typename EnumStoreTypeAndDictionaryType> void -EnumStoreDictionaryTest<EnumStoreTypeAndOrdering>::update_posting_idx(EnumIndex enum_idx, EntryRef old_posting_idx, EntryRef new_posting_idx) +EnumStoreDictionaryTest<EnumStoreTypeAndDictionaryType>::update_posting_idx(EnumIndex enum_idx, EntryRef old_posting_idx, EntryRef new_posting_idx) { auto& dict = store.get_dictionary(); EntryRef old_posting_idx_check; @@ -588,9 +610,9 @@ EnumStoreDictionaryTest<EnumStoreTypeAndOrdering>::update_posting_idx(EnumIndex EXPECT_EQ(old_posting_idx, old_posting_idx_check); } -template <typename EnumStoreTypeAndOrdering> +template <typename EnumStoreTypeAndDictionaryType> EnumIndex -EnumStoreDictionaryTest<EnumStoreTypeAndOrdering>::insert_value(size_t value_idx) +EnumStoreDictionaryTest<EnumStoreTypeAndDictionaryType>::insert_value(size_t value_idx) { assert(value_idx < values().size()); auto enum_idx = store.insert(values()[value_idx]); @@ -604,7 +626,7 @@ EnumStoreDictionaryTest<EnumStoreTypeAndOrdering>::insert_value(size_t value_idx #pragma GCC diagnostic ignored "-Wsuggest-override" #endif -using EnumStoreDictionaryTestTypes = ::testing::Types<OrderedNumericEnumStore, UnorderedNumericEnumStore>; +using EnumStoreDictionaryTestTypes = ::testing::Types<BTreeNumericEnumStore, HybridNumericEnumStore, HashNumericEnumStore>; VESPA_GTEST_TYPED_TEST_SUITE(EnumStoreDictionaryTest, EnumStoreDictionaryTestTypes); TYPED_TEST(EnumStoreDictionaryTest, find_frozen_index_works) @@ -664,7 +686,7 @@ TYPED_TEST(EnumStoreDictionaryTest, normalize_posting_lists_works) auto find_result = dict.find_posting_list(this->make_bound_comparator(0), root); EXPECT_EQ(value_0_idx, find_result.first); EXPECT_EQ(this->fake_pidx(), find_result.second); - auto dummy = [](EntryRef posting_idx) { return posting_idx; }; + auto dummy = [](EntryRef posting_idx) noexcept { return posting_idx; }; std::vector<EntryRef> saved_refs; auto save_refs_and_clear = [&saved_refs](EntryRef posting_idx) { saved_refs.push_back(posting_idx); return EntryRef(); }; EXPECT_FALSE(dict.normalize_posting_lists(dummy)); diff --git a/searchlib/src/tests/attribute/searchable/attribute_searchable_adapter_test.cpp b/searchlib/src/tests/attribute/searchable/attribute_searchable_adapter_test.cpp index 04b16a73029..29ef4e9e6ef 100644 --- a/searchlib/src/tests/attribute/searchable/attribute_searchable_adapter_test.cpp +++ b/searchlib/src/tests/attribute/searchable/attribute_searchable_adapter_test.cpp @@ -425,11 +425,11 @@ TEST("require that attribute dot product works") { bool fast_search = ((i & 0x1) != 0); bool strict = ((i & 0x2) != 0); MyAttributeManager attribute_manager = make_weighted_string_attribute_manager(fast_search); - SimpleDotProduct node(field, 0, Weight(1)); - node.append(Node::UP(new SimpleStringTerm("foo", "", 0, Weight(1)))); - node.append(Node::UP(new SimpleStringTerm("bar", "", 0, Weight(1)))); - node.append(Node::UP(new SimpleStringTerm("baz", "", 0, Weight(1)))); - node.append(Node::UP(new SimpleStringTerm("fox", "", 0, Weight(1)))); + SimpleDotProduct node(4, field, 0, Weight(1)); + node.addTerm("foo", Weight(1)); + node.addTerm("bar", Weight(1)); + node.addTerm("baz", Weight(1)); + node.addTerm("fox", Weight(1)); Result result = do_search(attribute_manager, node, strict); ASSERT_EQUAL(5u, result.hits.size()); if (fast_search) { @@ -457,11 +457,11 @@ TEST("require that attribute dot product can produce no hits") { bool fast_search = ((i & 0x1) != 0); bool strict = ((i & 0x2) != 0); MyAttributeManager attribute_manager = make_weighted_string_attribute_manager(fast_search); - SimpleDotProduct node(field, 0, Weight(1)); - node.append(Node::UP(new SimpleStringTerm("notfoo", "", 0, Weight(1)))); - node.append(Node::UP(new SimpleStringTerm("notbar", "", 0, Weight(1)))); - node.append(Node::UP(new SimpleStringTerm("notbaz", "", 0, Weight(1)))); - node.append(Node::UP(new SimpleStringTerm("notfox", "", 0, Weight(1)))); + SimpleDotProduct node(4, field, 0, Weight(1)); + node.addTerm("notfoo", Weight(1)); + node.addTerm("notbar", Weight(1)); + node.addTerm("notbaz", Weight(1)); + node.addTerm("notfox", Weight(1)); Result result = do_search(attribute_manager, node, strict); ASSERT_EQUAL(0u, result.hits.size()); EXPECT_EQUAL(0u, result.est_hits); @@ -525,11 +525,11 @@ TEST("require that attribute parallel wand works") { bool fast_search = ((i & 0x1) != 0); bool strict = ((i & 0x2) != 0); MyAttributeManager attribute_manager = make_weighted_string_attribute_manager(fast_search); - SimpleWandTerm node(field, 0, Weight(1), 10, 500, 1.5); - node.append(Node::UP(new SimpleStringTerm("foo", "", 0, Weight(1)))); - node.append(Node::UP(new SimpleStringTerm("bar", "", 0, Weight(1)))); - node.append(Node::UP(new SimpleStringTerm("baz", "", 0, Weight(1)))); - node.append(Node::UP(new SimpleStringTerm("fox", "", 0, Weight(1)))); + SimpleWandTerm node(4, field, 0, Weight(1), 10, 500, 1.5); + node.addTerm("foo", Weight(1)); + node.addTerm("bar", Weight(1)); + node.addTerm("baz", Weight(1)); + node.addTerm("fox", Weight(1)); Result result = do_search(attribute_manager, node, strict); EXPECT_FALSE(result.est_empty); if (fast_search) { @@ -561,11 +561,11 @@ TEST("require that attribute weighted set term works") { bool fast_search = ((i & 0x1) != 0); bool strict = ((i & 0x2) != 0); MyAttributeManager attribute_manager = make_weighted_string_attribute_manager(fast_search); - SimpleWeightedSetTerm node(field, 0, Weight(1)); - node.append(Node::UP(new SimpleStringTerm("foo", "", 0, Weight(10)))); - node.append(Node::UP(new SimpleStringTerm("bar", "", 0, Weight(20)))); - node.append(Node::UP(new SimpleStringTerm("baz", "", 0, Weight(30)))); - node.append(Node::UP(new SimpleStringTerm("fox", "", 0, Weight(40)))); + SimpleWeightedSetTerm node(4, field, 0, Weight(1)); + node.addTerm("foo", Weight(10)); + node.addTerm("bar", Weight(20)); + node.addTerm("baz", Weight(30)); + node.addTerm("fox", Weight(40)); Result result = do_search(attribute_manager, node, strict); EXPECT_FALSE(result.est_empty); ASSERT_EQUAL(5u, result.hits.size()); diff --git a/searchlib/src/tests/attribute/searchable/attribute_weighted_set_blueprint_test.cpp b/searchlib/src/tests/attribute/searchable/attribute_weighted_set_blueprint_test.cpp index fe5014b6607..328cdcf663f 100644 --- a/searchlib/src/tests/attribute/searchable/attribute_weighted_set_blueprint_test.cpp +++ b/searchlib/src/tests/attribute/searchable/attribute_weighted_set_blueprint_test.cpp @@ -89,9 +89,9 @@ struct WS { } Node::UP createNode() const { - SimpleWeightedSetTerm *node = new SimpleWeightedSetTerm("view", 0, Weight(0)); + SimpleWeightedSetTerm *node = new SimpleWeightedSetTerm(tokens.size(), "view", 0, Weight(0)); for (size_t i = 0; i < tokens.size(); ++i) { - node->append(Node::UP(new SimpleStringTerm(tokens[i].first, "view", 0, Weight(tokens[i].second)))); + node->addTerm(tokens[i].first, Weight(tokens[i].second)); } return Node::UP(node); } @@ -138,42 +138,30 @@ struct WS { } // namespace <unnamed> -class Test : public vespalib::TestApp -{ -public: - int Main() override; -}; - -int -Test::Main() -{ - TEST_INIT("attribute_weighted_set_test"); - { - MockAttributeManager manager; - setupAttributeManager(manager); - AttributeBlueprintFactory adapter; - - FakeResult expect = FakeResult() - .doc(3).elem(0).weight(30).pos(0) - .doc(5).elem(0).weight(50).pos(0) - .doc(7).elem(0).weight(70).pos(0); - WS ws = WS(manager).add("7", 70).add("5", 50).add("3", 30); - - EXPECT_TRUE(ws.isGenericSearch(adapter, "integer", true)); - EXPECT_TRUE(!ws.isGenericSearch(adapter, "integer", false)); - EXPECT_TRUE(ws.isGenericSearch(adapter, "string", true)); - EXPECT_TRUE(!ws.isGenericSearch(adapter, "string", false)); - EXPECT_TRUE(ws.isGenericSearch(adapter, "multi", true)); - EXPECT_TRUE(ws.isGenericSearch(adapter, "multi", false)); - - EXPECT_EQUAL(expect, ws.search(adapter, "integer", true)); - EXPECT_EQUAL(expect, ws.search(adapter, "integer", false)); - EXPECT_EQUAL(expect, ws.search(adapter, "string", true)); - EXPECT_EQUAL(expect, ws.search(adapter, "string", false)); - EXPECT_EQUAL(expect, ws.search(adapter, "multi", true)); - EXPECT_EQUAL(expect, ws.search(adapter, "multi", false)); - } - TEST_DONE(); +TEST("attribute_weighted_set_test") { + MockAttributeManager manager; + setupAttributeManager(manager); + AttributeBlueprintFactory adapter; + + FakeResult expect = FakeResult() + .doc(3).elem(0).weight(30).pos(0) + .doc(5).elem(0).weight(50).pos(0) + .doc(7).elem(0).weight(70).pos(0); + WS ws = WS(manager).add("7", 70).add("5", 50).add("3", 30); + + EXPECT_TRUE(ws.isGenericSearch(adapter, "integer", true)); + EXPECT_TRUE(!ws.isGenericSearch(adapter, "integer", false)); + EXPECT_TRUE(ws.isGenericSearch(adapter, "string", true)); + EXPECT_TRUE(!ws.isGenericSearch(adapter, "string", false)); + EXPECT_TRUE(ws.isGenericSearch(adapter, "multi", true)); + EXPECT_TRUE(ws.isGenericSearch(adapter, "multi", false)); + + EXPECT_EQUAL(expect, ws.search(adapter, "integer", true)); + EXPECT_EQUAL(expect, ws.search(adapter, "integer", false)); + EXPECT_EQUAL(expect, ws.search(adapter, "string", true)); + EXPECT_EQUAL(expect, ws.search(adapter, "string", false)); + EXPECT_EQUAL(expect, ws.search(adapter, "multi", true)); + EXPECT_EQUAL(expect, ws.search(adapter, "multi", false)); } -TEST_APPHOOK(Test); +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp index 8efccf9b51b..f1d910f2635 100644 --- a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp +++ b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp @@ -26,7 +26,6 @@ #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/test/insertion_operators.h> #include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/vespalib/util/mmap_file_allocator_factory.h> #include <vespa/searchlib/util/bufferwriter.h> #include <vespa/log/log.h> @@ -257,7 +256,6 @@ struct FixtureTraits { bool use_direct_tensor_attribute = false; bool enable_hnsw_index = false; bool use_mock_index = false; - bool use_mmap_file_allocator = false; FixtureTraits dense() && { use_dense_tensor_attribute = true; @@ -265,11 +263,6 @@ struct FixtureTraits { return *this; } - FixtureTraits mmap_file_allocator() && { - use_mmap_file_allocator = true; - return *this; - } - FixtureTraits hnsw() && { use_dense_tensor_attribute = true; enable_hnsw_index = true; @@ -334,9 +327,6 @@ struct Fixture { if (_cfg.tensorType().is_dense()) { _denseTensors = true; } - if (_traits.use_mmap_file_allocator) { - _cfg.setHuge(true); - } if (_traits.use_mock_index) { _index_factory = std::make_unique<MockNearestNeighborIndexFactory>(); } else { @@ -1008,17 +998,4 @@ TEST_F("NN blueprint handles strong filter triggering brute force search", Neare EXPECT_FALSE(bp->may_approximate()); } -TEST("Dense tensor attribute with huge flag uses mmap file allocator") -{ - vespalib::string basedir("mmap-file-allocator-factory-dir"); - vespalib::alloc::MmapFileAllocatorFactory::instance().setup(basedir); - { - Fixture f(vec_2d_spec, FixtureTraits().dense().mmap_file_allocator()); - vespalib::string allocator_dir(basedir + "/0.my_attr"); - EXPECT_TRUE(vespalib::isDirectory(allocator_dir)); - } - vespalib::alloc::MmapFileAllocatorFactory::instance().setup(""); - vespalib::rmdir(basedir, true); -} - TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/fef/properties/properties_test.cpp b/searchlib/src/tests/fef/properties/properties_test.cpp index b7478da3f71..7b919e4aaa6 100644 --- a/searchlib/src/tests/fef/properties/properties_test.cpp +++ b/searchlib/src/tests/fef/properties/properties_test.cpp @@ -216,15 +216,23 @@ TEST("test stuff") { { // test index properties known by the framework { // vespa.eval.lazy_expressions EXPECT_EQUAL(eval::LazyExpressions::NAME, vespalib::string("vespa.eval.lazy_expressions")); - Properties p; - EXPECT_TRUE(eval::LazyExpressions::check(p, true)); - EXPECT_TRUE(!eval::LazyExpressions::check(p, false)); - p = Properties().add("vespa.eval.lazy_expressions", "true"); - EXPECT_TRUE(eval::LazyExpressions::check(p, true)); - EXPECT_TRUE(eval::LazyExpressions::check(p, false)); - p = Properties().add("vespa.eval.lazy_expressions", "false"); - EXPECT_TRUE(!eval::LazyExpressions::check(p, true)); - EXPECT_TRUE(!eval::LazyExpressions::check(p, false)); + { + Properties p; + EXPECT_TRUE(eval::LazyExpressions::check(p, true)); + EXPECT_TRUE(!eval::LazyExpressions::check(p, false)); + } + { + Properties p; + p.add("vespa.eval.lazy_expressions", "true"); + EXPECT_TRUE(eval::LazyExpressions::check(p, true)); + EXPECT_TRUE(eval::LazyExpressions::check(p, false)); + } + { + Properties p; + p.add("vespa.eval.lazy_expressions", "false"); + EXPECT_TRUE(!eval::LazyExpressions::check(p, true)); + EXPECT_TRUE(!eval::LazyExpressions::check(p, false)); + } } { // vespa.eval.use_fast_forest EXPECT_EQUAL(eval::UseFastForest::NAME, vespalib::string("vespa.eval.use_fast_forest")); diff --git a/searchlib/src/tests/nativerank/nativerank.cpp b/searchlib/src/tests/nativerank/nativerank.cpp index b28e385b597..a929d6b5b23 100644 --- a/searchlib/src/tests/nativerank/nativerank.cpp +++ b/searchlib/src/tests/nativerank/nativerank.cpp @@ -228,7 +228,8 @@ Test::testNativeFieldMatch() EXPECT_TRUE(assertNativeFieldMatch(34, "a%0.1 b%0.4", "x a x x x b")); // change firstOccImportance - Properties p = Properties().add("nativeFieldMatch.firstOccurrenceImportance", "1"); + Properties p; + p.add("nativeFieldMatch.firstOccurrenceImportance", "1"); EXPECT_TRUE(assertNativeFieldMatch(100, "a", "a", p)); p.clear().add("nativeFieldMatch.firstOccurrenceImportance", "0"); EXPECT_TRUE(assertNativeFieldMatch(10, "a", "a", p)); @@ -375,7 +376,8 @@ Test::testNativeAttributeMatch() { // use table normalization // foo: max table value: 255 // bar: max table value: 510 - Properties p = Properties().add("nativeRank.useTableNormalization", "true"); + Properties p; + p.add("nativeRank.useTableNormalization", "true"); EXPECT_TRUE(assertNativeAttributeMatch(0.2941, ANAM(100), ANAM(50), p)); // (100/255 + 100/510)*0.5 EXPECT_TRUE(assertNativeAttributeMatch(1, ANAM(255), ANAM(255), p)); // (255/255 + 510/510)*0.5 p.add("nativeAttributeMatch.weightTable.foo", "linear(0,0)"); @@ -630,7 +632,8 @@ Test::testNativeProximity() EXPECT_TRUE(assertNativeProximity(3.667, "a 0.5:b 1:c", "a b x x c")); // (5*0.5 + 3*1) / (0.5 + 1) // change proximityImportance - Properties p = Properties().add("nativeProximity.proximityImportance", "1"); + Properties p; + p.add("nativeProximity.proximityImportance", "1"); EXPECT_TRUE(assertNativeProximity(10, "a b", "a b x x x a", p)); p.clear().add("nativeProximity.proximityImportance", "0"); EXPECT_TRUE(assertNativeProximity(4, "a b", "a b x x x a", p)); diff --git a/searchlib/src/tests/query/customtypevisitor_test.cpp b/searchlib/src/tests/query/customtypevisitor_test.cpp index 3f7d57b7aa4..d0812a00ebf 100644 --- a/searchlib/src/tests/query/customtypevisitor_test.cpp +++ b/searchlib/src/tests/query/customtypevisitor_test.cpp @@ -15,18 +15,6 @@ using namespace search::query; namespace { -class Test : public vespalib::TestApp { - const char *current_state; - virtual void DumpState(bool) { - fprintf(stderr, "%s: ERROR: in %s\n", GetName(), current_state); - } - - template <class T> void requireThatNodeIsVisited(); - -public: - int Main() override; -}; - template <class Base> struct InitTerm : Base { InitTerm() : Base(typename Base::Type(), "view", 0, Weight(0)) {} @@ -49,9 +37,9 @@ struct MyStringTerm : InitTerm<StringTerm> {}; struct MySubstrTerm : InitTerm<SubstringTerm> {}; struct MySuffixTerm : InitTerm<SuffixTerm> {}; struct MyWeakAnd : WeakAnd { MyWeakAnd() : WeakAnd(1234, "view") {} }; -struct MyWeightedSetTerm : WeightedSetTerm { MyWeightedSetTerm() : WeightedSetTerm("view", 0, Weight(42)) {} }; -struct MyDotProduct : DotProduct { MyDotProduct() : DotProduct("view", 0, Weight(42)) {} }; -struct MyWandTerm : WandTerm { MyWandTerm() : WandTerm("view", 0, Weight(42), 57, 67, 77.7) {} }; +struct MyWeightedSetTerm : WeightedSetTerm { MyWeightedSetTerm() : WeightedSetTerm(0, "view", 0, Weight(42)) {} }; +struct MyDotProduct : DotProduct { MyDotProduct() : DotProduct(0, "view", 0, Weight(42)) {} }; +struct MyWandTerm : WandTerm { MyWandTerm() : WandTerm(0, "view", 0, Weight(42), 57, 67, 77.7) {} }; struct MyPredicateQuery : InitTerm<PredicateQuery> {}; struct MyRegExpTerm : InitTerm<RegExpTerm> {}; struct MyNearestNeighborTerm : NearestNeighborTerm {}; @@ -119,7 +107,7 @@ public: }; template <class T> -void Test::requireThatNodeIsVisited() { +void requireThatNodeIsVisited() { MyCustomVisitor visitor; Node::UP query(new T); visitor.isVisited<T>() = false; @@ -127,37 +115,28 @@ void Test::requireThatNodeIsVisited() { ASSERT_TRUE(visitor.isVisited<T>()); } -#define TEST_CALL(func) \ - current_state = #func; \ - func(); - -int -Test::Main() -{ - TEST_INIT("customtypevisitor_test"); - - TEST_CALL(requireThatNodeIsVisited<MyAnd>); - TEST_CALL(requireThatNodeIsVisited<MyAndNot>); - TEST_CALL(requireThatNodeIsVisited<MyNear>); - TEST_CALL(requireThatNodeIsVisited<MyONear>); - TEST_CALL(requireThatNodeIsVisited<MyOr>); - TEST_CALL(requireThatNodeIsVisited<MyPhrase>); - TEST_CALL(requireThatNodeIsVisited<MySameElement>); - TEST_CALL(requireThatNodeIsVisited<MyRangeTerm>); - TEST_CALL(requireThatNodeIsVisited<MyRank>); - TEST_CALL(requireThatNodeIsVisited<MyNumberTerm>); - TEST_CALL(requireThatNodeIsVisited<MyPrefixTerm>); - TEST_CALL(requireThatNodeIsVisited<MyStringTerm>); - TEST_CALL(requireThatNodeIsVisited<MySubstrTerm>); - TEST_CALL(requireThatNodeIsVisited<MySuffixTerm>); - TEST_CALL(requireThatNodeIsVisited<MyWeightedSetTerm>); - TEST_CALL(requireThatNodeIsVisited<MyDotProduct>); - TEST_CALL(requireThatNodeIsVisited<MyWandTerm>); - TEST_CALL(requireThatNodeIsVisited<MyPredicateQuery>); - TEST_CALL(requireThatNodeIsVisited<MyRegExpTerm>); - - TEST_DONE(); +TEST("customtypevisitor_test") { + + requireThatNodeIsVisited<MyAnd>(); + requireThatNodeIsVisited<MyAndNot>(); + requireThatNodeIsVisited<MyNear>(); + requireThatNodeIsVisited<MyONear>(); + requireThatNodeIsVisited<MyOr>(); + requireThatNodeIsVisited<MyPhrase>(); + requireThatNodeIsVisited<MySameElement>(); + requireThatNodeIsVisited<MyRangeTerm>(); + requireThatNodeIsVisited<MyRank>(); + requireThatNodeIsVisited<MyNumberTerm>(); + requireThatNodeIsVisited<MyPrefixTerm>(); + requireThatNodeIsVisited<MyStringTerm>(); + requireThatNodeIsVisited<MySubstrTerm>(); + requireThatNodeIsVisited<MySuffixTerm>(); + requireThatNodeIsVisited<MyWeightedSetTerm>(); + requireThatNodeIsVisited<MyDotProduct>(); + requireThatNodeIsVisited<MyWandTerm>(); + requireThatNodeIsVisited<MyPredicateQuery>(); + requireThatNodeIsVisited<MyRegExpTerm>(); } } // namespace -TEST_APPHOOK(Test); +TEST_MAIN() { TEST_RUN_ALL(); }
\ No newline at end of file diff --git a/searchlib/src/tests/query/query_visitor_test.cpp b/searchlib/src/tests/query/query_visitor_test.cpp index 946ad17352d..ef255ad6878 100644 --- a/searchlib/src/tests/query/query_visitor_test.cpp +++ b/searchlib/src/tests/query/query_visitor_test.cpp @@ -15,25 +15,6 @@ using namespace search::query; namespace { -class Test : public vespalib::TestApp { - void requireThatAllNodesCanBeVisited(); - - template <class T> void checkVisit(T *node); - -public: - int Main() override; -}; - -int -Test::Main() -{ - TEST_INIT("query_visitor_test"); - - TEST_DO(requireThatAllNodesCanBeVisited()); - - TEST_DONE(); -} - class MyVisitor : public QueryVisitor { public: @@ -69,7 +50,7 @@ public: }; template <class T> -void Test::checkVisit(T *node) { +void checkVisit(T *node) { Node::UP query(node); MyVisitor visitor; visitor.isVisited<T>() = false; @@ -77,7 +58,7 @@ void Test::checkVisit(T *node) { ASSERT_TRUE(visitor.isVisited<T>()); } -void Test::requireThatAllNodesCanBeVisited() { +TEST("requireThatAllNodesCanBeVisited") { checkVisit<And>(new SimpleAnd); checkVisit<AndNot>(new SimpleAndNot); checkVisit<Near>(new SimpleNear(0)); @@ -85,9 +66,9 @@ void Test::requireThatAllNodesCanBeVisited() { checkVisit<Or>(new SimpleOr); checkVisit<Phrase>(new SimplePhrase("field", 0, Weight(42))); checkVisit<SameElement>(new SimpleSameElement("field")); - checkVisit<WeightedSetTerm>(new SimpleWeightedSetTerm("field", 0, Weight(42))); - checkVisit<DotProduct>(new SimpleDotProduct("field", 0, Weight(42))); - checkVisit<WandTerm>(new SimpleWandTerm("field", 0, Weight(42), 57, 67, 77.7)); + checkVisit<WeightedSetTerm>(new SimpleWeightedSetTerm(0, "field", 0, Weight(42))); + checkVisit<DotProduct>(new SimpleDotProduct(0, "field", 0, Weight(42))); + checkVisit<WandTerm>(new SimpleWandTerm(0, "field", 0, Weight(42), 57, 67, 77.7)); checkVisit<Rank>(new SimpleRank); checkVisit<NumberTerm>(new SimpleNumberTerm("0.42", "field", 0, Weight(0))); const Location location(Point{10, 10}, 20, 0); @@ -104,4 +85,4 @@ void Test::requireThatAllNodesCanBeVisited() { } // namespace -TEST_APPHOOK(Test); +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/query/querybuilder_test.cpp b/searchlib/src/tests/query/querybuilder_test.cpp index 30b4d2ae264..06098afe3eb 100644 --- a/searchlib/src/tests/query/querybuilder_test.cpp +++ b/searchlib/src/tests/query/querybuilder_test.cpp @@ -90,18 +90,17 @@ Node::UP createQueryTree() { builder.addStringTerm(str[4], view[4], id[4], weight[4]); builder.addStringTerm(str[5], view[5], id[5], weight[5]); } - builder.addPredicateQuery(getPredicateQueryTerm(), - view[3], id[3], weight[3]); - builder.addDotProduct(3, view[2], id[2], weight[2]); + builder.addPredicateQuery(getPredicateQueryTerm(), view[3], id[3], weight[3]); { - builder.addStringTerm(str[3], view[3], id[3], weight[3]); - builder.addStringTerm(str[4], view[4], id[4], weight[4]); - builder.addStringTerm(str[5], view[5], id[5], weight[5]); + auto & n = builder.addDotProduct(3, view[2], id[2], weight[2]); + n.addTerm(str[3], weight[3]); + n.addTerm(str[4], weight[4]); + n.addTerm(str[5], weight[5]); } - builder.addWandTerm(2, view[0], id[0], weight[0], 57, 67, 77.7); { - builder.addStringTerm(str[1], view[1], id[1], weight[1]); - builder.addStringTerm(str[2], view[2], id[2], weight[2]); + auto & n = builder.addWandTerm(2, view[0], id[0], weight[0], 57, 67, 77.7); + n.addTerm(str[1], weight[1]); + n.addTerm(str[2], weight[2]); } builder.addRegExpTerm(str[5], view[5], id[5], weight[5]); builder.addSameElement(3, view[4]); @@ -246,23 +245,33 @@ void checkQueryTreeTypes(Node *node) { EXPECT_TRUE(checkTerm(predicateQuery, getPredicateQueryTerm(), view[3], id[3], weight[3])); auto* dotProduct = as_node<DotProduct>(and_node->getChildren()[6]); - EXPECT_EQUAL(3u, dotProduct->getChildren().size()); - string_term = as_node<StringTerm>(dotProduct->getChildren()[0]); - EXPECT_TRUE(checkTerm(string_term, str[3], view[3], id[3], weight[3])); - string_term = as_node<StringTerm>(dotProduct->getChildren()[1]); - EXPECT_TRUE(checkTerm(string_term, str[4], view[4], id[4], weight[4])); - string_term = as_node<StringTerm>(dotProduct->getChildren()[2]); - EXPECT_TRUE(checkTerm(string_term, str[5], view[5], id[5], weight[5])); + EXPECT_EQUAL(3u, dotProduct->getNumTerms()); + + { + const auto &w1 = dotProduct->getAsString(0); + EXPECT_EQUAL(w1.first, str[3]); + EXPECT_TRUE(w1.second == weight[3]); + const auto &w2 = dotProduct->getAsString(1); + EXPECT_EQUAL(w2.first, str[4]); + EXPECT_TRUE(w2.second == weight[4]); + const auto &w3 = dotProduct->getAsString(2); + EXPECT_EQUAL(w3.first, str[5]); + EXPECT_TRUE(w3.second == weight[5]); + } auto* wandTerm = as_node<WandTerm>(and_node->getChildren()[7]); EXPECT_EQUAL(57u, wandTerm->getTargetNumHits()); EXPECT_EQUAL(67, wandTerm->getScoreThreshold()); EXPECT_EQUAL(77.7, wandTerm->getThresholdBoostFactor()); - EXPECT_EQUAL(2u, wandTerm->getChildren().size()); - string_term = as_node<StringTerm>(wandTerm->getChildren()[0]); - EXPECT_TRUE(checkTerm(string_term, str[1], view[1], id[1], weight[1])); - string_term = as_node<StringTerm>(wandTerm->getChildren()[1]); - EXPECT_TRUE(checkTerm(string_term, str[2], view[2], id[2], weight[2])); + EXPECT_EQUAL(2u, wandTerm->getNumTerms()); + { + const auto &w1 = wandTerm->getAsString(0); + EXPECT_EQUAL(w1.first, str[1]); + EXPECT_TRUE(w1.second == weight[1]); + const auto &w2 = wandTerm->getAsString(1); + EXPECT_EQUAL(w2.first, str[2]); + EXPECT_TRUE(w2.second == weight[2]); + } auto* regexp_term = as_node<RegExpTerm>(and_node->getChildren()[8]); EXPECT_TRUE(checkTerm(regexp_term, str[5], view[5], id[5], weight[5])); @@ -336,15 +345,15 @@ struct MyPhrase : Phrase { MyPhrase(const string &f, int32_t i, Weight w) : Phra struct MySameElement : SameElement { MySameElement(const string &f) : SameElement(f) {}}; struct MyWeightedSetTerm : WeightedSetTerm { - MyWeightedSetTerm(const string &f, int32_t i, Weight w) : WeightedSetTerm(f, i, w) {} + MyWeightedSetTerm(uint32_t n, const string &f, int32_t i, Weight w) : WeightedSetTerm(n, f, i, w) {} }; struct MyDotProduct : DotProduct { - MyDotProduct(const string &f, int32_t i, Weight w) : DotProduct(f, i, w) {} + MyDotProduct(uint32_t n, const string &f, int32_t i, Weight w) : DotProduct(n, f, i, w) {} }; struct MyWandTerm : WandTerm { - MyWandTerm(const string &f, int32_t i, Weight w, uint32_t targetNumHits, + MyWandTerm(uint32_t n, const string &f, int32_t i, Weight w, uint32_t targetNumHits, int64_t scoreThreshold, double thresholdBoostFactor) - : WandTerm(f, i, w, targetNumHits, scoreThreshold, thresholdBoostFactor) {} + : WandTerm(n, f, i, w, targetNumHits, scoreThreshold, thresholdBoostFactor) {} }; struct MyRank : Rank {}; struct MyNumberTerm : NumberTerm { @@ -592,6 +601,10 @@ TEST("require that empty intermediate node can be added") { EXPECT_EQUAL(0u, and_node->getChildren().size()); } +TEST("control size of SimpleQueryStackDumpIterator") { + EXPECT_EQUAL(128u, sizeof(SimpleQueryStackDumpIterator)); +} + TEST("test query parsing error") { const char * STACK = "\001\002\001\003\000\005\002\004\001\034F\001\002\004term\004\004term\002dx\004\004term\002ifD\002\004term\001xD\003\004term\002dxE\004\004term\001\060F\005\002\004term" @@ -629,6 +642,78 @@ TEST("test query parsing error") { EXPECT_FALSE(new_node); } +class SimpleMultiTerm : public MultiTerm { +public: + SimpleMultiTerm(size_t numTerms) : MultiTerm(numTerms) {} + void accept(QueryVisitor & ) override { } +}; + +TEST("initial state of MultiTerm") { + SimpleMultiTerm mt(7); + EXPECT_EQUAL(7u, mt.getNumTerms()); + EXPECT_TRUE(MultiTerm::Type::UNKNOWN == mt.getType()); +} + +void +verify_multiterm_get(const MultiTerm & mt) { + EXPECT_EQUAL(7u, mt.getNumTerms()); + for (int64_t i(0); i < mt.getNumTerms(); i++) { + auto v = mt.getAsInteger(i); + EXPECT_EQUAL(v.first, i-3); + EXPECT_EQUAL(v.second.percent(), i-4); + } + for (int64_t i(0); i < mt.getNumTerms(); i++) { + auto v = mt.getAsString(i); + char buf[24]; + auto res = std::to_chars(buf, buf + sizeof(buf), i-3); + EXPECT_EQUAL(v.first, vespalib::stringref(buf, res.ptr - buf)); + EXPECT_EQUAL(v.second.percent(), i-4); + } +} + +TEST("add and get of integer MultiTerm") { + SimpleMultiTerm mt(7); + for (int64_t i(0); i < mt.getNumTerms(); i++) { + mt.addTerm(i-3, Weight(i-4)); + } + EXPECT_TRUE(MultiTerm::Type::INTEGER == mt.getType()); + verify_multiterm_get(mt); +} + +TEST("add and get of string MultiTerm") { + SimpleMultiTerm mt(7); + for (int64_t i(0); i < mt.getNumTerms(); i++) { + char buf[24]; + auto res = std::to_chars(buf, buf + sizeof(buf), i-3); + mt.addTerm(vespalib::stringref(buf, res.ptr - buf), Weight(i-4)); + } + EXPECT_TRUE(MultiTerm::Type::STRING == mt.getType()); + verify_multiterm_get(mt); +} + +TEST("first string then integer MultiTerm") { + SimpleMultiTerm mt(7); + mt.addTerm("-3", Weight(-4)); + for (int64_t i(1); i < mt.getNumTerms(); i++) { + mt.addTerm(i-3, Weight(i-4)); + } + EXPECT_TRUE(MultiTerm::Type::STRING == mt.getType()); + verify_multiterm_get(mt); +} + +TEST("first integer then string MultiTerm") { + SimpleMultiTerm mt(7); + mt.addTerm(-3, Weight(-4)); + EXPECT_TRUE(MultiTerm::Type::INTEGER == mt.getType()); + for (int64_t i(1); i < mt.getNumTerms(); i++) { + char buf[24]; + auto res = std::to_chars(buf, buf + sizeof(buf), i-3); + mt.addTerm(vespalib::stringref(buf, res.ptr - buf), Weight(i-4)); + } + EXPECT_TRUE(MultiTerm::Type::STRING == mt.getType()); + verify_multiterm_get(mt); +} + } // namespace TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/query/stackdumpquerycreator_test.cpp b/searchlib/src/tests/query/stackdumpquerycreator_test.cpp index cb3989387b7..c5ef71621d1 100644 --- a/searchlib/src/tests/query/stackdumpquerycreator_test.cpp +++ b/searchlib/src/tests/query/stackdumpquerycreator_test.cpp @@ -48,8 +48,7 @@ TEST("requireThatTooLargeNumTermIsTreatedAsFloat") { appendNumTerm(buf, term_string); SimpleQueryStackDumpIterator query_stack(vespalib::stringref(buf.GetDrainPos(), buf.GetUsedLen())); - Node::UP node = - StackDumpQueryCreator<SimpleQueryNodeTypes>::create(query_stack); + Node::UP node = StackDumpQueryCreator<SimpleQueryNodeTypes>::create(query_stack); ASSERT_TRUE(node.get()); NumberTerm *term = dynamic_cast<NumberTerm *>(node.get()); ASSERT_TRUE(term); diff --git a/searchlib/src/tests/queryeval/blueprint/blueprint_test.cpp b/searchlib/src/tests/queryeval/blueprint/blueprint_test.cpp index 0672e51378e..34ab547b85d 100644 --- a/searchlib/src/tests/queryeval/blueprint/blueprint_test.cpp +++ b/searchlib/src/tests/queryeval/blueprint/blueprint_test.cpp @@ -43,7 +43,7 @@ public: createIntermediateSearch(MultiSearch::Children subSearches, bool strict, MatchData &md) const override { - return SearchIterator::UP(new MySearch("or", std::move(subSearches), &md, strict)); + return std::make_unique<MySearch>("or", std::move(subSearches), &md, strict); } static MyOr& create() { return *(new MyOr()); } @@ -60,7 +60,7 @@ public: createIntermediateSearch(MultiSearch::Children subSearches, bool strict, MatchData &md) const override { - return SearchIterator::UP(new MySearch("or", std::move(subSearches), &md, strict)); + return std::make_unique<MySearch>("or", std::move(subSearches), &md, strict); } static OtherOr& create() { return *(new OtherOr()); } @@ -74,15 +74,15 @@ class MyAnd : public AndBlueprint { private: public: - virtual HitEstimate combine(const std::vector<HitEstimate> &data) const override { + HitEstimate combine(const std::vector<HitEstimate> &data) const override { return min(data); } - virtual FieldSpecBaseList exposeFields() const override { + FieldSpecBaseList exposeFields() const override { return FieldSpecBaseList(); } - virtual bool inheritStrict(size_t i) const override { + bool inheritStrict(size_t i) const override { return (i == 0); } @@ -90,7 +90,7 @@ public: createIntermediateSearch(MultiSearch::Children subSearches, bool strict, MatchData &md) const override { - return SearchIterator::UP(new MySearch("and", std::move(subSearches), &md, strict)); + return std::make_unique<MySearch>("and", std::move(subSearches), &md, strict); } static MyAnd& create() { return *(new MyAnd()); } @@ -107,7 +107,7 @@ public: createIntermediateSearch(MultiSearch::Children subSearches, bool strict, MatchData &md) const override { - return SearchIterator::UP(new MySearch("and", std::move(subSearches), &md, strict)); + return std::make_unique<MySearch>("and", std::move(subSearches), &md, strict); } static OtherAnd& create() { return *(new OtherAnd()); } @@ -122,7 +122,7 @@ public: createIntermediateSearch(MultiSearch::Children subSearches, bool strict, MatchData &md) const override { - return SearchIterator::UP(new MySearch("andnot", std::move(subSearches), &md, strict)); + return std::make_unique<MySearch>("andnot", std::move(subSearches), &md, strict); } static OtherAndNot& create() { return *(new OtherAndNot()); } @@ -641,7 +641,7 @@ Test::testSearchCreation() template<typename T> Blueprint::UP makeNew(T *orig) { - return Blueprint::UP(new T(*orig)); + return std::make_unique<T>(*orig); } void diff --git a/searchlib/src/tests/queryeval/dot_product/dot_product_test.cpp b/searchlib/src/tests/queryeval/dot_product/dot_product_test.cpp index 7414e8b10f2..a05dcc4c6ea 100644 --- a/searchlib/src/tests/queryeval/dot_product/dot_product_test.cpp +++ b/searchlib/src/tests/queryeval/dot_product/dot_product_test.cpp @@ -45,9 +45,9 @@ struct DP { } Node::UP createNode() const { - SimpleDotProduct *node = new SimpleDotProduct("view", 0, Weight(0)); + SimpleDotProduct *node = new SimpleDotProduct(tokens.size(), "view", 0, Weight(0)); for (size_t i = 0; i < tokens.size(); ++i) { - node->append(Node::UP(new SimpleStringTerm(tokens[i].first, "view", 0, Weight(tokens[i].second)))); + node->addTerm(tokens[i].first, Weight(tokens[i].second)); } return Node::UP(node); } diff --git a/searchlib/src/tests/queryeval/fake_searchable/fake_searchable_test.cpp b/searchlib/src/tests/queryeval/fake_searchable/fake_searchable_test.cpp index cbad6de25bb..167fc706f4d 100644 --- a/searchlib/src/tests/queryeval/fake_searchable/fake_searchable_test.cpp +++ b/searchlib/src/tests/queryeval/fake_searchable/fake_searchable_test.cpp @@ -156,9 +156,9 @@ TEST_F(FakeSearchableTest, require_that_weigheted_set_search_works) { source.addResult("fieldfoo", "friend3", FakeResult().doc(5)); - SimpleWeightedSetTerm weightedSet("fieldfoo", 1, w); - weightedSet.append(Node::UP(new SimpleStringTerm("friend1", "fieldfoo", 2, Weight(1)))); - weightedSet.append(Node::UP(new SimpleStringTerm("friend2", "fieldfoo", 3, Weight(2)))); + SimpleWeightedSetTerm weightedSet(2, "fieldfoo", 1, w); + weightedSet.addTerm("friend1", Weight(1)); + weightedSet.addTerm("friend2", Weight(2)); FieldSpecList fields; fields.add(FieldSpec("fieldfoo", 1, 1)); diff --git a/searchlib/src/tests/queryeval/getnodeweight/getnodeweight_test.cpp b/searchlib/src/tests/queryeval/getnodeweight/getnodeweight_test.cpp index 359a7f9c074..24253469dfc 100644 --- a/searchlib/src/tests/queryeval/getnodeweight/getnodeweight_test.cpp +++ b/searchlib/src/tests/queryeval/getnodeweight/getnodeweight_test.cpp @@ -34,9 +34,9 @@ TEST("test variations of getWeight") EXPECT_EQUAL(42, getWeight(SimpleStringTerm("foo", "bar", 1, Weight(42)))); EXPECT_EQUAL(42, getWeight(SimpleSubstringTerm("foo", "bar", 1, Weight(42)))); EXPECT_EQUAL(42, getWeight(SimpleSuffixTerm("foo", "bar", 1, Weight(42)))); - EXPECT_EQUAL(42, getWeight(SimpleWeightedSetTerm("bar", 1, Weight(42)))); - EXPECT_EQUAL(42, getWeight(SimpleDotProduct("bar", 1, Weight(42)))); - EXPECT_EQUAL(42, getWeight(SimpleWandTerm("bar", 1, Weight(42), 57, 67, 77.7))); + EXPECT_EQUAL(42, getWeight(SimpleWeightedSetTerm(0, "bar", 1, Weight(42)))); + EXPECT_EQUAL(42, getWeight(SimpleDotProduct(0, "bar", 1, Weight(42)))); + EXPECT_EQUAL(42, getWeight(SimpleWandTerm(0, "bar", 1, Weight(42), 57, 67, 77.7))); } TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/queryeval/parallel_weak_and/parallel_weak_and_test.cpp b/searchlib/src/tests/queryeval/parallel_weak_and/parallel_weak_and_test.cpp index f2c02d02080..b820a96fab6 100644 --- a/searchlib/src/tests/queryeval/parallel_weak_and/parallel_weak_and_test.cpp +++ b/searchlib/src/tests/queryeval/parallel_weak_and/parallel_weak_and_test.cpp @@ -158,11 +158,10 @@ struct WandBlueprintSpec Node::UP createNode(uint32_t scoresToTrack = 100, score_t scoreThreshold = 0, double thresholdBoostFactor = 1) const { - SimpleWandTerm *node = new SimpleWandTerm("view", 0, Weight(0), + SimpleWandTerm *node = new SimpleWandTerm(tokens.size(), "view", 0, Weight(0), scoresToTrack, scoreThreshold, thresholdBoostFactor); for (size_t i = 0; i < tokens.size(); ++i) { - node->append(Node::UP(new SimpleStringTerm(tokens[i].first, "view", 0, - Weight(tokens[i].second)))); + node->addTerm(tokens[i].first, Weight(tokens[i].second)); } return Node::UP(node); } diff --git a/searchlib/src/tests/queryeval/weighted_set_term/weighted_set_term_test.cpp b/searchlib/src/tests/queryeval/weighted_set_term/weighted_set_term_test.cpp index 8514a221230..95553f68cbc 100644 --- a/searchlib/src/tests/queryeval/weighted_set_term/weighted_set_term_test.cpp +++ b/searchlib/src/tests/queryeval/weighted_set_term/weighted_set_term_test.cpp @@ -53,9 +53,9 @@ struct WS { } Node::UP createNode() const { - SimpleWeightedSetTerm *node = new SimpleWeightedSetTerm("view", 0, Weight(0)); + SimpleWeightedSetTerm *node = new SimpleWeightedSetTerm(tokens.size(), "view", 0, Weight(0)); for (size_t i = 0; i < tokens.size(); ++i) { - node->append(Node::UP(new SimpleStringTerm(tokens[i].first, "view", 0, Weight(tokens[i].second)))); + node->addTerm(tokens[i].first,Weight(tokens[i].second)); } return Node::UP(node); } diff --git a/searchlib/src/tests/tensor/dense_tensor_store/dense_tensor_store_test.cpp b/searchlib/src/tests/tensor/dense_tensor_store/dense_tensor_store_test.cpp index 75e231a815c..eb8d89edc07 100644 --- a/searchlib/src/tests/tensor/dense_tensor_store/dense_tensor_store_test.cpp +++ b/searchlib/src/tests/tensor/dense_tensor_store/dense_tensor_store_test.cpp @@ -1,8 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/log/log.h> -LOG_SETUP("dense_tensor_store_test"); #include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/vespalib/util/memory_allocator.h> #include <vespa/searchlib/tensor/dense_tensor_store.h> #include <vespa/eval/eval/simple_value.h> #include <vespa/eval/eval/tensor_spec.h> @@ -10,6 +7,9 @@ LOG_SETUP("dense_tensor_store_test"); #include <vespa/eval/eval/value_type.h> #include <vespa/eval/eval/test/value_compare.h> +#include <vespa/log/log.h> +LOG_SETUP("dense_tensor_store_test"); + using search::tensor::DenseTensorStore; using vespalib::eval::SimpleValue; using vespalib::eval::TensorSpec; @@ -28,7 +28,7 @@ struct Fixture { DenseTensorStore store; Fixture(const vespalib::string &tensorType) - : store(ValueType::from_spec(tensorType), {}) + : store(ValueType::from_spec(tensorType)) {} void assertSetAndGetTensor(const TensorSpec &tensorSpec) { Value::UP expTensor = makeTensor(tensorSpec); diff --git a/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp b/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp index 5ba38d803c8..449d5ac24ca 100644 --- a/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp @@ -37,6 +37,7 @@ #include <vespa/vespalib/util/regexp.h> #include <vespa/vespalib/util/stringfmt.h> #include <sstream> +#include <charconv> #include <vespa/log/log.h> LOG_SETUP(".searchlib.attribute.attribute_blueprint_factory"); @@ -58,6 +59,7 @@ using search::query::StackDumpCreator; using search::query::StringTerm; using search::query::SubstringTerm; using search::query::SuffixTerm; +using search::query::MultiTerm; using search::queryeval::AndBlueprint; using search::queryeval::AndSearchStrict; using search::queryeval::Blueprint; @@ -78,13 +80,30 @@ using search::queryeval::Searchable; using search::queryeval::SimpleLeafBlueprint; using search::queryeval::WeightedSetTermBlueprint; using search::tensor::DenseTensorAttribute; +using search::tensor::ITensorAttribute; using vespalib::geo::ZCurve; using vespalib::make_string; using vespalib::string; +using vespalib::stringref; namespace search { namespace { +class NodeAsKey final : public IDocumentWeightAttribute::LookupKey { +public: + NodeAsKey(const Node & node, vespalib::string & scratchPad) + : _node(node), + _scratchPad(scratchPad) + { } + + stringref asString() const override { + return queryeval::termAsString(_node, _scratchPad); + } + +private: + const Node & _node; + vespalib::string & _scratchPad; +}; //----------------------------------------------------------------------------- /** @@ -312,6 +331,24 @@ make_location_blueprint(const FieldSpec &field, const IAttributeVector &attribut return root; } +class LookupKey : public IDocumentWeightAttribute::LookupKey { +public: + LookupKey(MultiTerm & terms, uint32_t index) : _terms(terms), _index(index) {} + + stringref asString() const override { + return _terms.getAsString(_index).first; + } + + bool asInteger(int64_t &value) const override { + value = _terms.getAsInteger(_index).first; + return true; + } + +private: + const MultiTerm & _terms; + uint32_t _index; +}; + //----------------------------------------------------------------------------- template <typename SearchType> @@ -342,8 +379,8 @@ public: _terms.reserve(size_hint); } - void addTerm(const vespalib::string &term, int32_t weight) { - IDocumentWeightAttribute::LookupResult result = _attr.lookup(term, _dictionary_snapshot); + void addTerm(const IDocumentWeightAttribute::LookupKey & key, int32_t weight) { + IDocumentWeightAttribute::LookupResult result = _attr.lookup(key, _dictionary_snapshot); HitEstimate childEst(result.posting_size, (result.posting_size == 0)); if (!childEst.empty) { if (_estimate.empty) { @@ -422,14 +459,13 @@ public: _terms(), _attr(attr), _dictionary_snapshot(_attr.get_dictionary_snapshot()) - { _weights.reserve(size_hint); _terms.reserve(size_hint); } - void addTerm(const vespalib::string &term, int32_t weight) { - IDocumentWeightAttribute::LookupResult result = _attr.lookup(term, _dictionary_snapshot); + void addTerm(const IDocumentWeightAttribute::LookupKey & key, int32_t weight) { + IDocumentWeightAttribute::LookupResult result = _attr.lookup(key, _dictionary_snapshot); HitEstimate childEst(result.posting_size, (result.posting_size == 0)); if (!childEst.empty) { if (_estimate.empty) { @@ -487,13 +523,14 @@ private: public: DirectAttributeBlueprint(const FieldSpec &field, const vespalib::string & name, const IAttributeVector &iattr, - const IDocumentWeightAttribute &attr, const vespalib::string &term) + const IDocumentWeightAttribute &attr, + const IDocumentWeightAttribute::LookupKey & key) : SimpleLeafBlueprint(field), _attrName(name), _iattr(iattr), _attr(attr), _dictionary_snapshot(_attr.get_dictionary_snapshot()), - _dict_entry(_attr.lookup(term, _dictionary_snapshot)) + _dict_entry(_attr.lookup(key, _dictionary_snapshot)) { setEstimate(HitEstimate(_dict_entry.posting_size, (_dict_entry.posting_size == 0))); } @@ -547,6 +584,7 @@ private: const FieldSpec &_field; const IAttributeVector &_attr; const IDocumentWeightAttribute *_dwa; + vespalib::string _scratchPad; public: CreateBlueprintVisitor(Searchable &searchable, const IRequestContext &requestContext, @@ -554,15 +592,17 @@ public: : CreateBlueprintVisitorHelper(searchable, field, requestContext), _field(field), _attr(attr), - _dwa(attr.asDocumentWeightAttribute()) + _dwa(attr.asDocumentWeightAttribute()), + _scratchPad() { } + ~CreateBlueprintVisitor() override; template <class TermNode> void visitTerm(TermNode &n, bool simple = false) { if (simple && (_dwa != nullptr) && !_field.isFilter() && n.isRanked()) { - vespalib::string term = queryeval::termAsString(n); - setResult(std::make_unique<DirectAttributeBlueprint>(_field, _attr.getName(), _attr, *_dwa, term)); + NodeAsKey key(n, _scratchPad); + setResult(std::make_unique<DirectAttributeBlueprint>(_field, _attr.getName(), _attr, *_dwa, key)); } else { const string stack = StackDumpCreator::create(n); setResult(std::make_unique<AttributeFieldBlueprint>(_field, _attr, stack)); @@ -620,54 +660,34 @@ public: void visit(PredicateQuery &n) override { visitPredicate(n); } void visit(RegExpTerm & n) override { visitTerm(n); } - template <typename WS, typename NODE> - void createDirectWeightedSet(WS *bp, NODE &n) { - Blueprint::UP result(bp); - for (size_t i = 0; i < n.getChildren().size(); ++i) { - const query::Node &node = *n.getChildren()[i]; - vespalib::string term = queryeval::termAsString(node); - uint32_t weight = queryeval::getWeightFromNode(node).percent(); - bp->addTerm(term, weight); - } - setResult(std::move(result)); - } + template <typename WS> + void createDirectWeightedSet(WS *bp, MultiTerm &n); + + template <typename WS> + void createShallowWeightedSet(WS *bp, MultiTerm &n, const FieldSpec &fs, bool isInteger); static QueryTermSimple::UP - extractTerm(const query::Node &node, bool isInteger) { - vespalib::string term = queryeval::termAsString(node); + extractTerm(vespalib::stringref term, bool isInteger) { if (isInteger) { return std::make_unique<QueryTermSimple>(term, QueryTermSimple::Type::WORD); } return std::make_unique<QueryTermUCS4>(term, QueryTermSimple::Type::WORD); } - template <typename WS, typename NODE> - void createShallowWeightedSet(WS *bp, NODE &n, const FieldSpec &fs, bool isInteger) { - Blueprint::UP result(bp); - for (size_t i = 0; i < n.getChildren().size(); ++i) { - const query::Node &node = *n.getChildren()[i]; - uint32_t weight = queryeval::getWeightFromNode(node).percent(); - FieldSpec childfs = bp->getNextChildField(fs); - bp->addTerm(std::make_unique<AttributeFieldBlueprint>(childfs, _attr, extractTerm(node, isInteger)), weight); - } - setResult(std::move(result)); - } - void visit(query::WeightedSetTerm &n) override { bool isSingleValue = !_attr.hasMultiValue(); bool isString = (_attr.isStringType() && _attr.hasEnum()); bool isInteger = _attr.isIntegerType(); if (isSingleValue && (isString || isInteger)) { auto ws = std::make_unique<AttributeWeightedSetBlueprint>(_field, _attr); - for (size_t i = 0; i < n.getChildren().size(); ++i) { - const query::Node &node = *n.getChildren()[i]; - uint32_t weight = queryeval::getWeightFromNode(node).percent(); - ws->addToken(_attr.createSearchContext(extractTerm(node, isInteger), attribute::SearchContextParams()), weight); + for (size_t i = 0; i < n.getNumTerms(); ++i) { + auto term = n.getAsString(i); + ws->addToken(_attr.createSearchContext(extractTerm(term.first, isInteger), attribute::SearchContextParams()), term.second.percent()); } setResult(std::move(ws)); } else { if (_dwa != nullptr) { - auto *bp = new DirectWeightedSetBlueprint<queryeval::WeightedSetTermSearch>(_field, _attr, *_dwa, n.getChildren().size()); + auto *bp = new DirectWeightedSetBlueprint<queryeval::WeightedSetTermSearch>(_field, _attr, *_dwa, n.getNumTerms()); createDirectWeightedSet(bp, n); } else { auto *bp = new WeightedSetTermBlueprint(_field); @@ -678,7 +698,7 @@ public: void visit(query::DotProduct &n) override { if (_dwa != nullptr) { - auto *bp = new DirectWeightedSetBlueprint<queryeval::DotProductSearch>(_field, _attr, *_dwa, n.getChildren().size()); + auto *bp = new DirectWeightedSetBlueprint<queryeval::DotProductSearch>(_field, _attr, *_dwa, n.getNumTerms()); createDirectWeightedSet(bp, n); } else { auto *bp = new DotProductBlueprint(_field); @@ -690,7 +710,7 @@ public: if (_dwa != nullptr) { auto *bp = new DirectWandBlueprint(_field, *_dwa, n.getTargetNumHits(), n.getScoreThreshold(), n.getThresholdBoostFactor(), - n.getChildren().size()); + n.getNumTerms()); createDirectWeightedSet(bp, n); } else { auto *bp = new ParallelWeakAndBlueprint(_field, @@ -706,17 +726,14 @@ public: setResult(std::make_unique<queryeval::EmptyBlueprint>(_field)); } void visit(query::NearestNeighborTerm &n) override { - if (_attr.asTensorAttribute() == nullptr) { + const ITensorAttribute *tensor_attr = _attr.asTensorAttribute(); + if (tensor_attr == nullptr) { return fail_nearest_neighbor_term(n, "Attribute is not a tensor"); } - const auto* dense_attr_tensor = dynamic_cast<const DenseTensorAttribute*>(_attr.asTensorAttribute()); - if (dense_attr_tensor == nullptr) { - return fail_nearest_neighbor_term(n, make_string("Attribute is not a dense tensor (type=%s)", - _attr.asTensorAttribute()->getTensorType().to_spec().c_str())); - } - if (dense_attr_tensor->getTensorType().dimensions().size() != 1) { - return fail_nearest_neighbor_term(n, make_string("Attribute tensor type (%s) is not of order 1", - dense_attr_tensor->getTensorType().to_spec().c_str())); + const auto & ta_type = tensor_attr->getTensorType(); + if ((! ta_type.is_dense()) || (ta_type.dimensions().size() != 1)) { + return fail_nearest_neighbor_term(n, make_string("Attribute tensor type (%s) is not a dense tensor of order 1", + ta_type.to_spec().c_str())); } auto query_tensor = getRequestContext().get_query_tensor(n.get_query_tensor_name()); if (query_tensor.get() == nullptr) { @@ -727,11 +744,15 @@ public: return fail_nearest_neighbor_term(n, make_string("Query tensor is not a dense tensor (type=%s)", qt_type.to_spec().c_str())); } - if (!is_compatible_for_nearest_neighbor(dense_attr_tensor->getTensorType(), qt_type)) { + if (!is_compatible_for_nearest_neighbor(ta_type, qt_type)) { return fail_nearest_neighbor_term(n, make_string("Attribute tensor type (%s) and query tensor type (%s) are not compatible", - dense_attr_tensor->getTensorType().to_spec().c_str(), qt_type.to_spec().c_str())); + ta_type.to_spec().c_str(), qt_type.to_spec().c_str())); + } + if (tensor_attr->supports_extract_cells_ref() == false) { + return fail_nearest_neighbor_term(n, make_string("Attribute does not support access to tensor data (type=%s)", + ta_type.to_spec().c_str())); } - setResult(std::make_unique<queryeval::NearestNeighborBlueprint>(_field, *dense_attr_tensor, + setResult(std::make_unique<queryeval::NearestNeighborBlueprint>(_field, *tensor_attr, std::move(query_tensor), n.get_target_num_hits(), n.get_allow_approximate(), @@ -741,6 +762,30 @@ public: } }; +template <typename WS> +void +CreateBlueprintVisitor::createDirectWeightedSet(WS *bp, MultiTerm &n) { + Blueprint::UP result(bp); + for (uint32_t i(0); i < n.getNumTerms(); i++) { + bp->addTerm(LookupKey(n, i), n.weight(i).percent()); + } + setResult(std::move(result)); +} + +template <typename WS> +void +CreateBlueprintVisitor::createShallowWeightedSet(WS *bp, MultiTerm &n, const FieldSpec &fs, bool isInteger) { + Blueprint::UP result(bp); + for (uint32_t i(0); i < n.getNumTerms(); i++) { + FieldSpec childfs = bp->getNextChildField(fs); + auto term = n.getAsString(i); + bp->addTerm(std::make_unique<AttributeFieldBlueprint>(childfs, _attr, extractTerm(term.first, isInteger)), term.second.percent()); + } + setResult(std::move(result)); +} + +CreateBlueprintVisitor::~CreateBlueprintVisitor() = default; + } // namespace //----------------------------------------------------------------------------- diff --git a/searchlib/src/vespa/searchlib/attribute/createsinglestd.cpp b/searchlib/src/vespa/searchlib/attribute/createsinglestd.cpp index bfd03200c0c..291215b886a 100644 --- a/searchlib/src/vespa/searchlib/attribute/createsinglestd.cpp +++ b/searchlib/src/vespa/searchlib/attribute/createsinglestd.cpp @@ -9,7 +9,6 @@ #include "singlenumericattribute.hpp" #include <vespa/eval/eval/fast_value.h> #include <vespa/searchlib/tensor/dense_tensor_attribute.h> -#include <vespa/searchlib/tensor/serialized_tensor_attribute.h> #include <vespa/searchlib/tensor/serialized_fast_value_attribute.h> namespace search { diff --git a/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.cpp b/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.cpp index 5a65d4937e4..8bb4e3fc47c 100644 --- a/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.cpp +++ b/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.cpp @@ -53,8 +53,12 @@ EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::free_unused_values(const IndexSet unused; // find unused enums - for (auto iter = this->_btree_dict.begin(); iter.valid(); ++iter) { - _enumStore.free_value_if_unused(iter.getKey(), unused); + if constexpr (has_btree_dictionary) { + for (auto iter = this->_btree_dict.begin(); iter.valid(); ++iter) { + _enumStore.free_value_if_unused(iter.getKey(), unused); + } + } else { + this->_hash_dict.foreach_key([this, &unused](EntryRef ref) { _enumStore.free_value_if_unused(ref, unused); }); } remove_unused_values(unused, cmp); } @@ -76,12 +80,14 @@ void EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::remove(const EntryComparator &comp, EntryRef ref) { assert(ref.valid()); - auto itr = this->_btree_dict.lowerBound(ref, comp); - assert(itr.valid() && itr.getKey() == ref); - if constexpr (std::is_same_v<BTreeDictionaryT, EnumPostingTree>) { - assert(EntryRef(itr.getData()) == EntryRef()); + if constexpr (has_btree_dictionary) { + auto itr = this->_btree_dict.lowerBound(ref, comp); + assert(itr.valid() && itr.getKey() == ref); + if constexpr (std::is_same_v<BTreeDictionaryT, EnumPostingTree>) { + assert(EntryRef(itr.getData()) == EntryRef()); + } + this->_btree_dict.remove(itr); } - this->_btree_dict.remove(itr); if constexpr (has_hash_dictionary) { auto *result = this->_hash_dict.remove(comp, ref); assert(result != nullptr && result->first.load_relaxed() == ref); @@ -93,12 +99,21 @@ bool EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::find_index(const vespalib::datastore::EntryComparator& cmp, Index& idx) const { - auto itr = this->_btree_dict.find(Index(), cmp); - if (!itr.valid()) { + if constexpr (has_hash_dictionary) { + auto find_result = this->_hash_dict.find(cmp, EntryRef()); + if (find_result != nullptr) { + idx = find_result->first.load_acquire(); + return true; + } return false; + } else { + auto itr = this->_btree_dict.find(Index(), cmp); + if (!itr.valid()) { + return false; + } + idx = itr.getKey(); + return true; } - idx = itr.getKey(); - return true; } template <typename BTreeDictionaryT, typename HashDictionaryT> @@ -113,13 +128,14 @@ EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::find_frozen_index(const return true; } return false; + } else { + auto itr = this->_btree_dict.getFrozenView().find(Index(), cmp); + if (!itr.valid()) { + return false; + } + idx = itr.getKey(); + return true; } - auto itr = this->_btree_dict.getFrozenView().find(Index(), cmp); - if (!itr.valid()) { - return false; - } - idx = itr.getKey(); - return true; } template <typename BTreeDictionaryT, typename HashDictionaryT> @@ -127,10 +143,17 @@ std::vector<IEnumStore::EnumHandle> EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::find_matching_enums(const vespalib::datastore::EntryComparator& cmp) const { std::vector<IEnumStore::EnumHandle> result; - auto itr = this->_btree_dict.getFrozenView().find(Index(), cmp); - while (itr.valid() && !cmp.less(Index(), itr.getKey())) { - result.push_back(itr.getKey().ref()); - ++itr; + if constexpr (has_btree_dictionary) { + auto itr = this->_btree_dict.getFrozenView().find(Index(), cmp); + while (itr.valid() && !cmp.less(Index(), itr.getKey())) { + result.push_back(itr.getKey().ref()); + ++itr; + } + } else { + auto find_result = this->_hash_dict.find(cmp, EntryRef()); + if (find_result != nullptr) { + result.push_back(find_result->first.load_acquire().ref()); + } } return result; } @@ -139,7 +162,11 @@ template <typename BTreeDictionaryT, typename HashDictionaryT> EntryRef EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::get_frozen_root() const { - return this->_btree_dict.getFrozenView().getRoot(); + if constexpr (has_btree_dictionary) { + return this->_btree_dict.getFrozenView().getRoot(); + } else { + return EntryRef(); + } } template <> @@ -154,18 +181,20 @@ std::pair<IEnumStore::Index, EntryRef> EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::find_posting_list(const vespalib::datastore::EntryComparator& cmp, EntryRef root) const { if constexpr (has_hash_dictionary) { + (void) root; auto find_result = this->_hash_dict.find(cmp, EntryRef()); if (find_result != nullptr) { return std::make_pair(find_result->first.load_acquire(), find_result->second.load_acquire()); } return std::make_pair(Index(), EntryRef()); + } else { + typename BTreeDictionaryType::ConstIterator itr(vespalib::btree::BTreeNode::Ref(), this->_btree_dict.getAllocator()); + itr.lower_bound(root, Index(), cmp); + if (itr.valid() && !cmp.less(Index(), itr.getKey())) { + return std::make_pair(itr.getKey(), EntryRef(itr.getData())); + } + return std::make_pair(Index(), EntryRef()); } - typename BTreeDictionaryType::ConstIterator itr(vespalib::btree::BTreeNode::Ref(), this->_btree_dict.getAllocator()); - itr.lower_bound(root, Index(), cmp); - if (itr.valid() && !cmp.less(Index(), itr.getKey())) { - return std::make_pair(itr.getKey(), EntryRef(itr.getData())); - } - return std::make_pair(Index(), EntryRef()); } template <typename BTreeDictionaryT, typename HashDictionaryT> @@ -193,19 +222,23 @@ template <typename BTreeDictionaryT, typename HashDictionaryT> void EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::clear_all_posting_lists(std::function<void(EntryRef)> clearer) { - auto& dict = this->_btree_dict; - auto itr = dict.begin(); - EntryRef prev; - while (itr.valid()) { - EntryRef ref(itr.getData()); - if (ref.ref() != prev.ref()) { - if (ref.valid()) { - clearer(ref); + if constexpr (has_btree_dictionary) { + auto& dict = this->_btree_dict; + auto itr = dict.begin(); + EntryRef prev; + while (itr.valid()) { + EntryRef ref(itr.getData()); + if (ref.ref() != prev.ref()) { + if (ref.valid()) { + clearer(ref); + } + prev = ref; } - prev = ref; + itr.writeData(EntryRef().ref()); + ++itr; } - itr.writeData(EntryRef().ref()); - ++itr; + } else { + this->_hash_dict.normalize_values([&clearer](EntryRef ref) { clearer(ref); return EntryRef(); }); } } @@ -220,17 +253,25 @@ template <typename BTreeDictionaryT, typename HashDictionaryT> void EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::update_posting_list(Index idx, const vespalib::datastore::EntryComparator& cmp, std::function<EntryRef(EntryRef)> updater) { - auto& dict = this->_btree_dict; - auto itr = dict.lowerBound(idx, cmp); - assert(itr.valid() && itr.getKey() == idx); - EntryRef old_posting_idx(itr.getData()); - EntryRef new_posting_idx = updater(old_posting_idx); - dict.thaw(itr); - itr.writeData(new_posting_idx.ref()); - if constexpr (has_hash_dictionary) { + if constexpr (has_btree_dictionary) { + auto& dict = this->_btree_dict; + auto itr = dict.lowerBound(idx, cmp); + assert(itr.valid() && itr.getKey() == idx); + EntryRef old_posting_idx(itr.getData()); + EntryRef new_posting_idx = updater(old_posting_idx); + dict.thaw(itr); + itr.writeData(new_posting_idx.ref()); + if constexpr (has_hash_dictionary) { + auto find_result = this->_hash_dict.find(this->_hash_dict.get_default_comparator(), idx); + assert(find_result != nullptr && find_result->first.load_relaxed() == idx); + assert(find_result->second.load_relaxed() == old_posting_idx); + find_result->second.store_release(new_posting_idx); + } + } else { auto find_result = this->_hash_dict.find(this->_hash_dict.get_default_comparator(), idx); assert(find_result != nullptr && find_result->first.load_relaxed() == idx); - assert(find_result->second.load_relaxed() == old_posting_idx); + EntryRef old_posting_idx = find_result->second.load_relaxed(); + EntryRef new_posting_idx = updater(old_posting_idx); find_result->second.store_release(new_posting_idx); } } @@ -246,24 +287,28 @@ template <typename BTreeDictionaryT, typename HashDictionaryT> bool EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::normalize_posting_lists(std::function<EntryRef(EntryRef)> normalize) { - bool changed = false; - auto& dict = this->_btree_dict; - for (auto itr = dict.begin(); itr.valid(); ++itr) { - EntryRef old_posting_idx(itr.getData()); - EntryRef new_posting_idx = normalize(old_posting_idx); - if (new_posting_idx != old_posting_idx) { - changed = true; - dict.thaw(itr); - itr.writeData(new_posting_idx.ref()); - if constexpr (has_hash_dictionary) { - auto find_result = this->_hash_dict.find(this->_hash_dict.get_default_comparator(), itr.getKey()); - assert(find_result != nullptr && find_result->first.load_relaxed() == itr.getKey()); - assert(find_result->second.load_relaxed() == old_posting_idx); - find_result->second.store_release(new_posting_idx); + if constexpr (has_btree_dictionary) { + bool changed = false; + auto& dict = this->_btree_dict; + for (auto itr = dict.begin(); itr.valid(); ++itr) { + EntryRef old_posting_idx(itr.getData()); + EntryRef new_posting_idx = normalize(old_posting_idx); + if (new_posting_idx != old_posting_idx) { + changed = true; + dict.thaw(itr); + itr.writeData(new_posting_idx.ref()); + if constexpr (has_hash_dictionary) { + auto find_result = this->_hash_dict.find(this->_hash_dict.get_default_comparator(), itr.getKey()); + assert(find_result != nullptr && find_result->first.load_relaxed() == itr.getKey()); + assert(find_result->second.load_relaxed() == old_posting_idx); + find_result->second.store_release(new_posting_idx); + } } } + return changed; + } else { + return this->_hash_dict.normalize_values(normalize); } - return changed; } template <> @@ -273,6 +318,13 @@ EnumStoreDictionary<EnumTree>::get_posting_dictionary() const LOG_ABORT("should not be reached"); } +template <> +const EnumPostingTree & +EnumStoreDictionary<vespalib::datastore::NoBTreeDictionary, vespalib::datastore::ShardedHashMap>::get_posting_dictionary() const +{ + LOG_ABORT("should not be reached"); +} + template <typename BTreeDictionaryT, typename HashDictionaryT> const EnumPostingTree & EnumStoreDictionary<BTreeDictionaryT, HashDictionaryT>::get_posting_dictionary() const @@ -357,6 +409,8 @@ template class EnumStoreDictionary<EnumPostingTree>; template class EnumStoreDictionary<EnumPostingTree, vespalib::datastore::ShardedHashMap>; +template class EnumStoreDictionary<vespalib::datastore::NoBTreeDictionary, vespalib::datastore::ShardedHashMap>; + } namespace vespalib::btree { diff --git a/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.h b/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.h index 53ba81b1949..a39ff524618 100644 --- a/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.h +++ b/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.h @@ -25,6 +25,7 @@ private: using ParentUniqueStoreDictionary = vespalib::datastore::UniqueStoreDictionary<BTreeDictionaryT, IEnumStoreDictionary, HashDictionaryT>; using generation_t = IEnumStoreDictionary::generation_t; protected: + using ParentUniqueStoreDictionary::has_btree_dictionary; using ParentUniqueStoreDictionary::has_hash_dictionary; private: IEnumStore& _enumStore; diff --git a/searchlib/src/vespa/searchlib/attribute/enumhintsearchcontext.h b/searchlib/src/vespa/searchlib/attribute/enumhintsearchcontext.h index f649754e0fe..af64f17934d 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumhintsearchcontext.h +++ b/searchlib/src/vespa/searchlib/attribute/enumhintsearchcontext.h @@ -6,7 +6,10 @@ #include "ipostinglistsearchcontext.h" #include <vespa/searchlib/queryeval/searchiterator.h> -namespace vespalib::datastore { class EntryComparator; } +namespace vespalib::datastore { +class EntryComparator; +class IUniqueStoreDictionaryReadSnapshot; +} namespace search::attribute { @@ -17,7 +20,7 @@ namespace search::attribute { class EnumHintSearchContext : public IPostingListSearchContext { - const IEnumStoreDictionary::ReadSnapshot::UP _dict_snapshot; + const std::unique_ptr<vespalib::datastore::IUniqueStoreDictionaryReadSnapshot> _dict_snapshot; uint32_t _uniqueValues; uint32_t _docIdLimit; uint64_t _numValues; // attr.getStatus().getNumValues(); diff --git a/searchlib/src/vespa/searchlib/attribute/enumstore.cpp b/searchlib/src/vespa/searchlib/attribute/enumstore.cpp index 5065a4a6cdc..12dd504bad8 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumstore.cpp +++ b/searchlib/src/vespa/searchlib/attribute/enumstore.cpp @@ -50,6 +50,7 @@ make_enum_store_dictionary(IEnumStore &store, bool has_postings, search::Diction } else { switch (type) { case search::DictionaryConfig::Type::HASH: + return std::make_unique<EnumStoreDictionary<vespalib::datastore::NoBTreeDictionary, vespalib::datastore::ShardedHashMap>>(store, std::move(compare)); case search::DictionaryConfig::Type::BTREE_AND_HASH: return std::make_unique<EnumStoreDictionary<EnumPostingTree, vespalib::datastore::ShardedHashMap>>(store, std::move(compare)); default: diff --git a/searchlib/src/vespa/searchlib/attribute/i_document_weight_attribute.cpp b/searchlib/src/vespa/searchlib/attribute/i_document_weight_attribute.cpp index 12ce1a22209..6ee7bee6762 100644 --- a/searchlib/src/vespa/searchlib/attribute/i_document_weight_attribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/i_document_weight_attribute.cpp @@ -1,3 +1,32 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "i_document_weight_attribute.h" +#include <charconv> + +namespace search { +namespace { +class StringAsKey final : public IDocumentWeightAttribute::LookupKey { +public: + StringAsKey(vespalib::stringref key) + : _key(key) + { } + + vespalib::stringref asString() const override { return _key; } +private: + vespalib::string _key; +}; +} + +bool +IDocumentWeightAttribute::LookupKey::asInteger(int64_t &value) const { + vespalib::stringref str = asString(); + const char *end = str.data() + str.size(); + auto res = std::from_chars(str.data(), end, value); + return res.ptr == end; +} + +IDocumentWeightAttribute::LookupResult +IDocumentWeightAttribute::lookup(vespalib::stringref term, vespalib::datastore::EntryRef dictionary_snapshot) const { + return lookup(StringAsKey(term), dictionary_snapshot); +} +}
\ No newline at end of file diff --git a/searchlib/src/vespa/searchlib/attribute/i_document_weight_attribute.h b/searchlib/src/vespa/searchlib/attribute/i_document_weight_attribute.h index e0cfd446da5..ffca480ce8a 100644 --- a/searchlib/src/vespa/searchlib/attribute/i_document_weight_attribute.h +++ b/searchlib/src/vespa/searchlib/attribute/i_document_weight_attribute.h @@ -3,17 +3,21 @@ #pragma once #include "postinglisttraits.h" - #include <functional> namespace search { -namespace query { class Node; } - using DocumentWeightIterator = attribute::PostingListTraits<int32_t>::const_iterator; struct IDocumentWeightAttribute { + struct LookupKey { + virtual ~LookupKey() = default; + // It is required that the string is zero terminated + virtual vespalib::stringref asString() const = 0; + virtual bool asInteger(int64_t &value) const; + }; + struct LookupResult { const vespalib::datastore::EntryRef posting_idx; const uint32_t posting_size; @@ -25,7 +29,10 @@ struct IDocumentWeightAttribute : posting_idx(posting_idx_in), posting_size(posting_size_in), min_weight(min_weight_in), max_weight(max_weight_in), enum_idx(enum_idx_in) {} }; virtual vespalib::datastore::EntryRef get_dictionary_snapshot() const = 0; - virtual LookupResult lookup(const vespalib::string &term, vespalib::datastore::EntryRef dictionary_snapshot) const = 0; + virtual LookupResult lookup(const LookupKey & key, vespalib::datastore::EntryRef dictionary_snapshot) const = 0; + + // Convenience only use by various tests. + LookupResult lookup(vespalib::stringref term, vespalib::datastore::EntryRef dictionary_snapshot) const; /* * Collect enum indexes (via callback) where folded * (e.g. lowercased) value equals the folded value for enum_idx. @@ -33,7 +40,7 @@ struct IDocumentWeightAttribute virtual void collect_folded(vespalib::datastore::EntryRef enum_idx, vespalib::datastore::EntryRef dictionary_snapshot, const std::function<void(vespalib::datastore::EntryRef)>& callback) const = 0; virtual void create(vespalib::datastore::EntryRef idx, std::vector<DocumentWeightIterator> &dst) const = 0; virtual DocumentWeightIterator create(vespalib::datastore::EntryRef idx) const = 0; - virtual ~IDocumentWeightAttribute() {} + virtual ~IDocumentWeightAttribute() = default; }; } diff --git a/searchlib/src/vespa/searchlib/attribute/multinumericpostattribute.h b/searchlib/src/vespa/searchlib/attribute/multinumericpostattribute.h index c09366cdaea..d8d7e7f902c 100644 --- a/searchlib/src/vespa/searchlib/attribute/multinumericpostattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/multinumericpostattribute.h @@ -36,7 +36,7 @@ private: const MultiValueNumericPostingAttribute &self; DocumentWeightAttributeAdapter(const MultiValueNumericPostingAttribute &self_in) : self(self_in) {} vespalib::datastore::EntryRef get_dictionary_snapshot() const override; - LookupResult lookup(const vespalib::string &term, vespalib::datastore::EntryRef dictionary_snapshot) const override; + LookupResult lookup(const LookupKey & key, vespalib::datastore::EntryRef dictionary_snapshot) const override; void collect_folded(vespalib::datastore::EntryRef enum_idx, vespalib::datastore::EntryRef dictionary_snapshot, const std::function<void(vespalib::datastore::EntryRef)>& callback) const override; void create(vespalib::datastore::EntryRef idx, std::vector<DocumentWeightIterator> &dst) const override; DocumentWeightIterator create(vespalib::datastore::EntryRef idx) const override; diff --git a/searchlib/src/vespa/searchlib/attribute/multinumericpostattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multinumericpostattribute.hpp index f91dac630d3..697fa33a060 100644 --- a/searchlib/src/vespa/searchlib/attribute/multinumericpostattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multinumericpostattribute.hpp @@ -3,6 +3,7 @@ #pragma once #include "multinumericpostattribute.h" +#include <charconv> namespace search { @@ -92,21 +93,21 @@ MultiValueNumericPostingAttribute<B, M>::DocumentWeightAttributeAdapter::get_dic template <typename B, typename M> IDocumentWeightAttribute::LookupResult -MultiValueNumericPostingAttribute<B, M>::DocumentWeightAttributeAdapter::lookup(const vespalib::string &term, vespalib::datastore::EntryRef dictionary_snapshot) const +MultiValueNumericPostingAttribute<B, M>::DocumentWeightAttributeAdapter::lookup(const LookupKey & key, vespalib::datastore::EntryRef dictionary_snapshot) const { const IEnumStoreDictionary& dictionary = self._enumStore.get_dictionary(); - char *end = nullptr; - int64_t int_term = strtoll(term.c_str(), &end, 10); - if (*end == '\0') { - auto comp = self._enumStore.make_comparator(int_term); - auto find_result = dictionary.find_posting_list(comp, dictionary_snapshot); - if (find_result.first.valid()) { - auto pidx = find_result.second; - if (pidx.valid()) { - const PostingList &plist = self.getPostingList(); - auto minmax = plist.getAggregated(pidx); - return LookupResult(pidx, plist.frozenSize(pidx), minmax.getMin(), minmax.getMax(), find_result.first); - } + int64_t int_term; + if ( !key.asInteger(int_term)) { + return LookupResult(); + } + auto comp = self._enumStore.make_comparator(int_term); + auto find_result = dictionary.find_posting_list(comp, dictionary_snapshot); + if (find_result.first.valid()) { + auto pidx = find_result.second; + if (pidx.valid()) { + const PostingList &plist = self.getPostingList(); + auto minmax = plist.getAggregated(pidx); + return LookupResult(pidx, plist.frozenSize(pidx), minmax.getMin(), minmax.getMax(), find_result.first); } } return LookupResult(); diff --git a/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.h b/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.h index c324ecdf125..f80d85dadee 100644 --- a/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.h @@ -34,7 +34,7 @@ private: const MultiValueStringPostingAttributeT &self; DocumentWeightAttributeAdapter(const MultiValueStringPostingAttributeT &self_in) : self(self_in) {} vespalib::datastore::EntryRef get_dictionary_snapshot() const override; - LookupResult lookup(const vespalib::string &term, vespalib::datastore::EntryRef dictionary_snapshot) const override; + LookupResult lookup(const LookupKey & key, vespalib::datastore::EntryRef dictionary_snapshot) const override; void collect_folded(vespalib::datastore::EntryRef enum_idx, vespalib::datastore::EntryRef dictionary_snapshot, const std::function<void(vespalib::datastore::EntryRef)>& callback) const override; void create(vespalib::datastore::EntryRef idx, std::vector<DocumentWeightIterator> &dst) const override; DocumentWeightIterator create(vespalib::datastore::EntryRef idx) const override; diff --git a/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.hpp index 7dea95b2e55..d427df69903 100644 --- a/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.hpp @@ -108,10 +108,14 @@ MultiValueStringPostingAttributeT<B, T>::DocumentWeightAttributeAdapter::get_dic template <typename B, typename T> IDocumentWeightAttribute::LookupResult -MultiValueStringPostingAttributeT<B, T>::DocumentWeightAttributeAdapter::lookup(const vespalib::string &term, vespalib::datastore::EntryRef dictionary_snapshot) const +MultiValueStringPostingAttributeT<B, T>::DocumentWeightAttributeAdapter::lookup(const LookupKey & key, vespalib::datastore::EntryRef dictionary_snapshot) const { const IEnumStoreDictionary& dictionary = self._enumStore.get_dictionary(); - auto comp = self._enumStore.make_folded_comparator(term.c_str()); + vespalib::stringref keyAsString = key.asString(); + // Assert the unfortunate assumption of the comparators. + // Should be lifted once they take the length too. + assert(keyAsString.data()[keyAsString.size()] == '\0'); + auto comp = self._enumStore.make_folded_comparator(keyAsString.data()); auto find_result = dictionary.find_posting_list(comp, dictionary_snapshot); if (find_result.first.valid()) { auto pidx = find_result.second; diff --git a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.cpp b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.cpp index 08db7b2b6b7..f270b78f84f 100644 --- a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.cpp +++ b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.cpp @@ -19,7 +19,7 @@ PostingListSearchContext(const IEnumStoreDictionary& dictionary, bool useBitVector, const ISearchContext &baseSearchCtx) : _dictionary(dictionary), - _frozenDictionary(_dictionary.get_posting_dictionary().getFrozenView()), + _frozenDictionary(_dictionary.get_has_btree_dictionary() ? _dictionary.get_posting_dictionary().getFrozenView() : FrozenDictionary()), _lowerDictItr(BTreeNode::Ref(), _frozenDictionary.getAllocator()), _upperDictItr(BTreeNode::Ref(), _frozenDictionary.getAllocator()), _uniqueValues(0u), @@ -57,6 +57,10 @@ void PostingListSearchContext::lookupRange(const vespalib::datastore::EntryComparator &low, const vespalib::datastore::EntryComparator &high) { + if (!_dictionary.get_has_btree_dictionary()) { + _uniqueValues = 2; // Avoid zero and single value optimizations, use filtering + return; + } _lowerDictItr.lower_bound(_frozenDictionary.getRoot(), EnumIndex(), low); _upperDictItr = _lowerDictItr; if (_upperDictItr.valid() && !high.less(EnumIndex(), _upperDictItr.getKey())) { diff --git a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.h b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.h index 3777f3cf4ea..22e9987aa9e 100644 --- a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.h +++ b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.h @@ -80,6 +80,9 @@ protected: } virtual bool fallbackToFiltering() const { + if (_uniqueValues >= 2 && !_dictionary.get_has_btree_dictionary()) { + return true; // force filtering for range search + } uint32_t numHits = calculateApproxNumHits(); // numHits > 1000: make sure that posting lists are unit tested. return (numHits > 1000) && @@ -216,7 +219,7 @@ private: bool fallbackToFiltering() const override { return (this->getRangeLimit() != 0) - ? false + ? (this->_uniqueValues >= 2 && !this->_dictionary.get_has_btree_dictionary()) : Parent::fallbackToFiltering(); } unsigned int approximateHits() const override { @@ -339,6 +342,11 @@ getIterators(bool shouldApplyRangeLimit) auto compHigh = _enumStore.make_comparator(capped.upper()); this->lookupRange(compLow, compHigh); + if (!this->_dictionary.get_has_btree_dictionary()) { + _low = capped.lower(); + _high = capped.upper(); + return; + } if (shouldApplyRangeLimit) { this->applyRangeLimit(this->getRangeLimit()); } diff --git a/searchlib/src/vespa/searchlib/diskindex/diskindex.cpp b/searchlib/src/vespa/searchlib/diskindex/diskindex.cpp index ddcee50c219..0125a4d40b4 100644 --- a/searchlib/src/vespa/searchlib/diskindex/diskindex.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/diskindex.cpp @@ -2,6 +2,8 @@ #include "diskindex.h" #include "disktermblueprint.h" +#include "pagedict4randread.h" +#include "fileheader.h" #include <vespa/searchlib/index/schemautil.h> #include <vespa/searchlib/queryeval/create_blueprint_visitor_helper.h> #include <vespa/searchlib/queryeval/leaf_blueprints.h> @@ -10,8 +12,6 @@ #include <vespa/vespalib/stllike/hash_set.h> #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/vespalib/stllike/cache.hpp> -#include "pagedict4randread.h" -#include "fileheader.h" #include <vespa/log/log.h> LOG_SETUP(".diskindex.diskindex"); diff --git a/searchlib/src/vespa/searchlib/engine/docsumrequest.cpp b/searchlib/src/vespa/searchlib/engine/docsumrequest.cpp index 0fc258363e5..31ee88d3d69 100644 --- a/searchlib/src/vespa/searchlib/engine/docsumrequest.cpp +++ b/searchlib/src/vespa/searchlib/engine/docsumrequest.cpp @@ -16,7 +16,6 @@ DocsumRequest::DocsumRequest(RelativeTime relativeTime, bool useRootSlime_) : Request(std::move(relativeTime)), _flags(0u), resultClassName(), - useWideHits(false), _useRootSlime(useRootSlime_), hits() { diff --git a/searchlib/src/vespa/searchlib/engine/docsumrequest.h b/searchlib/src/vespa/searchlib/engine/docsumrequest.h index 8fe5aa6f465..4849fda8629 100644 --- a/searchlib/src/vespa/searchlib/engine/docsumrequest.h +++ b/searchlib/src/vespa/searchlib/engine/docsumrequest.h @@ -8,15 +8,11 @@ #include <vespa/document/base/globalid.h> #include <vespa/searchlib/common/hitrank.h> -namespace search::fs4transport { class FS4Packet_GETDOCSUMSX; } - namespace search::engine { class DocsumRequest : public Request { public: - using FS4Packet_GETDOCSUMSX = fs4transport::FS4Packet_GETDOCSUMSX; - using UP = std::unique_ptr<DocsumRequest>; using SP = std::shared_ptr<DocsumRequest>; using Source = LazySource<DocsumRequest>; @@ -34,7 +30,6 @@ public: public: uint32_t _flags; vespalib::string resultClassName; - bool useWideHits; private: const bool _useRootSlime; public: diff --git a/searchlib/src/vespa/searchlib/engine/propertiesmap.cpp b/searchlib/src/vespa/searchlib/engine/propertiesmap.cpp index 4726a3b9c27..f5d4e79c0d3 100644 --- a/searchlib/src/vespa/searchlib/engine/propertiesmap.cpp +++ b/searchlib/src/vespa/searchlib/engine/propertiesmap.cpp @@ -11,6 +11,10 @@ PropertiesMap::PropertiesMap() : _propertiesMap() { } +PropertiesMap::PropertiesMap(uint32_t sz) + : _propertiesMap(sz) +{ } + PropertiesMap::~PropertiesMap() = default; fef::Properties & diff --git a/searchlib/src/vespa/searchlib/engine/propertiesmap.h b/searchlib/src/vespa/searchlib/engine/propertiesmap.h index a1884e33200..57d7dbb4d39 100644 --- a/searchlib/src/vespa/searchlib/engine/propertiesmap.h +++ b/searchlib/src/vespa/searchlib/engine/propertiesmap.h @@ -34,6 +34,9 @@ public: typedef PropsMap::const_iterator ITR; PropertiesMap(); + PropertiesMap(uint32_t sz); + PropertiesMap(const PropertiesMap &) = delete; + PropertiesMap & operator=(const PropertiesMap &) = delete; ~PropertiesMap(); /** diff --git a/searchlib/src/vespa/searchlib/engine/proto_converter.cpp b/searchlib/src/vespa/searchlib/engine/proto_converter.cpp index 344e4b72755..f58bcb58949 100644 --- a/searchlib/src/vespa/searchlib/engine/proto_converter.cpp +++ b/searchlib/src/vespa/searchlib/engine/proto_converter.cpp @@ -6,6 +6,9 @@ #include <vespa/vespalib/data/slime/binary_format.h> #include <vespa/vespalib/data/smart_buffer.h> #include <vespa/vespalib/util/size_literals.h> +#include <vespa/log/log.h> + +LOG_SETUP(".searchlib.engine.proto_converter"); namespace search::engine { @@ -91,6 +94,15 @@ ProtoConverter::search_reply_to_proto(const SearchReply &reply, ProtoSearchReply proto.set_degraded_by_soft_timeout(reply.coverage.wasDegradedByTimeout()); bool has_sort_data = (reply.sortIndex.size() > 0); assert(!has_sort_data || (reply.sortIndex.size() == (reply.hits.size() + 1))); + if (reply.request) { + uint32_t asked_offset = reply.request->offset; + uint32_t asked_hits = reply.request->maxhits; + size_t got_hits = reply.hits.size(); + if (got_hits < asked_hits && asked_offset + got_hits < reply.totalHitCount) { + LOG(warning, "asked for %u hits [at offset %u] but only returning %zu hits from %zu available", + asked_hits, asked_offset, got_hits, reply.totalHitCount); + } + } for (size_t i = 0; i < reply.hits.size(); ++i) { auto *hit = proto.add_hits(); hit->set_global_id(reply.hits[i].gid.get(), document::GlobalId::LENGTH); diff --git a/searchlib/src/vespa/searchlib/engine/request.cpp b/searchlib/src/vespa/searchlib/engine/request.cpp index cb00dfcf09b..d47eeb7b25b 100644 --- a/searchlib/src/vespa/searchlib/engine/request.cpp +++ b/searchlib/src/vespa/searchlib/engine/request.cpp @@ -5,13 +5,16 @@ namespace search::engine { Request::Request(RelativeTime relativeTime) + : Request(std::move(relativeTime), 0) +{} + +Request::Request(RelativeTime relativeTime, uint32_t reservePropMaps) : _relativeTime(std::move(relativeTime)), _timeOfDoom(vespalib::steady_time::max()), dumpFeatures(false), ranking(), location(), - propertiesMap(), - stackItems(0), + propertiesMap(reservePropMaps), stackDump(), _trace(_relativeTime, 0) { diff --git a/searchlib/src/vespa/searchlib/engine/request.h b/searchlib/src/vespa/searchlib/engine/request.h index ef90e38dc3d..6375588df46 100644 --- a/searchlib/src/vespa/searchlib/engine/request.h +++ b/searchlib/src/vespa/searchlib/engine/request.h @@ -11,6 +11,7 @@ class Request { public: Request(RelativeTime relativeTime); + Request(RelativeTime relativeTime, uint32_t reservePropMaps); Request(const Request &) = delete; Request & operator =(const Request &) = delete; virtual ~Request(); @@ -44,7 +45,6 @@ public: vespalib::string ranking; vespalib::string location; PropertiesMap propertiesMap; - uint32_t stackItems; std::vector<char> stackDump; private: mutable Trace _trace; diff --git a/searchlib/src/vespa/searchlib/engine/searchrequest.cpp b/searchlib/src/vespa/searchlib/engine/searchrequest.cpp index b9f3e62afd7..fcae7ffd4f3 100644 --- a/searchlib/src/vespa/searchlib/engine/searchrequest.cpp +++ b/searchlib/src/vespa/searchlib/engine/searchrequest.cpp @@ -8,7 +8,7 @@ SearchRequest::SearchRequest() : SearchRequest(RelativeTime(std::make_unique<SteadyClock>())) {} SearchRequest::SearchRequest(RelativeTime relativeTime) - : Request(std::move(relativeTime)), + : Request(std::move(relativeTime), 5), offset(0), maxhits(10), sortSpec(), diff --git a/searchlib/src/vespa/searchlib/fef/properties.cpp b/searchlib/src/vespa/searchlib/fef/properties.cpp index 95b74cd5dfd..3e6a37e44dd 100644 --- a/searchlib/src/vespa/searchlib/fef/properties.cpp +++ b/searchlib/src/vespa/searchlib/fef/properties.cpp @@ -10,44 +10,6 @@ namespace search::fef { const Property::Value Property::_emptyValue; const Property::Values Property::_emptyValues; -Property::Property(const Property::Values &values) - : _values(&values) -{ } - -Property::Property() - : _values(&_emptyValues) -{ } - -bool -Property::found() const -{ - return !(*_values).empty(); -} - -const Property::Value & -Property::get() const -{ - if ((*_values).empty()) { - return _emptyValue; - } - return (*_values)[0]; -} - -const Property::Value & -Property::get(const Property::Value &fallBack) const -{ - if ((*_values).empty()) { - return fallBack; - } - return (*_values)[0]; -} - -uint32_t -Property::size() const -{ - return (*_values).size(); -} - const Property::Value & Property::getAt(uint32_t idx) const { @@ -77,6 +39,9 @@ Properties::Properties() { } +Properties::Properties(const Properties &) = default; +Properties & Properties::operator=(const Properties &) = default; + Properties::~Properties() { assert(_numValues >= _data.size()); diff --git a/searchlib/src/vespa/searchlib/fef/properties.h b/searchlib/src/vespa/searchlib/fef/properties.h index 377926509bd..468e9a90fab 100644 --- a/searchlib/src/vespa/searchlib/fef/properties.h +++ b/searchlib/src/vespa/searchlib/fef/properties.h @@ -37,7 +37,7 @@ private: * * @param values the values for this property **/ - Property(const Values &values); + Property(const Values &values) : _values(&values) { } public: /** @@ -46,14 +46,16 @@ public: * object on the stack in the application, and will also be used * by the @ref Properties class when a lookup gives no results. **/ - Property(); + Property() : _values(&_emptyValues) { } /** * Check if we found what we were looking for or not. * * @return true if the key we looked up had at least one value **/ - bool found() const; + bool found() const { + return !(*_values).empty(); + } /** * Get the first value assigned to the looked up key. This method @@ -61,7 +63,12 @@ public: * * @return first value for the looked up key, or "" **/ - const Value &get() const; + const Value &get() const { + if ((*_values).empty()) { + return _emptyValue; + } + return (*_values)[0]; + } /** * Get the first value assigned to the looked up key. This method @@ -71,14 +78,19 @@ public: * @return first value for the looked up key, or fallBack * @param fallBack value to return if no values were found **/ - const Value & get(const Value &fallBack) const; + const Value & get(const Value &fallBack) const { + if ((*_values).empty()) { + return fallBack; + } + return (*_values)[0]; + } /** * The number of values found for the looked up key. * * @return number of values for this property **/ - uint32_t size() const; + uint32_t size() const { return (*_values).size(); } /** * Obtain a specific value for the looked up key. @@ -153,6 +165,10 @@ public: * Create an empty properties object. **/ Properties(); + Properties(Properties &&) noexcept = default; + Properties & operator=(Properties &&) noexcept = default; + Properties(const Properties &); + Properties & operator=(const Properties &); /** * The destructor asserts that key/value counts look sane before diff --git a/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.cpp b/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.cpp index 9b0196f4f29..0af36d6bb94 100644 --- a/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.cpp +++ b/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.cpp @@ -4,65 +4,50 @@ #include <vespa/vespalib/util/compress.h> #include <vespa/vespalib/objects/nbo.h> #include <cassert> +#include <charconv> using search::query::PredicateQueryTerm; namespace search { -SimpleQueryStackDumpIterator::SimpleQueryStackDumpIterator(vespalib::stringref buf) : - _buf(buf.begin()), - _bufEnd(buf.end()), - _bufLen(buf.size()), - _currPos(_buf), - _currEnd(_buf), - _currType(ParseItem::ITEM_UNDEF), - _currCreator(ParseItem::CREA_ORIG), - _currWeight(100), - _currUniqueId(0), - _currFlags(0), - _currArity(0), - _extraIntArg1(0), - _extraIntArg2(0), - _extraIntArg3(0), - _extraDoubleArg4(0), - _extraDoubleArg5(0), - _predicate_query_term(), - _curr_index_name(), - _curr_term(), - _generatedTerm() +SimpleQueryStackDumpIterator::SimpleQueryStackDumpIterator(vespalib::stringref buf) + : _buf(buf.begin()), + _bufEnd(buf.end()), + _currPos(0), + _currEnd(0), + _currType(ParseItem::ITEM_UNDEF), + _currFlags(0), + _currWeight(100), + _currUniqueId(0), + _currArity(0), + _curr_index_name(), + _curr_term(), + _curr_integer_term(0), + _extraIntArg1(0), + _extraIntArg2(0), + _extraIntArg3(0), + _extraDoubleArg4(0), + _extraDoubleArg5(0), + _predicate_query_term() { } SimpleQueryStackDumpIterator::~SimpleQueryStackDumpIterator() = default; -vespalib::string -SimpleQueryStackDumpIterator::readString(const char *&p) -{ - if (p >= _bufEnd) throw false; - uint64_t tmp; - p += vespalib::compress::Integer::decompressPositive(tmp, p); - vespalib::string s(p, tmp); - p += s.size(); - return s; -} - vespalib::stringref SimpleQueryStackDumpIterator::read_stringref(const char *&p) { - if (p >= _bufEnd) throw false; uint64_t len; p += vespalib::compress::Integer::decompressPositive(len, p); - if (p > _bufEnd) throw false; + if ((p + len) > _bufEnd) throw false; vespalib::stringref result(p, len); p += len; - if (p > _bufEnd) throw false; return result; } uint64_t SimpleQueryStackDumpIterator::readUint64(const char *&p) { - if (p + sizeof(uint64_t) > _bufEnd) throw false; uint64_t l = vespalib::nbo::n2h(*(const uint64_t *)(const void *)p); p += sizeof(uint64_t); return l; @@ -71,17 +56,14 @@ SimpleQueryStackDumpIterator::readUint64(const char *&p) double SimpleQueryStackDumpIterator::read_double(const char *&p) { - if (p >= _bufEnd) throw false; double result = vespalib::nbo::n2h(*reinterpret_cast<const double *>(p)); p += sizeof(double); - if (p > _bufEnd) throw false; return result; } uint64_t SimpleQueryStackDumpIterator::readCompressedPositiveInt(const char *&p) { - if (p >= _bufEnd) throw false; uint64_t tmp; p += vespalib::compress::Integer::decompressPositive(tmp, p); if (p > _bufEnd) throw false; @@ -89,9 +71,16 @@ SimpleQueryStackDumpIterator::readCompressedPositiveInt(const char *&p) } bool -SimpleQueryStackDumpIterator::next() -{ - if (_currEnd >= _bufEnd) +SimpleQueryStackDumpIterator::next() { + try { + return readNext(); + } catch (...) { + return false; + } +} + +bool SimpleQueryStackDumpIterator::readNext() { + if ((_buf + _currEnd) >= _bufEnd) // End of buffer, so no more items available return false; @@ -99,7 +88,7 @@ SimpleQueryStackDumpIterator::next() _currPos = _currEnd; // Find an item at the current position - const char *p = _currPos; + const char *p = _buf + _currPos; uint8_t typefield = *p++; _currType = ParseItem::GetType(typefield); @@ -112,24 +101,17 @@ SimpleQueryStackDumpIterator::next() } else { _currWeight.setPercent(100); } - if (ParseItem::getFeature_UniqueId(typefield)) { - try { - _currUniqueId = readCompressedPositiveInt(p); - } catch (...) { - return false; - } + if (__builtin_expect(ParseItem::getFeature_UniqueId(typefield), false)) { + _currUniqueId = readCompressedPositiveInt(p); } else { _currUniqueId = 0; } - if (ParseItem::getFeature_Flags(typefield)) { - if ((p + sizeof(uint32_t)) > _bufEnd) { - return false; - } + if (__builtin_expect(ParseItem::getFeature_Flags(typefield), false)) { + if ((p + sizeof(uint8_t)) > _bufEnd) return false; _currFlags = (uint8_t)*p++; } else { _currFlags = 0; } - _currCreator = ParseItem::GetCreator(_currFlags); switch (_currType) { case ParseItem::ITEM_OR: @@ -138,73 +120,47 @@ SimpleQueryStackDumpIterator::next() case ParseItem::ITEM_NOT: case ParseItem::ITEM_RANK: case ParseItem::ITEM_ANY: - try { - _currArity = readCompressedPositiveInt(p); - _curr_index_name = vespalib::stringref(); - _curr_term = vespalib::stringref(); - } catch (...) { - return false; - } + _currArity = readCompressedPositiveInt(p); + _curr_index_name = vespalib::stringref(); + _curr_term = vespalib::stringref(); break; case ParseItem::ITEM_NEAR: case ParseItem::ITEM_ONEAR: - try { - _currArity = readCompressedPositiveInt(p); - _extraIntArg1 = readCompressedPositiveInt(p); - _curr_index_name = vespalib::stringref(); - _curr_term = vespalib::stringref(); - } catch (...) { - return false; - } + _currArity = readCompressedPositiveInt(p); + _extraIntArg1 = readCompressedPositiveInt(p); + _curr_index_name = vespalib::stringref(); + _curr_term = vespalib::stringref(); break; case ParseItem::ITEM_WEAK_AND: - try { - _currArity = readCompressedPositiveInt(p); - _extraIntArg1 = readCompressedPositiveInt(p); // targetNumHits - _curr_index_name = read_stringref(p); - _curr_term = vespalib::stringref(); - } catch (...) { - return false; - } + _currArity = readCompressedPositiveInt(p); + _extraIntArg1 = readCompressedPositiveInt(p); // targetNumHits + _curr_index_name = read_stringref(p); + _curr_term = vespalib::stringref(); break; case ParseItem::ITEM_SAME_ELEMENT: - try { - _currArity = readCompressedPositiveInt(p); - _curr_index_name = read_stringref(p); - _curr_term = vespalib::stringref(); - } catch (...) { - return false; - } + _currArity = readCompressedPositiveInt(p); + _curr_index_name = read_stringref(p); + _curr_term = vespalib::stringref(); break; case ParseItem::ITEM_PURE_WEIGHTED_STRING: - try { - _curr_term = read_stringref(p); - _currArity = 0; - } catch (...) { - return false; - } + _curr_term = read_stringref(p); + _currArity = 0; break; case ParseItem::ITEM_PURE_WEIGHTED_LONG: - if (p + sizeof(int64_t) > _bufEnd) return false; - _generatedTerm.clear(); - _generatedTerm << vespalib::nbo::n2h(*reinterpret_cast<const int64_t *>(p)); - _curr_term = vespalib::stringref(_generatedTerm.c_str(), _generatedTerm.size()); - p += sizeof(int64_t); - if (p > _bufEnd) return false; - - _currArity = 0; + { + if (p + sizeof(int64_t) > _bufEnd) return false; + _curr_integer_term = vespalib::nbo::n2h(*reinterpret_cast<const int64_t *>(p)); + p += sizeof(int64_t); + _currArity = 0; + } break; case ParseItem::ITEM_WORD_ALTERNATIVES: - try { - _curr_index_name = read_stringref(p); - _currArity = readCompressedPositiveInt(p); - _curr_term = vespalib::stringref(); - } catch (...) { - return false; - } + _curr_index_name = read_stringref(p); + _currArity = readCompressedPositiveInt(p); + _curr_term = vespalib::stringref(); break; case ParseItem::ITEM_NUMTERM: case ParseItem::ITEM_GEO_LOCATION_TERM: @@ -214,84 +170,84 @@ SimpleQueryStackDumpIterator::next() case ParseItem::ITEM_EXACTSTRINGTERM: case ParseItem::ITEM_SUFFIXTERM: case ParseItem::ITEM_REGEXP: - try { - _curr_index_name = read_stringref(p); - _curr_term = read_stringref(p); - _currArity = 0; - } catch (...) { - return false; - } + _curr_index_name = read_stringref(p); + _curr_term = read_stringref(p); + _currArity = 0; break; case ParseItem::ITEM_PREDICATE_QUERY: - try { - _curr_index_name = read_stringref(p); - _predicate_query_term = std::make_unique<PredicateQueryTerm>(); - - size_t count = readCompressedPositiveInt(p); - for (size_t i = 0; i < count; ++i) { - vespalib::string key = readString(p); - vespalib::string value = readString(p); - uint64_t sub_queries = readUint64(p); - _predicate_query_term->addFeature(key, value, sub_queries); - } - count = readCompressedPositiveInt(p); - for (size_t i = 0; i < count; ++i) { - vespalib::string key = readString(p); - uint64_t value = readUint64(p); - uint64_t sub_queries = readUint64(p); - _predicate_query_term->addRangeFeature( - key, value, sub_queries); - } - if (p > _bufEnd) return false; - } catch (...) { - return false; - } + if ( ! readPredicate(p)) return false; break; case ParseItem::ITEM_WEIGHTED_SET: case ParseItem::ITEM_DOT_PRODUCT: case ParseItem::ITEM_WAND: case ParseItem::ITEM_PHRASE: - try { - _currArity = readCompressedPositiveInt(p); - _curr_index_name = read_stringref(p); - if (_currType == ParseItem::ITEM_WAND) { - _extraIntArg1 = readCompressedPositiveInt(p); // targetNumHits - _extraDoubleArg4 = read_double(p); // scoreThreshold - _extraDoubleArg5 = read_double(p); // thresholdBoostFactor - } - _curr_term = vespalib::stringref(); - } catch (...) { - return false; - } + if (!readComplexTerm(p)) return false; break; - case ParseItem::ITEM_NEAREST_NEIGHBOR: - try { - _curr_index_name = read_stringref(p); - _curr_term = read_stringref(p); // query_tensor_name - _extraIntArg1 = readCompressedPositiveInt(p); // targetNumHits - _extraIntArg2 = readCompressedPositiveInt(p); // allow_approximate - _extraIntArg3 = readCompressedPositiveInt(p); // explore_additional_hits - // XXX: remove later when QRS doesn't send this extra flag - _extraIntArg2 &= ~0x40; - // QRS always sends this now: - _extraDoubleArg4 = read_double(p); // distance threshold - _currArity = 0; - } catch (...) { - return false; - } + if ( ! readNN(p)) return false; break; - default: // Unknown item, so report that no more are available return false; } - _currEnd = p; + _currEnd = p - _buf; // We should not have passed the buffer - assert(_currEnd <= _bufEnd); + return (p <= _bufEnd); +} +bool +SimpleQueryStackDumpIterator::readPredicate(const char *&p) { + _curr_index_name = read_stringref(p); + _predicate_query_term = std::make_unique<PredicateQueryTerm>(); + + size_t count = readCompressedPositiveInt(p); + for (size_t i = 0; i < count; ++i) { + vespalib::stringref key = read_stringref(p); + vespalib::stringref value = read_stringref(p); + if (p + sizeof(uint64_t) > _bufEnd) return false; + uint64_t sub_queries = readUint64(p); + _predicate_query_term->addFeature(key, value, sub_queries); + } + count = readCompressedPositiveInt(p); + for (size_t i = 0; i < count; ++i) { + vespalib::stringref key = read_stringref(p); + if (p + 2*sizeof(uint64_t) > _bufEnd) return false; + uint64_t value = readUint64(p); + uint64_t sub_queries = readUint64(p); + _predicate_query_term->addRangeFeature(key, value, sub_queries); + } + return true; +} + +bool +SimpleQueryStackDumpIterator::readNN(const char *& p) { + _curr_index_name = read_stringref(p); + _curr_term = read_stringref(p); // query_tensor_name + _extraIntArg1 = readCompressedPositiveInt(p); // targetNumHits + _extraIntArg2 = readCompressedPositiveInt(p); // allow_approximate + _extraIntArg3 = readCompressedPositiveInt(p); // explore_additional_hits + // XXX: remove later when QRS doesn't send this extra flag + _extraIntArg2 &= ~0x40; + // QRS always sends this now: + if ((p + sizeof(double))> _bufEnd) return false; + _extraDoubleArg4 = read_double(p); // distance threshold + _currArity = 0; + return true; +} + +bool +SimpleQueryStackDumpIterator::readComplexTerm(const char *& p) { + _currArity = readCompressedPositiveInt(p); + _curr_index_name = read_stringref(p); + if (_currType == ParseItem::ITEM_WAND) { + _extraIntArg1 = readCompressedPositiveInt(p); // targetNumHits + if ((p + 2*sizeof(double))> _bufEnd) return false; + _extraDoubleArg4 = read_double(p); // scoreThreshold + _extraDoubleArg5 = read_double(p); // thresholdBoostFactor + } + _curr_term = vespalib::stringref(); return true; } diff --git a/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.h b/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.h index 301929c8919..8a9a28ebacb 100644 --- a/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.h +++ b/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.h @@ -4,7 +4,6 @@ #include <vespa/searchlib/parsequery/parse.h> #include <vespa/searchlib/query/tree/predicate_query_term.h> -#include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/stllike/string.h> namespace search { @@ -15,67 +14,58 @@ namespace search { class SimpleQueryStackDumpIterator { private: - SimpleQueryStackDumpIterator(const SimpleQueryStackDumpIterator &); - SimpleQueryStackDumpIterator& operator=(const SimpleQueryStackDumpIterator &); - /** Pointer to the start of the input buffer */ const char *_buf; /** Pointer to just past the input buffer */ const char *_bufEnd; - /** Total length of the input buffer */ - size_t _bufLen; - /** Pointer to the position of the current item in the buffer */ - const char *_currPos; + uint32_t _currPos; /** Pointer to after the current item */ - const char *_currEnd; + uint32_t _currEnd; /** The type of the current item */ ParseItem::ItemType _currType; - ParseItem::ItemCreator _currCreator; + /** flags of the current item **/ + uint8_t _currFlags; /** Rank weight of current item **/ query::Weight _currWeight; /** unique id of the current item **/ uint32_t _currUniqueId; - - /** flags of the current item **/ - uint32_t _currFlags; - /** The arity of the current item */ uint32_t _currArity; + /** The index name (field name) in the current item */ + vespalib::stringref _curr_index_name; + /** The term in the current item */ + vespalib::stringref _curr_term; + int64_t _curr_integer_term; /* extra arguments */ uint32_t _extraIntArg1; uint32_t _extraIntArg2; uint32_t _extraIntArg3; - double _extraDoubleArg4; - double _extraDoubleArg5; + double _extraDoubleArg4; + double _extraDoubleArg5; /** The predicate query specification */ query::PredicateQueryTerm::UP _predicate_query_term; - /** The index name (field name) in the current item */ - vespalib::stringref _curr_index_name; - /** The term in the current item */ - vespalib::stringref _curr_term; - vespalib::asciistream _generatedTerm; - - vespalib::string readString(const char *&p); - vespalib::stringref read_stringref(const char *&p); - uint64_t readUint64(const char *&p); - double read_double(const char *&p); - uint64_t readCompressedPositiveInt(const char *&p); + VESPA_DLL_LOCAL vespalib::stringref read_stringref(const char *&p); + VESPA_DLL_LOCAL uint64_t readUint64(const char *&p); + VESPA_DLL_LOCAL double read_double(const char *&p); + VESPA_DLL_LOCAL uint64_t readCompressedPositiveInt(const char *&p); + VESPA_DLL_LOCAL bool readPredicate(const char *&p); + VESPA_DLL_LOCAL bool readNN(const char *&p); + VESPA_DLL_LOCAL bool readComplexTerm(const char *& p); + VESPA_DLL_LOCAL bool readNext(); public: /** - * Make an iterator on a buffer. To get the first item, next - * must be called. - * - * @param buf A pointer to the buffer holding the stackdump - * @param buflen The length of the buffer in bytes + * Make an iterator on a buffer. To get the first item, next must be called. */ SimpleQueryStackDumpIterator(vespalib::stringref buf); + SimpleQueryStackDumpIterator(const SimpleQueryStackDumpIterator &) = delete; + SimpleQueryStackDumpIterator& operator=(const SimpleQueryStackDumpIterator &) = delete; ~SimpleQueryStackDumpIterator(); - vespalib::stringref getStack() const { return vespalib::stringref(_buf, _bufLen); } - size_t getPosition() const { return _currPos - _buf; } + vespalib::stringref getStack() const { return vespalib::stringref(_buf, _bufEnd - _buf); } + size_t getPosition() const { return _currPos; } /** * Moves to the next item in the buffer. @@ -94,7 +84,7 @@ public: * Get the type of the current item. * @return the type. */ - ParseItem::ItemCreator getCreator() const { return _currCreator; } + ParseItem::ItemCreator getCreator() const { return ParseItem::GetCreator(_currFlags); } /** * Get the rank weight of the current item. @@ -125,12 +115,11 @@ public: bool getAllowApproximate() const { return (_extraIntArg2 != 0); } uint32_t getExploreAdditionalHits() const { return _extraIntArg3; } - query::PredicateQueryTerm::UP getPredicateQueryTerm() - { return std::move(_predicate_query_term); } + query::PredicateQueryTerm::UP getPredicateQueryTerm() { return std::move(_predicate_query_term); } vespalib::stringref getIndexName() const { return _curr_index_name; } vespalib::stringref getTerm() const { return _curr_term; } + int64_t getIntergerTerm() const { return _curr_integer_term; } }; } - diff --git a/searchlib/src/vespa/searchlib/query/streaming/querynode.cpp b/searchlib/src/vespa/searchlib/query/streaming/querynode.cpp index ec1b26ec143..cabc9b6dae4 100644 --- a/searchlib/src/vespa/searchlib/query/streaming/querynode.cpp +++ b/searchlib/src/vespa/searchlib/query/streaming/querynode.cpp @@ -2,7 +2,7 @@ #include "query.h" #include <vespa/searchlib/parsequery/stackdumpiterator.h> - +#include <charconv> #include <vespa/log/log.h> LOG_SETUP(".vsm.querynode"); @@ -90,7 +90,6 @@ QueryNode::Build(const QueryNode * parent, const QueryNodeResultFactory & factor if (dynamic_cast<const SameElementQueryNode *>(parent) != nullptr) { index = parent->getIndex() + "." + index; } - vespalib::stringref term = queryRep.getTerm(); using TermType = QueryTerm::Type; TermType sTerm(TermType::WORD); switch (type) { @@ -112,12 +111,19 @@ QueryNode::Build(const QueryNode * parent, const QueryNodeResultFactory & factor default: break; } - QueryTerm::string ssTerm(term); + QueryTerm::string ssTerm; + if (type == ParseItem::ITEM_PURE_WEIGHTED_LONG) { + char buf[24]; + auto res = std::to_chars(buf, buf + sizeof(buf), queryRep.getIntergerTerm(), 10); + ssTerm.assign(buf, res.ptr - buf); + } else { + ssTerm = queryRep.getTerm(); + } QueryTerm::string ssIndex(index); if (ssIndex == "sddocname") { // This is suboptimal as the term should be checked too. // But it will do for now as only correct sddocname queries are sent down. - qn.reset(new TrueNode()); + qn = std::make_unique<TrueNode>(); } else { auto qt = std::make_unique<QueryTerm>(factory.create(), ssTerm, ssIndex, sTerm); qt->setWeight(queryRep.GetWeight()); @@ -131,7 +137,7 @@ QueryNode::Build(const QueryNode * parent, const QueryNodeResultFactory & factor auto orqn = std::make_unique<EquivQueryNode>(); orqn->push_back(std::move(qt)); orqn->push_back(std::move(phrase)); - qn.reset(orqn.release()); + qn = std::move(orqn); } } } diff --git a/searchlib/src/vespa/searchlib/query/tree/intermediatenodes.cpp b/searchlib/src/vespa/searchlib/query/tree/intermediatenodes.cpp index 4a4b606ef8f..ceeacb759b2 100644 --- a/searchlib/src/vespa/searchlib/query/tree/intermediatenodes.cpp +++ b/searchlib/src/vespa/searchlib/query/tree/intermediatenodes.cpp @@ -13,8 +13,5 @@ Near::~Near() = default; ONear::~ONear() = default; Phrase::~Phrase() = default; SameElement::~SameElement() = default; -WeightedSetTerm::~WeightedSetTerm() = default; -DotProduct::~DotProduct() = default; -WandTerm::~WandTerm() = default; } diff --git a/searchlib/src/vespa/searchlib/query/tree/intermediatenodes.h b/searchlib/src/vespa/searchlib/query/tree/intermediatenodes.h index 0ff0b212dfd..06475c0cc63 100644 --- a/searchlib/src/vespa/searchlib/query/tree/intermediatenodes.h +++ b/searchlib/src/vespa/searchlib/query/tree/intermediatenodes.h @@ -122,36 +122,4 @@ private: bool _expensive; }; -class WeightedSetTerm : public QueryNodeMixin<WeightedSetTerm, Intermediate>, public Term { -public: - WeightedSetTerm(const vespalib::string &view, int32_t id, Weight weight) - : Term(view, id, weight) {} - virtual ~WeightedSetTerm() = 0; -}; - -class DotProduct : public QueryNodeMixin<DotProduct, Intermediate>, public Term { -public: - DotProduct(const vespalib::string &view, int32_t id, Weight weight) - : Term(view, id, weight) {} - virtual ~DotProduct() = 0; -}; - -class WandTerm : public QueryNodeMixin<WandTerm, Intermediate>, public Term { -private: - uint32_t _targetNumHits; - int64_t _scoreThreshold; - double _thresholdBoostFactor; -public: - WandTerm(const vespalib::string &view, int32_t id, Weight weight, - uint32_t targetNumHits, int64_t scoreThreshold, double thresholdBoostFactor) - : Term(view, id, weight), - _targetNumHits(targetNumHits), - _scoreThreshold(scoreThreshold), - _thresholdBoostFactor(thresholdBoostFactor) {} - virtual ~WandTerm() = 0; - uint32_t getTargetNumHits() const { return _targetNumHits; } - int64_t getScoreThreshold() const { return _scoreThreshold; } - double getThresholdBoostFactor() const { return _thresholdBoostFactor; } -}; - } diff --git a/searchlib/src/vespa/searchlib/query/tree/predicate_query_term.h b/searchlib/src/vespa/searchlib/query/tree/predicate_query_term.h index 8602eb1ac57..6c78ce51085 100644 --- a/searchlib/src/vespa/searchlib/query/tree/predicate_query_term.h +++ b/searchlib/src/vespa/searchlib/query/tree/predicate_query_term.h @@ -21,12 +21,12 @@ class PredicateQueryTerm { uint64_t _sub_query_bitmap; public: - Entry(const vespalib::string &key, const ValueType &value, + Entry(vespalib::stringref key, const ValueType &value, uint64_t sub_query_bitmap = ALL_SUB_QUERIES) noexcept : _key(key), _value(value), _sub_query_bitmap(sub_query_bitmap) {} - vespalib::string getKey() const { return _key; } - ValueType getValue() const { return _value; } + const vespalib::string & getKey() const { return _key; } + const ValueType & getValue() const { return _value; } uint64_t getSubQueryBitmap() const { return _sub_query_bitmap; } bool operator==(const Entry<ValueType> &other) const { return _key == other._key @@ -43,12 +43,12 @@ public: PredicateQueryTerm() noexcept : _features(), _range_features() {} - void addFeature(const vespalib::string &key, const vespalib::string &value, + void addFeature(vespalib::stringref key, vespalib::stringref value, uint64_t sub_query_bitmask = ALL_SUB_QUERIES) { _features.emplace_back(key, value, sub_query_bitmask); } - void addRangeFeature(const vespalib::string &key, uint64_t value, + void addRangeFeature(vespalib::stringref key, uint64_t value, uint64_t sub_query_bitmask = ALL_SUB_QUERIES) { _range_features.emplace_back(key, value, sub_query_bitmask); } diff --git a/searchlib/src/vespa/searchlib/query/tree/querybuilder.h b/searchlib/src/vespa/searchlib/query/tree/querybuilder.h index 8392730cd29..70cf2382523 100644 --- a/searchlib/src/vespa/searchlib/query/tree/querybuilder.h +++ b/searchlib/src/vespa/searchlib/query/tree/querybuilder.h @@ -50,9 +50,6 @@ class QueryBuilderBase std::stack<NodeInfo> _nodes; vespalib::string _error_msg; - void reportError(const vespalib::string &msg); - void reportError(const vespalib::string &msg, const Node & incomming, const Node & root); - protected: QueryBuilderBase(); ~QueryBuilderBase(); @@ -91,6 +88,9 @@ public: * build a new query tree with the same builder. */ void reset(); + + void reportError(const vespalib::string &msg); + void reportError(const vespalib::string &msg, const Node & incomming, const Node & root); }; @@ -126,17 +126,17 @@ typename NodeTypes::SameElement *createSameElement(vespalib::stringref view) { return new typename NodeTypes::SameElement(view); } template <class NodeTypes> -typename NodeTypes::WeightedSetTerm *createWeightedSetTerm(vespalib::stringref view, int32_t id, Weight weight) { - return new typename NodeTypes::WeightedSetTerm(view, id, weight); +typename NodeTypes::WeightedSetTerm *createWeightedSetTerm(uint32_t num_terms, vespalib::stringref view, int32_t id, Weight weight) { + return new typename NodeTypes::WeightedSetTerm(num_terms, view, id, weight); } template <class NodeTypes> -typename NodeTypes::DotProduct *createDotProduct(vespalib::stringref view, int32_t id, Weight weight) { - return new typename NodeTypes::DotProduct(view, id, weight); +typename NodeTypes::DotProduct *createDotProduct(uint32_t num_terms, vespalib::stringref view, int32_t id, Weight weight) { + return new typename NodeTypes::DotProduct(num_terms, view, id, weight); } template <class NodeTypes> typename NodeTypes::WandTerm * -createWandTerm(vespalib::stringref view, int32_t id, Weight weight, uint32_t targetNumHits, int64_t scoreThreshold, double thresholdBoostFactor) { - return new typename NodeTypes::WandTerm(view, id, weight, targetNumHits, scoreThreshold, thresholdBoostFactor); +createWandTerm(uint32_t num_terms, vespalib::stringref view, int32_t id, Weight weight, uint32_t targetNumHits, int64_t scoreThreshold, double thresholdBoostFactor) { + return new typename NodeTypes::WandTerm(num_terms, view, id, weight, targetNumHits, scoreThreshold, thresholdBoostFactor); } template <class NodeTypes> typename NodeTypes::Rank *createRank() { @@ -262,12 +262,12 @@ public: } typename NodeTypes::WeightedSetTerm &addWeightedSetTerm( int child_count, stringref view, int32_t id, Weight weight) { adjustWeight(weight); - typename NodeTypes::WeightedSetTerm &node = addIntermediate(createWeightedSetTerm<NodeTypes>(view, id, weight), child_count); + typename NodeTypes::WeightedSetTerm &node = addTerm(createWeightedSetTerm<NodeTypes>(child_count, view, id, weight)); return node; } typename NodeTypes::DotProduct &addDotProduct( int child_count, stringref view, int32_t id, Weight weight) { adjustWeight(weight); - typename NodeTypes::DotProduct &node = addIntermediate( createDotProduct<NodeTypes>(view, id, weight), child_count); + typename NodeTypes::DotProduct &node = addTerm( createDotProduct<NodeTypes>(child_count, view, id, weight)); return node; } typename NodeTypes::WandTerm &addWandTerm( @@ -276,9 +276,8 @@ public: int64_t scoreThreshold, double thresholdBoostFactor) { adjustWeight(weight); - typename NodeTypes::WandTerm &node = addIntermediate( - createWandTerm<NodeTypes>(view, id, weight, targetNumHits, scoreThreshold, thresholdBoostFactor), - child_count); + typename NodeTypes::WandTerm &node = addTerm( + createWandTerm<NodeTypes>(child_count, view, id, weight, targetNumHits, scoreThreshold, thresholdBoostFactor)); return node; } typename NodeTypes::Rank &addRank(int child_count) { diff --git a/searchlib/src/vespa/searchlib/query/tree/querynodemixin.h b/searchlib/src/vespa/searchlib/query/tree/querynodemixin.h index 9e8c97cff94..dd398e11844 100644 --- a/searchlib/src/vespa/searchlib/query/tree/querynodemixin.h +++ b/searchlib/src/vespa/searchlib/query/tree/querynodemixin.h @@ -8,7 +8,7 @@ namespace search::query { template <typename T, typename Base> struct QueryNodeMixin : Base { - typedef QueryNodeMixin<T, Base> QueryNodeMixinType; + using QueryNodeMixinType = QueryNodeMixin<T, Base>; ~QueryNodeMixin() = 0; void accept(QueryVisitor &visitor) override { diff --git a/searchlib/src/vespa/searchlib/query/tree/queryreplicator.h b/searchlib/src/vespa/searchlib/query/tree/queryreplicator.h index 4b9226f6112..43364dc8575 100644 --- a/searchlib/src/vespa/searchlib/query/tree/queryreplicator.h +++ b/searchlib/src/vespa/searchlib/query/tree/queryreplicator.h @@ -78,26 +78,42 @@ private: visitNodes(node.getChildren()); } + void replicateMultiTerm(const MultiTerm &original, MultiTerm & replica) { + if (original.getType() == MultiTerm::Type::STRING) { + for (uint32_t i(0); i < original.getNumTerms(); i++) { + auto v = original.getAsString(i); + replica.addTerm(v.first, v.second); + } + } else if (original.getType() == MultiTerm::Type::INTEGER) { + for (uint32_t i(0); i < original.getNumTerms(); i++) { + auto v = original.getAsInteger(i); + replica.addTerm(v.first, v.second); + } + } else { + assert (original.getType() == MultiTerm::Type::UNKNOWN); + assert (original.getNumTerms() == 0); + } + } + void visit(WeightedSetTerm &node) override { - replicate(node, _builder.addWeightedSetTerm(node.getChildren().size(), node.getView(), - node.getId(), node.getWeight())); - visitNodes(node.getChildren()); + auto & replica = _builder.addWeightedSetTerm(node.getNumTerms(), node.getView(), node.getId(), node.getWeight()); + replicate(node, replica); + replicateMultiTerm(node, replica); } void visit(DotProduct &node) override { - replicate(node, _builder.addDotProduct(node.getChildren().size(), node.getView(), - node.getId(), node.getWeight())); - visitNodes(node.getChildren()); + auto & replica = _builder.addDotProduct(node.getNumTerms(), node.getView(), node.getId(), node.getWeight()); + replicate(node, replica); + replicateMultiTerm(node, replica); } void visit(WandTerm &node) override { - replicate(node, _builder.addWandTerm(node.getChildren().size(), - node.getView(), - node.getId(), node.getWeight(), - node.getTargetNumHits(), - node.getScoreThreshold(), - node.getThresholdBoostFactor())); - visitNodes(node.getChildren()); + auto & replica = _builder.addWandTerm(node.getNumTerms(), node.getView(), node.getId(), node.getWeight(), + node.getTargetNumHits(), + node.getScoreThreshold(), + node.getThresholdBoostFactor()); + replicate(node, replica); + replicateMultiTerm(node, replica); } void visit(Rank &node) override { diff --git a/searchlib/src/vespa/searchlib/query/tree/simplequery.h b/searchlib/src/vespa/searchlib/query/tree/simplequery.h index db517edc348..bdf1141fde5 100644 --- a/searchlib/src/vespa/searchlib/query/tree/simplequery.h +++ b/searchlib/src/vespa/searchlib/query/tree/simplequery.h @@ -35,17 +35,17 @@ struct SimpleSameElement : SameElement { SimpleSameElement(vespalib::stringref view) : SameElement(view) {} }; struct SimpleWeightedSetTerm : WeightedSetTerm { - SimpleWeightedSetTerm(vespalib::stringref view, int32_t id, Weight weight) - : WeightedSetTerm(view, id, weight) {} + SimpleWeightedSetTerm(uint32_t num_terms, vespalib::stringref view, int32_t id, Weight weight) + : WeightedSetTerm(num_terms, view, id, weight) {} }; struct SimpleDotProduct : DotProduct { - SimpleDotProduct(vespalib::stringref view, int32_t id, Weight weight) - : DotProduct(view, id, weight) {} + SimpleDotProduct(uint32_t num_terms, vespalib::stringref view, int32_t id, Weight weight) + : DotProduct(num_terms, view, id, weight) {} }; struct SimpleWandTerm : WandTerm { - SimpleWandTerm(vespalib::stringref view, int32_t id, Weight weight, + SimpleWandTerm(uint32_t num_terms, vespalib::stringref view, int32_t id, Weight weight, uint32_t targetNumHits, int64_t scoreThreshold, double thresholdBoostFactor) - : WandTerm(view, id, weight, targetNumHits, scoreThreshold, thresholdBoostFactor) {} + : WandTerm(num_terms, view, id, weight, targetNumHits, scoreThreshold, thresholdBoostFactor) {} }; struct SimpleRank : Rank {}; struct SimpleNumberTerm : NumberTerm { diff --git a/searchlib/src/vespa/searchlib/query/tree/stackdumpcreator.cpp b/searchlib/src/vespa/searchlib/query/tree/stackdumpcreator.cpp index 13756af3c27..5ad5fa80da4 100644 --- a/searchlib/src/vespa/searchlib/query/tree/stackdumpcreator.cpp +++ b/searchlib/src/vespa/searchlib/query/tree/stackdumpcreator.cpp @@ -169,18 +169,28 @@ class QueryNodeConverter : public QueryVisitor { if (typefield & ParseItem::IF_FLAGS) { appendByte(flags); } - appendCompressedPositiveNumber(node.getChildren().size()); + appendCompressedPositiveNumber(node.getNumTerms()); appendString(node.getView()); } + void createMultiTermNodes(const MultiTerm & mt) { + for (size_t i = 0; i < mt.getNumTerms(); ++i) { + auto term = mt.getAsString(i); + uint8_t typeField = static_cast<uint8_t>(ParseItem::ITEM_PURE_WEIGHTED_STRING) | static_cast<uint8_t>(ParseItem::IF_WEIGHT); + appendByte(typeField); + appendCompressedNumber(term.second.percent()); + appendString(term.first); + } + } + void visit(WeightedSetTerm &node) override { createWeightedSet(node, static_cast<uint8_t>(ParseItem::ITEM_WEIGHTED_SET) | static_cast<uint8_t>(ParseItem::IF_WEIGHT)); - visitNodes(node.getChildren()); + createMultiTermNodes(node); } void visit(DotProduct &node) override { createWeightedSet(node, static_cast<uint8_t>(ParseItem::ITEM_DOT_PRODUCT) | static_cast<uint8_t>(ParseItem::IF_WEIGHT)); - visitNodes(node.getChildren()); + createMultiTermNodes(node); } void visit(WandTerm &node) override { @@ -188,7 +198,7 @@ class QueryNodeConverter : public QueryVisitor { appendCompressedPositiveNumber(node.getTargetNumHits()); appendDouble(node.getScoreThreshold()); appendDouble(node.getThresholdBoostFactor()); - visitNodes(node.getChildren()); + createMultiTermNodes(node); } void visit(Rank &node) override { diff --git a/searchlib/src/vespa/searchlib/query/tree/stackdumpquerycreator.h b/searchlib/src/vespa/searchlib/query/tree/stackdumpquerycreator.h index 040ac751d25..27df4f6b6e5 100644 --- a/searchlib/src/vespa/searchlib/query/tree/stackdumpquerycreator.h +++ b/searchlib/src/vespa/searchlib/query/tree/stackdumpquerycreator.h @@ -8,6 +8,8 @@ #include <vespa/searchlib/parsequery/stackdumpiterator.h> #include <vespa/searchlib/common/geo_location_parser.h> #include <vespa/vespalib/objects/hexdump.h> +#include <vespa/vespalib/util/stringfmt.h> +#include <charconv> namespace search::query { @@ -45,12 +47,33 @@ public: return builder.build(); } -private: - static Term * createQueryTerm(search::SimpleQueryStackDumpIterator &queryStack, QueryBuilder<NodeTypes> & builder, vespalib::stringref & pureTermView) { +private: + static void populateMultiTerm(search::SimpleQueryStackDumpIterator &queryStack, QueryBuilderBase & builder, MultiTerm & mt) { + uint32_t added(0); + for (added = 0; (added < mt.getNumTerms()) && queryStack.next(); added++) { + ParseItem::ItemType type = queryStack.getType(); + switch (type) { + case ParseItem::ITEM_PURE_WEIGHTED_LONG: + mt.addTerm(queryStack.getIntergerTerm(), queryStack.GetWeight()); + break; + case ParseItem::ITEM_PURE_WEIGHTED_STRING: + mt.addTerm(queryStack.getTerm(), queryStack.GetWeight()); + break; + default: + builder.reportError(vespalib::make_string("Got unexpected node %d for multiterm node at child term %d", type, added)); + return; + } + } + if (added < mt.getNumTerms()) { + builder.reportError(vespalib::make_string("Too few nodes(%d) for multiterm(%d)", added, mt.getNumTerms())); + } + } + static Term * + createQueryTerm(search::SimpleQueryStackDumpIterator &queryStack, QueryBuilder<NodeTypes> & builder, vespalib::stringref & pureTermView) { uint32_t arity = queryStack.getArity(); ParseItem::ItemType type = queryStack.getType(); Node::UP node; - Term *t = 0; + Term *t = nullptr; if (type == ParseItem::ITEM_AND) { builder.addAnd(arity); } else if (type == ParseItem::ITEM_RANK) { @@ -92,14 +115,18 @@ private: vespalib::stringref view = queryStack.getIndexName(); int32_t id = queryStack.getUniqueId(); Weight weight = queryStack.GetWeight(); - t = &builder.addWeightedSetTerm(arity, view, id, weight); + auto & ws = builder.addWeightedSetTerm(arity, view, id, weight); pureTermView = vespalib::stringref(); + populateMultiTerm(queryStack, builder, ws); + t = &ws; } else if (type == ParseItem::ITEM_DOT_PRODUCT) { vespalib::stringref view = queryStack.getIndexName(); int32_t id = queryStack.getUniqueId(); Weight weight = queryStack.GetWeight(); - t = &builder.addDotProduct(arity, view, id, weight); + auto & dotProduct = builder.addDotProduct(arity, view, id, weight); pureTermView = vespalib::stringref(); + populateMultiTerm(queryStack, builder, dotProduct); + t = &dotProduct; } else if (type == ParseItem::ITEM_WAND) { vespalib::stringref view = queryStack.getIndexName(); int32_t id = queryStack.getUniqueId(); @@ -107,9 +134,10 @@ private: uint32_t targetNumHits = queryStack.getTargetNumHits(); double scoreThreshold = queryStack.getScoreThreshold(); double thresholdBoostFactor = queryStack.getThresholdBoostFactor(); - t = &builder.addWandTerm(arity, view, id, weight, - targetNumHits, scoreThreshold, thresholdBoostFactor); + auto & wand = builder.addWandTerm(arity, view, id, weight, targetNumHits, scoreThreshold, thresholdBoostFactor); pureTermView = vespalib::stringref(); + populateMultiTerm(queryStack, builder, wand); + t = & wand; } else if (type == ParseItem::ITEM_NOT) { builder.addAndNot(arity); } else if (type == ParseItem::ITEM_NEAREST_NEIGHBOR) { @@ -135,7 +163,9 @@ private: } else if (type == ParseItem::ITEM_PURE_WEIGHTED_STRING) { t = &builder.addStringTerm(term, pureTermView, id, weight); } else if (type == ParseItem::ITEM_PURE_WEIGHTED_LONG) { - t = &builder.addNumberTerm(term, pureTermView, id, weight); + char buf[24]; + auto res = std::to_chars(buf, buf + sizeof(buf), queryStack.getIntergerTerm(), 10); + t = &builder.addNumberTerm(vespalib::stringref(buf, res.ptr - buf), pureTermView, id, weight); } else if (type == ParseItem::ITEM_PREFIXTERM) { t = &builder.addPrefixTerm(term, view, id, weight); } else if (type == ParseItem::ITEM_SUBSTRINGTERM) { diff --git a/searchlib/src/vespa/searchlib/query/tree/termnodes.cpp b/searchlib/src/vespa/searchlib/query/tree/termnodes.cpp index b9a9687c85b..21ae11c1106 100644 --- a/searchlib/src/vespa/searchlib/query/tree/termnodes.cpp +++ b/searchlib/src/vespa/searchlib/query/tree/termnodes.cpp @@ -1,26 +1,130 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "termnodes.h" +#include <vespa/vespalib/util/exceptions.h> +#include <charconv> +using vespalib::IllegalArgumentException; +using vespalib::stringref; +using vespalib::make_string_short::fmt; namespace search::query { -NumberTerm::~NumberTerm() = default; +StringTerm::StringTerm(const Type &term, vespalib::stringref view, int32_t id, Weight weight) + : QueryNodeMixinType(term, view, id, weight) +{} +NumberTerm::~NumberTerm() = default; PrefixTerm::~PrefixTerm() = default; - RangeTerm::~RangeTerm() = default; +StringTerm::~StringTerm() = default; +SubstringTerm::~SubstringTerm() = default; +SuffixTerm::~SuffixTerm() = default; +LocationTerm::~LocationTerm() = default; +RegExpTerm::~RegExpTerm() = default; +WeightedSetTerm::~WeightedSetTerm() = default; +DotProduct::~DotProduct() = default; +WandTerm::~WandTerm() = default; -StringTerm::StringTerm(const Type &term, vespalib::stringref view, int32_t id, Weight weight) - : QueryNodeMixinType(term, view, id, weight) +namespace { + +class StringTermVector final : public MultiTerm::TermVector { +public: + StringTermVector(uint32_t sz) : _terms() { _terms.reserve(sz); } + void addTerm(stringref term, Weight weight) override { + _terms.emplace_back(term, weight); + } + void addTerm(int64_t value, Weight weight) override { + char buf[24]; + auto res = std::to_chars(buf, buf + sizeof(buf), value, 10); + addTerm(stringref(buf, res.ptr - buf), weight); + } + StringAndWeight getAsString(uint32_t index) const override { + const auto & v = _terms[index]; + return StringAndWeight(v.first, v.second); + } + IntegerAndWeight getAsInteger(uint32_t index) const override { + const auto & v = _terms[index]; + int64_t value(0); + std::from_chars(v.first.c_str(), v.first.c_str() + v.first.size(), value); + return IntegerAndWeight(value, v.second); + } + Weight getWeight(uint32_t index) const override { + return _terms[index].second; + } + uint32_t size() const override { return _terms.size(); } +private: + std::vector<std::pair<vespalib::string, Weight>> _terms; +}; + +class IntegerTermVector final : public MultiTerm::TermVector { +public: + IntegerTermVector(uint32_t sz) : _terms() { _terms.reserve(sz); } + void addTerm(stringref, Weight) override { + // Will/should never happen + assert(false); + } + void addTerm(int64_t term, Weight weight) override { + _terms.emplace_back(term, weight); + } + StringAndWeight getAsString(uint32_t index) const override { + const auto & v = _terms[index]; + auto res = std::to_chars(_scratchPad, _scratchPad + sizeof(_scratchPad)-1, v.first, 10); + res.ptr[0] = '\0'; + return StringAndWeight(stringref(_scratchPad, res.ptr - _scratchPad), v.second); + } + IntegerAndWeight getAsInteger(uint32_t index) const override { + return _terms[index]; + } + Weight getWeight(uint32_t index) const override { + return _terms[index].second; + } + uint32_t size() const override { return _terms.size(); } +private: + std::vector<IntegerAndWeight> _terms; + mutable char _scratchPad[24]; +}; + +} + +MultiTerm::MultiTerm(uint32_t num_terms) + : _terms(), + _num_terms(num_terms), + _type(Type::UNKNOWN) {} -StringTerm::~StringTerm() = default; -SubstringTerm::~SubstringTerm() = default; +MultiTerm::~MultiTerm() = default; -SuffixTerm::~SuffixTerm() = default; +std::unique_ptr<MultiTerm::TermVector> +MultiTerm::downgrade() { + // Downgrade all number to string. This should really not happen + auto new_terms = std::make_unique<StringTermVector>(_num_terms); + for (uint32_t i(0), m(_terms->size()); i < m; i++) { + auto v = _terms->getAsString(i); + new_terms->addTerm(v.first, v.second); + } + return new_terms; +} -LocationTerm::~LocationTerm() = default; +void +MultiTerm::addTerm(vespalib::stringref term, Weight weight) { + if ( ! _terms) { + _terms = std::make_unique<StringTermVector>(_num_terms); + _type = Type::STRING; + } + if (_type == Type::INTEGER) { + _terms = downgrade(); + _type = Type::STRING; + } + _terms->addTerm(term, weight); +} -RegExpTerm::~RegExpTerm() = default; +void +MultiTerm::addTerm(int64_t term, Weight weight) { + if ( ! _terms) { + _terms = std::make_unique<IntegerTermVector>(_num_terms); + _type = Type::INTEGER; + } + _terms->addTerm(term, weight); +} } diff --git a/searchlib/src/vespa/searchlib/query/tree/termnodes.h b/searchlib/src/vespa/searchlib/query/tree/termnodes.h index 3eda0732470..0373ebbb653 100644 --- a/searchlib/src/vespa/searchlib/query/tree/termnodes.h +++ b/searchlib/src/vespa/searchlib/query/tree/termnodes.h @@ -153,5 +153,77 @@ public: double get_distance_threshold() const { return _distance_threshold; } }; +class MultiTerm : public Node { +public: + enum class Type {STRING, INTEGER, UNKNOWN}; + using StringAndWeight = std::pair<vespalib::stringref, Weight>; + using IntegerAndWeight = std::pair<int64_t, Weight>; + struct TermVector { + using StringAndWeight = MultiTerm::StringAndWeight; + using IntegerAndWeight = MultiTerm::IntegerAndWeight; + virtual ~TermVector() = default; + virtual void addTerm(vespalib::stringref term, Weight weight) = 0; + virtual void addTerm(int64_t term, Weight weight) = 0; + virtual StringAndWeight getAsString(uint32_t index) const = 0; + virtual IntegerAndWeight getAsInteger(uint32_t index) const = 0; + virtual Weight getWeight(uint32_t index) const = 0; + virtual uint32_t size() const = 0; + }; + ~MultiTerm() override; + void addTerm(vespalib::stringref term, Weight weight); + void addTerm(int64_t term, Weight weight); + // Note that the first refers to a zero terminated string. + // That is required as the comparator for the enum store requires it. + StringAndWeight getAsString(uint32_t index) const { return _terms->getAsString(index); } + IntegerAndWeight getAsInteger(uint32_t index) const { return _terms->getAsInteger(index); } + Weight weight(uint32_t index) const { return _terms->getWeight(index); } + uint32_t getNumTerms() const { return _num_terms; } + Type getType() const { return _type; } +protected: + MultiTerm(uint32_t num_terms); +private: + VESPA_DLL_LOCAL std::unique_ptr<TermVector> downgrade() __attribute__((noinline)); + std::unique_ptr<TermVector> _terms; + uint32_t _num_terms; + Type _type; +}; + +class WeightedSetTerm : public QueryNodeMixin<WeightedSetTerm, MultiTerm>, public Term { +public: + WeightedSetTerm(uint32_t num_terms, const vespalib::string &view, int32_t id, Weight weight) + : QueryNodeMixinType(num_terms), + Term(view, id, weight) + {} + virtual ~WeightedSetTerm() = 0; +}; + +class DotProduct : public QueryNodeMixin<DotProduct, MultiTerm>, public Term { +public: + DotProduct(uint32_t num_terms, const vespalib::string &view, int32_t id, Weight weight) + : QueryNodeMixinType(num_terms), + Term(view, id, weight) + {} + virtual ~DotProduct() = 0; +}; + +class WandTerm : public QueryNodeMixin<WandTerm, MultiTerm>, public Term { +private: + uint32_t _targetNumHits; + int64_t _scoreThreshold; + double _thresholdBoostFactor; +public: + WandTerm(uint32_t num_terms, const vespalib::string &view, int32_t id, Weight weight, + uint32_t targetNumHits, int64_t scoreThreshold, double thresholdBoostFactor) + : QueryNodeMixinType(num_terms), + Term(view, id, weight), + _targetNumHits(targetNumHits), + _scoreThreshold(scoreThreshold), + _thresholdBoostFactor(thresholdBoostFactor) + {} + virtual ~WandTerm() = 0; + uint32_t getTargetNumHits() const { return _targetNumHits; } + int64_t getScoreThreshold() const { return _scoreThreshold; } + double getThresholdBoostFactor() const { return _thresholdBoostFactor; } +}; } diff --git a/searchlib/src/vespa/searchlib/query/weight.h b/searchlib/src/vespa/searchlib/query/weight.h index 51596642cd2..3f0d3b12b99 100644 --- a/searchlib/src/vespa/searchlib/query/weight.h +++ b/searchlib/src/vespa/searchlib/query/weight.h @@ -19,7 +19,7 @@ public: * constructor. * @param value The initial weight in percent; should be 100 unless a specific value is set. **/ - explicit Weight(int32_t value) : _weight(value) {} + explicit Weight(int32_t value) noexcept : _weight(value) {} /** * change the weight value. @@ -40,7 +40,7 @@ public: double multiplier() const { return 0.01 * _weight; } /** compare two weights */ - bool operator== (const Weight& other) const { return _weight == other._weight; } + bool operator== (const Weight& other) const noexcept { return _weight == other._weight; } }; } diff --git a/searchlib/src/vespa/searchlib/queryeval/blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/blueprint.cpp index f600673b179..897d36150d9 100644 --- a/searchlib/src/vespa/searchlib/queryeval/blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/blueprint.cpp @@ -449,20 +449,6 @@ IntermediateBlueprint::createSearch(fef::MatchData &md, bool strict) const IntermediateBlueprint::IntermediateBlueprint() = default; -const Blueprint & -IntermediateBlueprint::getChild(size_t n) const -{ - assert(n < _children.size()); - return *_children[n]; -} - -Blueprint & -IntermediateBlueprint::getChild(size_t n) -{ - assert(n < _children.size()); - return *_children[n]; -} - IntermediateBlueprint & IntermediateBlueprint::addChild(Blueprint::UP child) { diff --git a/searchlib/src/vespa/searchlib/queryeval/blueprint.h b/searchlib/src/vespa/searchlib/queryeval/blueprint.h index 722b3701881..b75c903234f 100644 --- a/searchlib/src/vespa/searchlib/queryeval/blueprint.h +++ b/searchlib/src/vespa/searchlib/queryeval/blueprint.h @@ -220,6 +220,11 @@ public: virtual bool isEquiv() const { return false; } virtual bool isWhiteList() const { return false; } virtual bool isIntermediate() const { return false; } + virtual bool isAnd() const { return false; } + virtual bool isAndNot() const { return false; } + virtual bool isOr() const { return false; } + virtual bool isSourceBlender() const { return false; } + virtual bool isRank() const { return false; } virtual const attribute::ISearchContext *get_attribute_search_context() const { return nullptr; } // For document summaries with matched-elements-only set. @@ -295,8 +300,8 @@ public: IndexList find(const IPredicate & check) const; size_t childCnt() const { return _children.size(); } - const Blueprint &getChild(size_t n) const; - Blueprint &getChild(size_t n); + const Blueprint &getChild(size_t n) const { return *_children[n]; } + Blueprint &getChild(size_t n) { return *_children[n]; } IntermediateBlueprint & insertChild(size_t n, Blueprint::UP child); IntermediateBlueprint &addChild(Blueprint::UP child); Blueprint::UP removeChild(size_t n); diff --git a/searchlib/src/vespa/searchlib/queryeval/create_blueprint_visitor_helper.cpp b/searchlib/src/vespa/searchlib/queryeval/create_blueprint_visitor_helper.cpp index 3731b2ff6a8..a642010139b 100644 --- a/searchlib/src/vespa/searchlib/queryeval/create_blueprint_visitor_helper.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/create_blueprint_visitor_helper.cpp @@ -66,12 +66,12 @@ template <typename WS, typename NODE> void CreateBlueprintVisitorHelper::createWeightedSet(std::unique_ptr<WS> bp, NODE &n) { FieldSpecList fields; - for (size_t i = 0; i < n.getChildren().size(); ++i) { + for (size_t i = 0; i < n.getNumTerms(); ++i) { fields.clear(); fields.add(bp->getNextChildField(_field)); - const query::Node &node = *n.getChildren()[i]; - uint32_t weight = getWeightFromNode(node).percent(); - bp->addTerm(_searchable.createBlueprint(_requestContext, fields, node), weight); + auto term = n.getAsString(i); + query::SimpleStringTerm node(term.first, n.getView(), 0, term.second); // TODO Temporary + bp->addTerm(_searchable.createBlueprint(_requestContext, fields, node), term.second.percent()); } setResult(std::move(bp)); } diff --git a/searchlib/src/vespa/searchlib/queryeval/get_weight_from_node.cpp b/searchlib/src/vespa/searchlib/queryeval/get_weight_from_node.cpp index bd9de0a1762..344cf90da12 100644 --- a/searchlib/src/vespa/searchlib/queryeval/get_weight_from_node.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/get_weight_from_node.cpp @@ -18,7 +18,8 @@ struct WeightExtractor : public TemplateTermVisitor<WeightExtractor, SimpleQuery WeightExtractor() : weight(0) {} - template <class TermType> void visitTerm(TermType &n) { + template <class TermType> + void visitTerm(TermType &n) { weight = n.getWeight(); } diff --git a/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.cpp b/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.cpp index 3f632cfc377..561c532430b 100644 --- a/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.cpp @@ -35,8 +35,8 @@ void optimize_source_blenders(IntermediateBlueprint &self, size_t begin_idx) { std::vector<size_t> source_blenders; SourceBlenderBlueprint *reference = nullptr; for (size_t i = begin_idx; i < self.childCnt(); ++i) { - SourceBlenderBlueprint *child = dynamic_cast<SourceBlenderBlueprint *>(&self.getChild(i)); - if (child != nullptr) { + if (self.getChild(i).isSourceBlender()) { + auto *child = static_cast<SourceBlenderBlueprint *>(&self.getChild(i)); if (reference == nullptr || reference->isCompatibleWith(*child)) { source_blenders.push_back(i); reference = child; @@ -49,16 +49,16 @@ void optimize_source_blenders(IntermediateBlueprint &self, size_t begin_idx) { while (!source_blenders.empty()) { blender_up = self.removeChild(source_blenders.back()); source_blenders.pop_back(); - SourceBlenderBlueprint *blender = dynamic_cast<SourceBlenderBlueprint *>(blender_up.get()); - assert(blender != nullptr); + assert(blender_up->isSourceBlender()); + auto *blender = static_cast<SourceBlenderBlueprint *>(blender_up.get()); while (blender->childCnt() > 0) { Blueprint::UP child_up = blender->removeChild(blender->childCnt() - 1); size_t source_idx = lookup_create_source(sources, child_up->getSourceId()); sources[source_idx]->addChild(std::move(child_up)); } } - SourceBlenderBlueprint *top = dynamic_cast<SourceBlenderBlueprint *>(blender_up.get()); - assert(top != nullptr); + assert(blender_up->isSourceBlender()); + auto *top = static_cast<SourceBlenderBlueprint *>(blender_up.get()); while (!sources.empty()) { top->addChild(std::move(sources.back())); sources.pop_back(); @@ -117,8 +117,8 @@ AndNotBlueprint::optimize_self() if (childCnt() == 0) { return; } - AndNotBlueprint *child = dynamic_cast<AndNotBlueprint *>(&getChild(0)); - if (child != nullptr) { + if (getChild(0).isAndNot()) { + auto *child = static_cast<AndNotBlueprint *>(&getChild(0)); while (child->childCnt() > 1) { addChild(child->removeChild(1)); } @@ -130,7 +130,7 @@ AndNotBlueprint::optimize_self() removeChild(i--); } } - if (dynamic_cast<AndNotBlueprint *>(getParent()) == nullptr) { + if ( !(getParent() && getParent()->isAndNot()) ) { optimize_source_blenders<OrBlueprint>(*this, 1); } } @@ -224,15 +224,15 @@ void AndBlueprint::optimize_self() { for (size_t i = 0; i < childCnt(); ++i) { - AndBlueprint *child = dynamic_cast<AndBlueprint *>(&getChild(i)); - if (child != nullptr) { + if (getChild(i).isAnd()) { + auto *child = static_cast<AndBlueprint *>(&getChild(i)); while (child->childCnt() > 0) { addChild(child->removeChild(0)); } removeChild(i--); } } - if (dynamic_cast<AndBlueprint *>(getParent()) == nullptr) { + if ( !(getParent() && getParent()->isAnd()) ) { optimize_source_blenders<AndBlueprint>(*this, 0); } } @@ -311,8 +311,8 @@ void OrBlueprint::optimize_self() { for (size_t i = 0; (childCnt() > 1) && (i < childCnt()); ++i) { - OrBlueprint *child = dynamic_cast<OrBlueprint *>(&getChild(i)); - if (child != nullptr) { + if (getChild(i).isOr()) { + auto *child = static_cast<OrBlueprint *>(&getChild(i)); while (child->childCnt() > 0) { addChild(child->removeChild(0)); } @@ -321,7 +321,7 @@ OrBlueprint::optimize_self() removeChild(i--); } } - if (dynamic_cast<OrBlueprint *>(getParent()) == nullptr) { + if ( !(getParent() && getParent()->isOr()) ) { optimize_source_blenders<OrBlueprint>(*this, 0); } } diff --git a/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.h b/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.h index bc635952d55..9e9d715e45d 100644 --- a/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.h +++ b/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.h @@ -18,6 +18,7 @@ public: HitEstimate combine(const std::vector<HitEstimate> &data) const override; FieldSpecBaseList exposeFields() const override; void optimize_self() override; + bool isAndNot() const override { return true; } Blueprint::UP get_replacement() override; void sort(std::vector<Blueprint*> &children) const override; bool inheritStrict(size_t i) const override; @@ -40,11 +41,7 @@ public: HitEstimate combine(const std::vector<HitEstimate> &data) const override; FieldSpecBaseList exposeFields() const override; void optimize_self() override; - -private: - double computeNextHitRate(const Blueprint & child, double hitRate) const override; - -public: + bool isAnd() const override { return true; } Blueprint::UP get_replacement() override; void sort(std::vector<Blueprint*> &children) const override; bool inheritStrict(size_t i) const override; @@ -53,6 +50,8 @@ public: bool strict, fef::MatchData &md) const override; SearchIterator::UP createFilterSearch(bool strict, FilterConstraint constraint) const override; +private: + double computeNextHitRate(const Blueprint & child, double hitRate) const override; }; //----------------------------------------------------------------------------- @@ -65,6 +64,7 @@ public: HitEstimate combine(const std::vector<HitEstimate> &data) const override; FieldSpecBaseList exposeFields() const override; void optimize_self() override; + bool isOr() const override { return true; } Blueprint::UP get_replacement() override; void sort(std::vector<Blueprint*> &children) const override; bool inheritStrict(size_t i) const override; @@ -150,7 +150,7 @@ public: //----------------------------------------------------------------------------- -class RankBlueprint : public IntermediateBlueprint +class RankBlueprint final : public IntermediateBlueprint { public: HitEstimate combine(const std::vector<HitEstimate> &data) const override; @@ -159,6 +159,7 @@ public: Blueprint::UP get_replacement() override; void sort(std::vector<Blueprint*> &children) const override; bool inheritStrict(size_t i) const override; + bool isRank() const override { return true; } SearchIterator::UP createIntermediateSearch(MultiSearch::Children subSearches, bool strict, fef::MatchData &md) const override; @@ -168,7 +169,7 @@ public: //----------------------------------------------------------------------------- -class SourceBlenderBlueprint : public IntermediateBlueprint +class SourceBlenderBlueprint final : public IntermediateBlueprint { private: const ISourceSelector &_selector; @@ -193,6 +194,7 @@ public: /** check if this blueprint has the same source selector as the other */ bool isCompatibleWith(const SourceBlenderBlueprint &other) const; + bool isSourceBlender() const override { return true; } }; } diff --git a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp index 4d69bebe065..4ec23b993b6 100644 --- a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp @@ -46,7 +46,7 @@ struct ConvertCellsSelector } // namespace <unnamed> NearestNeighborBlueprint::NearestNeighborBlueprint(const queryeval::FieldSpec& field, - const tensor::DenseTensorAttribute& attr_tensor, + const tensor::ITensorAttribute& attr_tensor, std::unique_ptr<Value> query_tensor, uint32_t target_num_hits, bool approximate, uint32_t explore_additional_hits, double distance_threshold, double brute_force_limit) @@ -68,7 +68,7 @@ NearestNeighborBlueprint::NearestNeighborBlueprint(const queryeval::FieldSpec& f using MyTypify = vespalib::eval::TypifyCellType; auto fixup_fun = vespalib::typify_invoke<2,MyTypify,ConvertCellsSelector>(lct, rct); fixup_fun(_query_tensor, _attr_tensor.getTensorType()); - _fallback_dist_fun = search::tensor::make_distance_function(_attr_tensor.getConfig().distance_metric(), rct); + _fallback_dist_fun = search::tensor::make_distance_function(_attr_tensor.distance_metric(), rct); _dist_fun = _fallback_dist_fun.get(); assert(_dist_fun); auto nns_index = _attr_tensor.nearest_neighbor_index(); @@ -80,7 +80,7 @@ NearestNeighborBlueprint::NearestNeighborBlueprint(const queryeval::FieldSpec& f _distance_threshold = _dist_fun->convert_threshold(distance_threshold); _distance_heap.set_distance_threshold(_distance_threshold); } - uint32_t est_hits = _attr_tensor.getNumDocs(); + uint32_t est_hits = _attr_tensor.get_num_docs(); setEstimate(HitEstimate(est_hits, false)); set_want_global_filter(true); } @@ -97,7 +97,7 @@ NearestNeighborBlueprint::set_global_filter(const GlobalFilter &global_filter) (nns_index ? "nns_index" : "no_index"), (_global_filter->has_filter() ? "has_filter" : "no_filter")); if (_approximate && nns_index) { - uint32_t est_hits = _attr_tensor.getNumDocs(); + uint32_t est_hits = _attr_tensor.get_num_docs(); if (_global_filter->has_filter()) { uint32_t max_hits = _global_filter->filter()->countTrueBits(); LOG(debug, "set_global_filter getNumDocs: %u / max_hits %u", est_hits, max_hits); diff --git a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.h b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.h index aad43c923a2..827dafacf57 100644 --- a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.h +++ b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.h @@ -6,7 +6,7 @@ #include <vespa/searchlib/tensor/distance_function.h> #include <vespa/searchlib/tensor/nearest_neighbor_index.h> -namespace search::tensor { class DenseTensorAttribute; } +namespace search::tensor { class ITensorAttribute; } namespace vespalib::eval { struct Value; } namespace search::queryeval { @@ -19,7 +19,7 @@ namespace search::queryeval { */ class NearestNeighborBlueprint : public ComplexLeafBlueprint { private: - const tensor::DenseTensorAttribute& _attr_tensor; + const tensor::ITensorAttribute& _attr_tensor; std::unique_ptr<vespalib::eval::Value> _query_tensor; uint32_t _target_num_hits; bool _approximate; @@ -35,7 +35,7 @@ private: void perform_top_k(); public: NearestNeighborBlueprint(const queryeval::FieldSpec& field, - const tensor::DenseTensorAttribute& attr_tensor, + const tensor::ITensorAttribute& attr_tensor, std::unique_ptr<vespalib::eval::Value> query_tensor, uint32_t target_num_hits, bool approximate, uint32_t explore_additional_hits, double distance_threshold, @@ -43,7 +43,7 @@ public: NearestNeighborBlueprint(const NearestNeighborBlueprint&) = delete; NearestNeighborBlueprint& operator=(const NearestNeighborBlueprint&) = delete; ~NearestNeighborBlueprint(); - const tensor::DenseTensorAttribute& get_attribute_tensor() const { return _attr_tensor; } + const tensor::ITensorAttribute& get_attribute_tensor() const { return _attr_tensor; } const vespalib::eval::Value& get_query_tensor() const { return *_query_tensor; } uint32_t get_target_num_hits() const { return _target_num_hits; } void set_global_filter(const GlobalFilter &global_filter) override; diff --git a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.cpp b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.cpp index 52814bb2631..d85da49c2f7 100644 --- a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.cpp @@ -3,7 +3,7 @@ #include "nearest_neighbor_iterator.h" #include <vespa/searchlib/common/bitvector.h> -using search::tensor::DenseTensorAttribute; +using search::tensor::ITensorAttribute; using vespalib::ConstArrayRef; using vespalib::eval::TypedCells; using vespalib::eval::CellType; @@ -109,7 +109,7 @@ NearestNeighborIterator::create( bool strict, fef::TermFieldMatchData &tfmd, const vespalib::eval::Value &queryTensor, - const search::tensor::DenseTensorAttribute &tensorAttribute, + const search::tensor::ITensorAttribute &tensorAttribute, NearestNeighborDistanceHeap &distanceHeap, const search::BitVector *filter, const search::tensor::DistanceFunction *dist_fun) diff --git a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.h b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.h index eec45ea1af0..64ac91e6db5 100644 --- a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.h +++ b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.h @@ -6,7 +6,7 @@ #include "nearest_neighbor_distance_heap.h" #include <vespa/eval/eval/value.h> #include <vespa/searchlib/fef/termfieldmatchdata.h> -#include <vespa/searchlib/tensor/dense_tensor_attribute.h> +#include <vespa/searchlib/tensor/i_tensor_attribute.h> #include <vespa/searchlib/tensor/distance_function.h> #include <vespa/vespalib/util/priority_queue.h> #include <cmath> @@ -16,20 +16,20 @@ namespace search::queryeval { class NearestNeighborIterator : public SearchIterator { public: - using DenseTensorAttribute = search::tensor::DenseTensorAttribute; + using ITensorAttribute = search::tensor::ITensorAttribute; using Value = vespalib::eval::Value; struct Params { fef::TermFieldMatchData &tfmd; const Value &queryTensor; - const DenseTensorAttribute &tensorAttribute; + const ITensorAttribute &tensorAttribute; NearestNeighborDistanceHeap &distanceHeap; const search::BitVector *filter; const search::tensor::DistanceFunction *distanceFunction; Params(fef::TermFieldMatchData &tfmd_in, const Value &queryTensor_in, - const DenseTensorAttribute &tensorAttribute_in, + const ITensorAttribute &tensorAttribute_in, NearestNeighborDistanceHeap &distanceHeap_in, const search::BitVector *filter_in, const search::tensor::DistanceFunction *distanceFunction_in) @@ -50,7 +50,7 @@ public: bool strict, fef::TermFieldMatchData &tfmd, const Value &queryTensor, - const search::tensor::DenseTensorAttribute &tensorAttribute, + const search::tensor::ITensorAttribute &tensorAttribute, NearestNeighborDistanceHeap &distanceHeap, const search::BitVector *filter, const search::tensor::DistanceFunction *dist_fun); diff --git a/searchlib/src/vespa/searchlib/queryeval/termasstring.cpp b/searchlib/src/vespa/searchlib/queryeval/termasstring.cpp index 7a97110713d..0c881bf32f3 100644 --- a/searchlib/src/vespa/searchlib/queryeval/termasstring.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/termasstring.cpp @@ -38,44 +38,46 @@ using search::query::WandTerm; using search::query::WeakAnd; using search::query::WeightedSetTerm; using vespalib::string; +using vespalib::stringref; namespace search::queryeval { -vespalib::string termAsString(double float_term) { - vespalib::asciistream os; - return (os << float_term).str(); -} +namespace { -vespalib::string termAsString(int64_t int_term) { +stringref +termAsString(const search::query::Range &term, string & scratchPad) { vespalib::asciistream os; - return (os << int_term).str(); + scratchPad = (os << term).str(); + return scratchPad; } -vespalib::string termAsString(const search::query::Range &term) { +stringref +termAsString(const search::query::Location &term, string & scratchPad) { vespalib::asciistream os; - return (os << term).str(); + scratchPad = (os << term).str(); + return scratchPad; } -vespalib::string termAsString(const search::query::Location &term) { - vespalib::asciistream os; - return (os << term).str(); +stringref +termAsString(const string &term, string &) { + return term; } -namespace { struct TermAsStringVisitor : public QueryVisitor { - string term; - bool isSet; + string & _scratchPad; + stringref term; + bool isSet; - TermAsStringVisitor() : term(), isSet(false) {} + TermAsStringVisitor(string & scratchPad) : _scratchPad(scratchPad), term(), isSet(false) {} template <class TermNode> void visitTerm(TermNode &n) { - term = termAsString(n.getTerm()); + term = termAsString(n.getTerm(), _scratchPad); isSet = true; } void illegalVisit() { - term.clear(); + term = stringref(); isSet = false; } @@ -104,15 +106,30 @@ struct TermAsStringVisitor : public QueryVisitor { void visit(PredicateQuery &) override {illegalVisit(); } void visit(NearestNeighborTerm &) override { illegalVisit(); } }; + +void throwFailure(const search::query::Node &term_node) __attribute((noinline)); + +void +throwFailure(const search::query::Node &term_node) { + string err(vespalib::make_string("Trying to convert a non-term node ('%s') to a term string.", typeid(term_node).name())); + LOG(warning, "%s", err.c_str()); + throw vespalib::IllegalArgumentException(err, VESPA_STRLOC); +} + } // namespace -string termAsString(const Node &term_node) { - TermAsStringVisitor visitor; +string +termAsString(const Node &term_node) { + string scratchPad; + return termAsString(term_node, scratchPad); +} + +stringref +termAsString(const search::query::Node &term_node, string & scratchPad) { + TermAsStringVisitor visitor(scratchPad); const_cast<Node &>(term_node).accept(visitor); if (!visitor.isSet) { - vespalib::string err(vespalib::make_string("Trying to convert a non-term node ('%s') to a term string.", typeid(term_node).name())); - LOG(warning, "%s", err.c_str()); - throw vespalib::IllegalArgumentException(err, VESPA_STRLOC); + throwFailure(term_node); } return visitor.term; } diff --git a/searchlib/src/vespa/searchlib/queryeval/termasstring.h b/searchlib/src/vespa/searchlib/queryeval/termasstring.h index c40050f0e2b..8b05bf9eddd 100644 --- a/searchlib/src/vespa/searchlib/queryeval/termasstring.h +++ b/searchlib/src/vespa/searchlib/queryeval/termasstring.h @@ -2,26 +2,13 @@ #pragma once -#include <vespa/searchlib/query/tree/location.h> -#include <vespa/searchlib/query/tree/range.h> -#include <string> +#include <vespa/vespalib/stllike/string.h> namespace search::query { class Node; } namespace search::queryeval { -inline const vespalib::string &termAsString(const vespalib::string &term) { - return term; -} - -vespalib::string termAsString(double float_term); - -vespalib::string termAsString(int64_t int_term); - -vespalib::string termAsString(const search::query::Range &term); - -vespalib::string termAsString(const search::query::Location &term); - vespalib::string termAsString(const search::query::Node &term_node); +vespalib::stringref termAsString(const search::query::Node &term_node, vespalib::string & scratchPad); } diff --git a/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt b/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt index 79b18a57a34..8af77a57a44 100644 --- a/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt @@ -6,8 +6,8 @@ vespa_add_library(searchlib_tensor OBJECT dense_tensor_attribute_saver.cpp dense_tensor_store.cpp direct_tensor_attribute.cpp - direct_tensor_store.cpp direct_tensor_saver.cpp + direct_tensor_store.cpp distance_function_factory.cpp distance_functions.cpp hnsw_graph.cpp @@ -19,14 +19,11 @@ vespa_add_library(searchlib_tensor OBJECT inv_log_level_generator.cpp nearest_neighbor_index.cpp nearest_neighbor_index_saver.cpp - serialized_tensor_attribute.cpp - serialized_tensor_attribute_saver.cpp - serialized_tensor_store.cpp - tensor_attribute.cpp - tensor_deserialize.cpp - tensor_store.cpp serialized_fast_value_attribute.cpp streamed_value_saver.cpp streamed_value_store.cpp + tensor_attribute.cpp + tensor_deserialize.cpp + tensor_store.cpp DEPENDS ) diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp index 0f39816257f..be127a8be3e 100644 --- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp @@ -10,8 +10,6 @@ #include <vespa/searchlib/attribute/load_utils.h> #include <vespa/searchlib/attribute/readerbase.h> #include <vespa/vespalib/data/slime/inserter.h> -#include <vespa/vespalib/util/memory_allocator.h> -#include <vespa/vespalib/util/mmap_file_allocator_factory.h> #include <vespa/log/log.h> LOG_SETUP(".searchlib.tensor.dense_tensor_attribute"); @@ -74,15 +72,6 @@ can_use_index_save_file(const search::attribute::Config &config, const search::a return true; } -std::unique_ptr<vespalib::alloc::MemoryAllocator> -make_memory_allocator(const vespalib::string& name, bool huge) -{ - if (huge) { - return vespalib::alloc::MmapFileAllocatorFactory::instance().make_memory_allocator(name); - } - return {}; -} - } void @@ -115,7 +104,7 @@ DenseTensorAttribute::memory_usage() const DenseTensorAttribute::DenseTensorAttribute(vespalib::stringref baseFileName, const Config& cfg, const NearestNeighborIndexFactory& index_factory) : TensorAttribute(baseFileName, cfg, _denseTensorStore), - _denseTensorStore(cfg.tensorType(), make_memory_allocator(getName(), cfg.huge())), + _denseTensorStore(cfg.tensorType()), _index() { if (cfg.hnsw_index_params().has_value()) { diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.h b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.h index 55e7b8cb464..097c3c93dad 100644 --- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.h +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.h @@ -48,7 +48,7 @@ public: // Implements DocVectorAccess vespalib::eval::TypedCells get_vector(uint32_t docid) const override; - const NearestNeighborIndex* nearest_neighbor_index() const { return _index.get(); } + const NearestNeighborIndex* nearest_neighbor_index() const override { return _index.get(); } }; } diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp b/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp index 13796d35dec..e6da839da2e 100644 --- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp @@ -40,9 +40,8 @@ DenseTensorStore::TensorSizeCalc::alignedSize() const return my_align(bufSize(), DENSE_TENSOR_ALIGNMENT); } -DenseTensorStore::BufferType::BufferType(const TensorSizeCalc &tensorSizeCalc, std::unique_ptr<vespalib::alloc::MemoryAllocator> allocator) - : vespalib::datastore::BufferType<char>(tensorSizeCalc.alignedSize(), MIN_BUFFER_ARRAYS, RefType::offsetSize()), - _allocator(std::move(allocator)) +DenseTensorStore::BufferType::BufferType(const TensorSizeCalc &tensorSizeCalc) + : vespalib::datastore::BufferType<char>(tensorSizeCalc.alignedSize(), MIN_BUFFER_ARRAYS, RefType::offsetSize()) {} DenseTensorStore::BufferType::~BufferType() = default; @@ -54,17 +53,11 @@ DenseTensorStore::BufferType::cleanHold(void *buffer, size_t offset, memset(static_cast<char *>(buffer) + offset, 0, numElems); } -const vespalib::alloc::MemoryAllocator* -DenseTensorStore::BufferType::get_memory_allocator() const -{ - return _allocator.get(); -} - -DenseTensorStore::DenseTensorStore(const ValueType &type, std::unique_ptr<vespalib::alloc::MemoryAllocator> allocator) +DenseTensorStore::DenseTensorStore(const ValueType &type) : TensorStore(_concreteStore), _concreteStore(), _tensorSizeCalc(type), - _bufferType(_tensorSizeCalc, std::move(allocator)), + _bufferType(_tensorSizeCalc), _type(type), _emptySpace() { diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.h b/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.h index dad28642e67..638f602d613 100644 --- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.h +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.h @@ -36,12 +36,10 @@ public: class BufferType : public vespalib::datastore::BufferType<char> { using CleanContext = vespalib::datastore::BufferType<char>::CleanContext; - std::unique_ptr<vespalib::alloc::MemoryAllocator> _allocator; public: - BufferType(const TensorSizeCalc &tensorSizeCalc, std::unique_ptr<vespalib::alloc::MemoryAllocator> allocator); + BufferType(const TensorSizeCalc &tensorSizeCalc); ~BufferType() override; void cleanHold(void *buffer, size_t offset, ElemCount numElems, CleanContext cleanCtx) override; - const vespalib::alloc::MemoryAllocator* get_memory_allocator() const override; }; private: DataStoreType _concreteStore; @@ -57,7 +55,7 @@ private: setDenseTensor(const TensorType &tensor); public: - DenseTensorStore(const ValueType &type, std::unique_ptr<vespalib::alloc::MemoryAllocator> allocator); + DenseTensorStore(const ValueType &type); ~DenseTensorStore() override; const ValueType &type() const { return _type; } diff --git a/searchlib/src/vespa/searchlib/tensor/i_tensor_attribute.h b/searchlib/src/vespa/searchlib/tensor/i_tensor_attribute.h index 360250c869e..c2896b6a12e 100644 --- a/searchlib/src/vespa/searchlib/tensor/i_tensor_attribute.h +++ b/searchlib/src/vespa/searchlib/tensor/i_tensor_attribute.h @@ -4,12 +4,15 @@ #include <memory> #include <vespa/eval/eval/typed_cells.h> +#include <vespa/searchcommon/attribute/distance_metric.h> namespace vespalib::eval { class ValueType; struct Value; } namespace vespalib::slime { struct Inserter; } namespace search::tensor { +class NearestNeighborIndex; + /** * Interface for tensor attribute used by feature executors to get information. */ @@ -26,6 +29,11 @@ public: virtual const vespalib::eval::ValueType & getTensorType() const = 0; + virtual const NearestNeighborIndex* nearest_neighbor_index() const { return nullptr; } + using DistanceMetric = search::attribute::DistanceMetric; + virtual DistanceMetric distance_metric() const = 0; + virtual uint32_t get_num_docs() const = 0; + /** * Gets custom state for this tensor attribute by inserting it into the given Slime inserter. * This function is only called by the writer thread or when the writer thread is blocked. diff --git a/searchlib/src/vespa/searchlib/tensor/imported_tensor_attribute_vector_read_guard.h b/searchlib/src/vespa/searchlib/tensor/imported_tensor_attribute_vector_read_guard.h index c55a922487f..a88f0a5de76 100644 --- a/searchlib/src/vespa/searchlib/tensor/imported_tensor_attribute_vector_read_guard.h +++ b/searchlib/src/vespa/searchlib/tensor/imported_tensor_attribute_vector_read_guard.h @@ -36,6 +36,9 @@ public: const vespalib::eval::Value& get_tensor_ref(uint32_t docid) const override; bool supports_extract_cells_ref() const override { return _target_tensor_attribute.supports_extract_cells_ref(); } bool supports_get_tensor_ref() const override { return _target_tensor_attribute.supports_get_tensor_ref(); } + DistanceMetric distance_metric() const override { return _target_tensor_attribute.distance_metric(); } + uint32_t get_num_docs() const override { return getNumDocs(); } + const vespalib::eval::ValueType &getTensorType() const override; void get_state(const vespalib::slime::Inserter& inserter) const override; }; diff --git a/searchlib/src/vespa/searchlib/tensor/serialized_tensor_attribute.cpp b/searchlib/src/vespa/searchlib/tensor/serialized_tensor_attribute.cpp deleted file mode 100644 index de6feb80958..00000000000 --- a/searchlib/src/vespa/searchlib/tensor/serialized_tensor_attribute.cpp +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "blob_sequence_reader.h" -#include "serialized_tensor_attribute.h" -#include "serialized_tensor_attribute_saver.h" -#include "tensor_attribute.hpp" -#include <vespa/eval/eval/value.h> -#include <vespa/fastlib/io/bufferedfile.h> -#include <vespa/searchlib/attribute/readerbase.h> -#include <vespa/searchlib/util/fileutil.h> -#include <vespa/vespalib/util/rcuvector.hpp> - -using vespalib::eval::Value; -using vespalib::eval::ValueType; - -namespace search::tensor { - -namespace { - -constexpr uint32_t TENSOR_ATTRIBUTE_VERSION = 0; - -} - -SerializedTensorAttribute::SerializedTensorAttribute(stringref name, const Config &cfg) - : TensorAttribute(name, cfg, _serializedTensorStore) -{ -} - - -SerializedTensorAttribute::~SerializedTensorAttribute() -{ - getGenerationHolder().clearHoldLists(); - _tensorStore.clearHoldLists(); -} - -void -SerializedTensorAttribute::setTensor(DocId docId, const vespalib::eval::Value &tensor) -{ - checkTensorType(tensor); - EntryRef ref = _serializedTensorStore.setTensor(tensor); - setTensorRef(docId, ref); -} - - -std::unique_ptr<Value> -SerializedTensorAttribute::getTensor(DocId docId) const -{ - EntryRef ref; - if (docId < getCommittedDocIdLimit()) { - ref = _refVector[docId]; - } - if (!ref.valid()) { - return {}; - } - return _serializedTensorStore.getTensor(ref); -} - -bool -SerializedTensorAttribute::onLoad() -{ - BlobSequenceReader tensorReader(*this); - if (!tensorReader.hasData()) { - return false; - } - setCreateSerialNum(tensorReader.getCreateSerialNum()); - assert(tensorReader.getVersion() == TENSOR_ATTRIBUTE_VERSION); - uint32_t numDocs(tensorReader.getDocIdLimit()); - _refVector.reset(); - _refVector.unsafe_reserve(numDocs); - for (uint32_t lid = 0; lid < numDocs; ++lid) { - uint32_t tensorSize = tensorReader.getNextSize(); - auto raw = _serializedTensorStore.allocRawBuffer(tensorSize); - if (tensorSize != 0) { - tensorReader.readBlob(raw.data, tensorSize); - } - _refVector.push_back(raw.ref); - } - setNumDocs(numDocs); - setCommittedDocIdLimit(numDocs); - return true; -} - - -std::unique_ptr<AttributeSaver> -SerializedTensorAttribute::onInitSave(vespalib::stringref fileName) -{ - vespalib::GenerationHandler::Guard guard(getGenerationHandler(). - takeGuard()); - return std::make_unique<SerializedTensorAttributeSaver> - (std::move(guard), - this->createAttributeHeader(fileName), - getRefCopy(), - _serializedTensorStore); -} - -void -SerializedTensorAttribute::compactWorst() -{ - doCompactWorst<SerializedTensorStore::RefType>(); -} - -} diff --git a/searchlib/src/vespa/searchlib/tensor/serialized_tensor_attribute.h b/searchlib/src/vespa/searchlib/tensor/serialized_tensor_attribute.h deleted file mode 100644 index fea11e3b0c4..00000000000 --- a/searchlib/src/vespa/searchlib/tensor/serialized_tensor_attribute.h +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include "serialized_tensor_store.h" -#include "tensor_attribute.h" - -namespace search::tensor { - -/** - * Attribute vector class used to store tensors for all documents in memory. - */ -class SerializedTensorAttribute : public TensorAttribute { - SerializedTensorStore _serializedTensorStore; // data store for serialized tensors -public: - SerializedTensorAttribute(vespalib::stringref baseFileName, const Config &cfg); - virtual ~SerializedTensorAttribute(); - virtual void setTensor(DocId docId, const vespalib::eval::Value &tensor) override; - virtual std::unique_ptr<vespalib::eval::Value> getTensor(DocId docId) const override; - virtual bool onLoad() override; - virtual std::unique_ptr<AttributeSaver> onInitSave(vespalib::stringref fileName) override; - virtual void compactWorst() override; -}; - -} diff --git a/searchlib/src/vespa/searchlib/tensor/serialized_tensor_attribute_saver.cpp b/searchlib/src/vespa/searchlib/tensor/serialized_tensor_attribute_saver.cpp deleted file mode 100644 index 4c41c3a449e..00000000000 --- a/searchlib/src/vespa/searchlib/tensor/serialized_tensor_attribute_saver.cpp +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "serialized_tensor_attribute_saver.h" -#include "serialized_tensor_store.h" -#include <vespa/searchlib/util/bufferwriter.h> -#include <vespa/searchlib/attribute/iattributesavetarget.h> - -using vespalib::GenerationHandler; - -namespace search::tensor { - -SerializedTensorAttributeSaver:: -SerializedTensorAttributeSaver(GenerationHandler::Guard &&guard, - const attribute::AttributeHeader &header, - RefCopyVector &&refs, - const SerializedTensorStore &tensorStore) - : AttributeSaver(std::move(guard), header), - _refs(std::move(refs)), - _tensorStore(tensorStore) -{ -} - - -SerializedTensorAttributeSaver::~SerializedTensorAttributeSaver() -{ -} - - -bool -SerializedTensorAttributeSaver::onSave(IAttributeSaveTarget &saveTarget) -{ - std::unique_ptr<BufferWriter> - datWriter(saveTarget.datWriter().allocBufferWriter()); - const uint32_t docIdLimit(_refs.size()); - for (uint32_t lid = 0; lid < docIdLimit; ++lid) { - auto raw = _tensorStore.getRawBuffer(_refs[lid]); - datWriter->write(&raw.second, sizeof(raw.second)); - if (raw.second != 0) { - datWriter->write(raw.first, raw.second); - } - } - datWriter->flush(); - return true; -} - -} diff --git a/searchlib/src/vespa/searchlib/tensor/serialized_tensor_attribute_saver.h b/searchlib/src/vespa/searchlib/tensor/serialized_tensor_attribute_saver.h deleted file mode 100644 index 1ae2279b893..00000000000 --- a/searchlib/src/vespa/searchlib/tensor/serialized_tensor_attribute_saver.h +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <vespa/searchlib/attribute/attributesaver.h> -#include "tensor_attribute.h" - -namespace search::tensor { - -class SerializedTensorStore; - -/* - * Class for saving a tensor attribute. - */ -class SerializedTensorAttributeSaver : public AttributeSaver { -public: - using RefCopyVector = TensorAttribute::RefCopyVector; -private: - RefCopyVector _refs; - const SerializedTensorStore &_tensorStore; - using GenerationHandler = vespalib::GenerationHandler; - - virtual bool onSave(IAttributeSaveTarget &saveTarget) override; -public: - SerializedTensorAttributeSaver(GenerationHandler::Guard &&guard, - const attribute::AttributeHeader &header, - RefCopyVector &&refs, - const SerializedTensorStore &tensorStore); - - virtual ~SerializedTensorAttributeSaver(); -}; - -} diff --git a/searchlib/src/vespa/searchlib/tensor/serialized_tensor_store.cpp b/searchlib/src/vespa/searchlib/tensor/serialized_tensor_store.cpp deleted file mode 100644 index f55b51875b2..00000000000 --- a/searchlib/src/vespa/searchlib/tensor/serialized_tensor_store.cpp +++ /dev/null @@ -1,108 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "serialized_tensor_store.h" -#include "tensor_deserialize.h" -#include <vespa/eval/eval/value.h> -#include <vespa/eval/eval/value_codec.h> -#include <vespa/vespalib/datastore/datastore.hpp> -#include <vespa/vespalib/objects/nbostream.h> -#include <vespa/vespalib/util/stringfmt.h> - -using vespalib::datastore::Handle; -using vespalib::eval::Value; - -namespace search::tensor { - -constexpr size_t MIN_BUFFER_ARRAYS = 1024; - -SerializedTensorStore::SerializedTensorStore() - : TensorStore(_concreteStore), - _concreteStore(), - _bufferType(RefType::align(1), - MIN_BUFFER_ARRAYS, - RefType::offsetSize() / RefType::align(1)) -{ - _store.addType(&_bufferType); - _store.init_primary_buffers(); -} - -SerializedTensorStore::~SerializedTensorStore() -{ - _store.dropBuffers(); -} - -std::pair<const void *, uint32_t> -SerializedTensorStore::getRawBuffer(RefType ref) const -{ - if (!ref.valid()) { - return std::make_pair(nullptr, 0u); - } - const char *buf = _store.getEntry<char>(ref); - uint32_t len = *reinterpret_cast<const uint32_t *>(buf); - return std::make_pair(buf + sizeof(uint32_t), len); -} - -Handle<char> -SerializedTensorStore::allocRawBuffer(uint32_t size) -{ - if (size == 0) { - return Handle<char>(); - } - size_t extSize = size + sizeof(uint32_t); - size_t bufSize = RefType::align(extSize); - auto result = _concreteStore.rawAllocator<char>(_typeId).alloc(bufSize); - *reinterpret_cast<uint32_t *>(result.data) = size; - char *padWritePtr = result.data + extSize; - for (size_t i = extSize; i < bufSize; ++i) { - *padWritePtr++ = 0; - } - // Hide length of buffer (first 4 bytes) from users of the buffer. - return Handle<char>(result.ref, result.data + sizeof(uint32_t)); -} - -void -SerializedTensorStore::holdTensor(EntryRef ref) -{ - if (!ref.valid()) { - return; - } - RefType iRef(ref); - const char *buf = _store.getEntry<char>(iRef); - uint32_t len = *reinterpret_cast<const uint32_t *>(buf); - _concreteStore.holdElem(ref, len + sizeof(uint32_t)); -} - -TensorStore::EntryRef -SerializedTensorStore::move(EntryRef ref) -{ - if (!ref.valid()) { - return RefType(); - } - auto oldraw = getRawBuffer(ref); - auto newraw = allocRawBuffer(oldraw.second); - memcpy(newraw.data, oldraw.first, oldraw.second); - _concreteStore.holdElem(ref, oldraw.second + sizeof(uint32_t)); - return newraw.ref; -} - -std::unique_ptr<Value> -SerializedTensorStore::getTensor(EntryRef ref) const -{ - auto raw = getRawBuffer(ref); - if (raw.second == 0u) { - return {}; - } - return deserialize_tensor(raw.first, raw.second); -} - -TensorStore::EntryRef -SerializedTensorStore::setTensor(const vespalib::eval::Value &tensor) -{ - vespalib::nbostream stream; - encode_value(tensor, stream); - auto raw = allocRawBuffer(stream.size()); - memcpy(raw.data, stream.peek(), stream.size()); - return raw.ref; -} - -} diff --git a/searchlib/src/vespa/searchlib/tensor/serialized_tensor_store.h b/searchlib/src/vespa/searchlib/tensor/serialized_tensor_store.h deleted file mode 100644 index 0f1b0129ecb..00000000000 --- a/searchlib/src/vespa/searchlib/tensor/serialized_tensor_store.h +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include "tensor_store.h" - -namespace vespalib::eval { struct Value; } - -namespace search::tensor { - -/** - * Class for storing serialized tensors in memory, used by TensorAttribute. - * - * Serialization format is subject to change. Changes to serialization format - * might also require corresponding changes to implemented optimized tensor - * operations that use the serialized tensor as argument. - */ -class SerializedTensorStore : public TensorStore { -public: - using RefType = vespalib::datastore::AlignedEntryRefT<22, 2>; - using DataStoreType = vespalib::datastore::DataStoreT<RefType>; -private: - DataStoreType _concreteStore; - vespalib::datastore::BufferType<char> _bufferType; -public: - SerializedTensorStore(); - - virtual ~SerializedTensorStore(); - - std::pair<const void *, uint32_t> getRawBuffer(RefType ref) const; - - vespalib::datastore::Handle<char> allocRawBuffer(uint32_t size); - - virtual void holdTensor(EntryRef ref) override; - - virtual EntryRef move(EntryRef ref) override; - - std::unique_ptr<vespalib::eval::Value> getTensor(EntryRef ref) const; - - EntryRef setTensor(const vespalib::eval::Value &tensor); -}; - -} diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_attribute.h b/searchlib/src/vespa/searchlib/tensor/tensor_attribute.h index 9d92e226139..601e19e54d1 100644 --- a/searchlib/src/vespa/searchlib/tensor/tensor_attribute.h +++ b/searchlib/src/vespa/searchlib/tensor/tensor_attribute.h @@ -62,6 +62,11 @@ public: virtual void update_tensor(DocId docId, const document::TensorUpdate &update, bool create_empty_if_non_existing); + DistanceMetric distance_metric() const override { + return getConfig().distance_metric(); + } + uint32_t get_num_docs() const override { return getNumDocs(); } + /** * Performs the prepare step in a two-phase operation to set a tensor for a document. * diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.cpp index 822017e0bdf..99a372d9676 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.cpp @@ -22,29 +22,30 @@ namespace search::docsummary { GetDocsumsState::GetDocsumsState(GetDocsumsStateCallback &callback) : _args(), - _docsumbuf(NULL), + _docsumbuf(nullptr), _docsumcnt(0), - _kwExtractor(NULL), - _keywords(NULL), + _kwExtractor(nullptr), + _keywords(nullptr), _callback(callback), _dynteaser(), + _docSumFieldSpaceStore(), _docSumFieldSpace(_docSumFieldSpaceStore, sizeof(_docSumFieldSpaceStore)), // only alloc buffer if needed _attrCtx(), _attributes(), _fieldWriterStates(), - _jsonStringer(), _parsedLocations(), - _summaryFeatures(NULL), + _summaryFeatures(nullptr), _summaryFeaturesCached(false), - _rankFeatures(NULL), - _matching_elements() + _rankFeatures(nullptr), + _matching_elements(), + _jsonStringer() { _dynteaser._docid = static_cast<uint32_t>(-1); _dynteaser._input = static_cast<uint32_t>(-1); _dynteaser._lang = static_cast<uint32_t>(-1); - _dynteaser._config = NULL; - _dynteaser._query = NULL; - _dynteaser._result = NULL; + _dynteaser._config = nullptr; + _dynteaser._query = nullptr; + _dynteaser._result = nullptr; } @@ -52,10 +53,10 @@ GetDocsumsState::~GetDocsumsState() { free(_docsumbuf); free(_keywords); - if (_dynteaser._result != NULL) { + if (_dynteaser._result != nullptr) { juniper::ReleaseResult(_dynteaser._result); } - if (_dynteaser._query != NULL) { + if (_dynteaser._query != nullptr) { juniper::ReleaseQueryHandle(_dynteaser._query); } } @@ -69,6 +70,14 @@ GetDocsumsState::get_matching_elements(const MatchingElementsFields &matching_el return *_matching_elements; } +vespalib::JSONStringer & +GetDocsumsState::jsonStringer() { + if (!_jsonStringer) { + _jsonStringer = std::make_unique<vespalib::JSONStringer>(); + } + return *_jsonStringer; +} + void GetDocsumsState::parse_locations() { diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h b/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h index 46f8e52dd37..db0f8e6e8ad 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h @@ -48,8 +48,6 @@ protected: **/ class GetDocsumsState { -private: - public: const search::attribute::IAttributeVector * getAttribute(size_t index) const { return _attributes[index]; } @@ -72,12 +70,12 @@ public: juniper::Result *_result; // juniper analyze result } _dynteaser; - search::RawBuf _docSumFieldSpace; + char _docSumFieldSpaceStore[2048]; + search::RawBuf _docSumFieldSpace; std::unique_ptr<search::attribute::IAttributeContext> _attrCtx; std::vector<const search::attribute::IAttributeVector *> _attributes; std::vector<std::unique_ptr<DocsumFieldWriterState>> _fieldWriterStates; - vespalib::JSONStringer _jsonStringer; // used by AbsDistanceDFW std::vector<search::common::GeoLocationSpec> _parsedLocations; @@ -97,7 +95,12 @@ public: GetDocsumsState& operator=(const GetDocsumsState &) = delete; GetDocsumsState(GetDocsumsStateCallback &callback); ~GetDocsumsState(); + const MatchingElements &get_matching_elements(const MatchingElementsFields &matching_elems_fields); + vespalib::JSONStringer & jsonStringer(); +private: + // Only used by rank/summary features, so make it lazy + std::unique_ptr<vespalib::JSONStringer> _jsonStringer; }; } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp index 8cc577355cf..0249b96f86b 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp @@ -18,17 +18,10 @@ struct ExplicitItemData { const char *_index; uint32_t _indexlen; - const char *_term; - uint32_t _termlen; uint32_t _weight; ExplicitItemData() - : _index(nullptr), _indexlen(0), _term(nullptr), _termlen(0), _weight(0) - {} - - ExplicitItemData(const char *index, uint32_t indexlen, const char* term, - uint32_t termlen, uint32_t weight = 0) - : _index(index), _indexlen(indexlen), _term(term), _termlen(termlen), _weight(weight) + : _index(nullptr), _indexlen(0), _weight(0) {} }; @@ -86,14 +79,10 @@ TermVisitor::visitProperty(const Property::Value &key, const Property &values) _visitor->VisitPHRASE(&item, phraseLen); s = & values.getAt(index++); while ((*s)[0] != '"') { - data._term = s->c_str(); - data._termlen = s->length(); _visitor->VisitKeyword(&item, s->c_str(), s->length()); s = & values.getAt(index++); } } else { - data._term = s->c_str(); - data._termlen = s->length(); _visitor->VisitKeyword(&item, s->c_str(), s->length()); } } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/getdocsumargs.cpp b/searchsummary/src/vespa/searchsummary/docsummary/getdocsumargs.cpp index 0af92adf2d2..c6148cff8b3 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/getdocsumargs.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/getdocsumargs.cpp @@ -5,38 +5,33 @@ namespace search::docsummary { GetDocsumArgs::GetDocsumArgs() - : _ranking(), - _resultClassName(), + : _resultClassName(), _dumpFeatures(false), _locations_possible(true), - _stackItems(0), _stackDump(), _location(), _timeout(30s), - _propertiesMap() + _highlightTerms() { } GetDocsumArgs::~GetDocsumArgs() = default; void -GetDocsumArgs::initFromDocsumRequest(const search::engine::DocsumRequest &req) +GetDocsumArgs::initFromDocsumRequest(const engine::DocsumRequest &req) { - _ranking = req.ranking; - _dumpFeatures = req.dumpFeatures; - _resultClassName = req.resultClassName; - _stackItems = req.stackItems; - _stackDump = req.stackDump; - _location = req.location; + _dumpFeatures = req.dumpFeatures; + _resultClassName = req.resultClassName; + _stackDump = req.stackDump; + _location = req.location; _locations_possible = true; - _timeout = req.getTimeLeft(); - _propertiesMap = req.propertiesMap; + _timeout = req.getTimeLeft(); + _highlightTerms = req.propertiesMap.highlightTerms(); } void -GetDocsumArgs::SetStackDump(uint32_t stackItems, uint32_t stackDumpLen, const char *stackDump) +GetDocsumArgs::SetStackDump(uint32_t stackDumpLen, const char *stackDump) { - _stackItems = stackItems; _stackDump.resize(stackDumpLen); memcpy(&_stackDump[0], stackDump, _stackDump.size()); } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/getdocsumargs.h b/searchsummary/src/vespa/searchsummary/docsummary/getdocsumargs.h index 0231b004674..17b514883d1 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/getdocsumargs.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/getdocsumargs.h @@ -10,28 +10,22 @@ namespace search::docsummary { class GetDocsumArgs { -public: - typedef engine::PropertiesMap PropsMap; - private: - vespalib::string _ranking; vespalib::string _resultClassName; bool _dumpFeatures; bool _locations_possible; - uint32_t _stackItems; std::vector<char> _stackDump; vespalib::string _location; vespalib::duration _timeout; - PropsMap _propertiesMap; + fef::Properties _highlightTerms; public: GetDocsumArgs(); ~GetDocsumArgs(); void initFromDocsumRequest(const search::engine::DocsumRequest &req); - void SetRankProfile(const vespalib::string &ranking) { _ranking = ranking; } void setResultClassName(vespalib::stringref name) { _resultClassName = name; } - void SetStackDump(uint32_t stackItems, uint32_t stackDumpLen, const char *stackDump); + void SetStackDump(uint32_t stackDumpLen, const char *stackDump); void locations_possible(bool value) { _locations_possible = value; } bool locations_possible() const { return _locations_possible; } const vespalib::string &getLocation() const { return _location; } @@ -47,10 +41,8 @@ public: void dumpFeatures(bool v) { _dumpFeatures = v; } bool dumpFeatures() const { return _dumpFeatures; } - const PropsMap &propertiesMap() const { return _propertiesMap; } - - const search::fef::Properties &highlightTerms() const { - return _propertiesMap.highlightTerms(); + const fef::Properties &highlightTerms() const { + return _highlightTerms; } }; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.cpp index 71febe10e59..d0ecb63a9a1 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.cpp @@ -43,7 +43,7 @@ RankFeaturesDFW::insertField(uint32_t docid, GetDocsumsState *state, } return; } - vespalib::JSONStringer & json(state->_jsonStringer); + vespalib::JSONStringer & json(state->jsonStringer()); if (values != nullptr) { json.clear(); json.beginObject(); diff --git a/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.cpp index c5b027a372d..425faff6a67 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.cpp @@ -55,7 +55,7 @@ SummaryFeaturesDFW::insertField(uint32_t docid, GetDocsumsState *state, ResType } return; } - vespalib::JSONStringer & json(state->_jsonStringer); + vespalib::JSONStringer & json(state->jsonStringer()); if (values != nullptr) { json.clear(); json.beginObject(); diff --git a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp index 6f4c17f179b..5e1e95b4681 100644 --- a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp +++ b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp @@ -265,11 +265,6 @@ void SearchVisitor::init(const Parameters & params) LOG(debug, "No rank properties received"); } - if (params.lookup("rankprofile", valueRef)) { - vespalib::string tmp(valueRef.data(), valueRef.size()); - _summaryGenerator.getDocsumState()._args.SetRankProfile(tmp); - } - vespalib::string location; if (params.lookup("location", valueRef)) { location = vespalib::string(valueRef.data(), valueRef.size()); @@ -299,7 +294,7 @@ void SearchVisitor::init(const Parameters & params) int stackCount = 0; if (params.get("querystackcount", stackCount)) { - _summaryGenerator.getDocsumState()._args.SetStackDump(stackCount, queryBlob.size(), (const char*)queryBlob.data()); + _summaryGenerator.getDocsumState()._args.SetStackDump(queryBlob.size(), (const char*)queryBlob.data()); } else { LOG(warning, "Request without query stack count"); } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/SignedIdentityDocumentEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/SignedIdentityDocumentEntity.java index 9cc21925f52..e6b74d9df4c 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/SignedIdentityDocumentEntity.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/SignedIdentityDocumentEntity.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.athenz.identityprovider.api.bindings; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; -import com.yahoo.restapi.RestApi; import java.time.Instant; import java.util.Objects; @@ -14,7 +13,7 @@ import java.util.Set; * @author bjorncs */ @JsonIgnoreProperties(ignoreUnknown = true) -public class SignedIdentityDocumentEntity implements RestApi.JacksonResponseEntity { +public class SignedIdentityDocumentEntity { @JsonProperty("signature") public final String signature; @JsonProperty("signing-key-version") public final int signingKeyVersion; diff --git a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/JunitRunner.java b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/JunitRunner.java index fc4b3e74b29..ae81b36fe59 100644 --- a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/JunitRunner.java +++ b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/JunitRunner.java @@ -19,7 +19,6 @@ import org.junit.platform.launcher.LauncherDiscoveryRequest; import org.junit.platform.launcher.core.LauncherConfig; import org.junit.platform.launcher.core.LauncherDiscoveryRequestBuilder; import org.junit.platform.launcher.core.LauncherFactory; -import org.junit.platform.launcher.listeners.LoggingListener; import org.junit.platform.launcher.listeners.SummaryGeneratingListener; import org.osgi.framework.Bundle; import org.osgi.framework.BundleContext; @@ -103,16 +102,15 @@ public class JunitRunner extends AbstractComponent implements TestRunner { } execution = CompletableFuture.supplyAsync(() -> launchJunit(loadClasses(testBundle.get(), testDescriptor.get(), category))); } catch (Exception e) { - execution = createReportWithFailedInitialization(e); + execution = CompletableFuture.completedFuture(createReportWithFailedInitialization(e)); } } - private static Future<TestReport> createReportWithFailedInitialization(Exception exception) { + private static TestReport createReportWithFailedInitialization(Exception exception) { TestReport.Failure failure = new TestReport.Failure("init", exception); - return CompletableFuture.completedFuture( - new TestReport.Builder() + return new TestReport.Builder() .withFailures(List.of(failure)) - .build()); + .build(); } @Override @@ -178,7 +176,7 @@ public class JunitRunner extends AbstractComponent implements TestRunner { // Create log listener: var logLines = new ArrayList<LogRecord>(); - var logListener = LoggingListener.forBiConsumer((t, m) -> log(logLines, m.get(), t)); + var logListener = VespaJunitLogListener.forBiConsumer((t, m) -> log(logLines, m.get(), t)); // Create a summary listener: var summaryListener = new SummaryGeneratingListener(); launcher.registerTestExecutionListeners(logListener, summaryListener); @@ -223,7 +221,8 @@ public class JunitRunner extends AbstractComponent implements TestRunner { } } catch (InterruptedException|ExecutionException e) { logger.log(Level.WARNING, "Error while getting test report", e); - return LegacyTestRunner.Status.ERROR; + // Return FAILURE to enforce getting the test report from the caller. + return LegacyTestRunner.Status.FAILURE; } } @@ -234,7 +233,9 @@ public class JunitRunner extends AbstractComponent implements TestRunner { return execution.get(); } catch (Exception e) { logger.log(Level.WARNING, "Error getting test report", e); - return null; + // Likely this is something wrong with the provided test bundle. Create a test report + // and present in the console to enable tenants to act on it. + return createReportWithFailedInitialization(e); } } else { return null; diff --git a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/VespaJunitLogListener.java b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/VespaJunitLogListener.java new file mode 100644 index 00000000000..a3c64a63365 --- /dev/null +++ b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/VespaJunitLogListener.java @@ -0,0 +1,59 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.testrunner; + +import org.junit.platform.commons.util.Preconditions; +import org.junit.platform.engine.TestExecutionResult; +import org.junit.platform.engine.reporting.ReportEntry; +import org.junit.platform.launcher.TestExecutionListener; +import org.junit.platform.launcher.TestIdentifier; + +import java.util.function.BiConsumer; +import java.util.function.Supplier; + +public class VespaJunitLogListener implements TestExecutionListener { + + public static VespaJunitLogListener forBiConsumer(BiConsumer<Throwable, Supplier<String>> logger) { + return new VespaJunitLogListener(logger); + } + + private final BiConsumer<Throwable, Supplier<String>> logger; + + private VespaJunitLogListener(BiConsumer<Throwable, Supplier<String>> logger) { + this.logger = Preconditions.notNull(logger, "logger must not be null"); + } + + @Override + public void dynamicTestRegistered(TestIdentifier testIdentifier) { + log("Registered dynamic test: %s - %s", testIdentifier.getDisplayName(), testIdentifier.getUniqueId()); + } + + @Override + public void executionStarted(TestIdentifier testIdentifier) { + log("Test started: %s - %s", testIdentifier.getDisplayName(), testIdentifier.getUniqueId()); + } + + @Override + public void executionSkipped(TestIdentifier testIdentifier, String reason) { + log("Test skipped: %s - %s - %s", testIdentifier.getDisplayName(), testIdentifier.getUniqueId(), reason); + } + + @Override + public void executionFinished(TestIdentifier testIdentifier, TestExecutionResult testExecutionResult) { + logWithThrowable("Test completed: %s - %s - %s", testExecutionResult.getThrowable().orElse(null), + testIdentifier.getDisplayName(), testIdentifier.getUniqueId(), testExecutionResult); + } + + @Override + public void reportingEntryPublished(TestIdentifier testIdentifier, ReportEntry entry) { + log("[" + testIdentifier.getDisplayName() + "]: " + entry.toString()); + } + + private void log(String message, Object... args) { + logWithThrowable(message, null, args); + } + + private void logWithThrowable(String message, Throwable t, Object... args) { + this.logger.accept(t, () -> String.format(message, args)); + } +} diff --git a/vespajlib/pom.xml b/vespajlib/pom.xml index f8be4c6f8f7..af3db750a71 100644 --- a/vespajlib/pom.xml +++ b/vespajlib/pom.xml @@ -68,11 +68,6 @@ <scope>test</scope> </dependency> <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - <scope>test</scope> - </dependency> - <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>testutil</artifactId> <version>${project.version}</version> @@ -84,6 +79,11 @@ <scope>test</scope> </dependency> <dependency> + <groupId>org.junit.vintage</groupId> + <artifactId>junit-vintage-engine</artifactId> + <scope>test</scope> + </dependency> + <dependency> <groupId>org.junit.jupiter</groupId> <artifactId>junit-jupiter</artifactId> <scope>test</scope> diff --git a/vespajlib/src/main/java/com/yahoo/collections/CopyOnWriteHashMap.java b/vespajlib/src/main/java/com/yahoo/collections/CopyOnWriteHashMap.java index 7db43a7442a..857b7cc6acd 100644 --- a/vespajlib/src/main/java/com/yahoo/collections/CopyOnWriteHashMap.java +++ b/vespajlib/src/main/java/com/yahoo/collections/CopyOnWriteHashMap.java @@ -1,8 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.collections; -import com.google.common.annotations.Beta; - import java.util.AbstractMap; import java.util.AbstractSet; import java.util.HashMap; diff --git a/vespajlib/src/main/java/com/yahoo/text/SimpleMapParser.java b/vespajlib/src/main/java/com/yahoo/text/SimpleMapParser.java index 8f4b49f6ca4..d71b73c6f9f 100644 --- a/vespajlib/src/main/java/com/yahoo/text/SimpleMapParser.java +++ b/vespajlib/src/main/java/com/yahoo/text/SimpleMapParser.java @@ -1,9 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.text; -import java.util.HashMap; -import java.util.Map; - /** * <p>Superclasses of parsers of a map represented textually as * <code>{key1:value1,"anystringkey":value2,'anystringkey2':value3 ...}</code>. diff --git a/vespalib/src/tests/btree/btree_test.cpp b/vespalib/src/tests/btree/btree_test.cpp index e1a9fa98a21..82fa4b6b50a 100644 --- a/vespalib/src/tests/btree/btree_test.cpp +++ b/vespalib/src/tests/btree/btree_test.cpp @@ -1549,6 +1549,38 @@ inc_generation(GenerationHandler &g, Tree &t) s.trimHoldLists(g.getFirstUsedGeneration()); } +template <typename Tree> +void +make_iterators(Tree& t, std::vector<int>& list, std::vector<typename Tree::ConstIterator>& iterators) +{ + for (auto key : list) { + iterators.emplace_back(t.lowerBound(key)); + } + iterators.emplace_back(t.lowerBound(300)); +} + +class KeyRangeValidator +{ + std::vector<int> &_list; + size_t _start_pos; + size_t _end_pos; + size_t _curr_pos; +public: + KeyRangeValidator(std::vector<int> &list, size_t start_pos, size_t end_pos) + : _list(list), + _start_pos(start_pos), + _end_pos(end_pos), + _curr_pos(start_pos) + { + } + void operator()(int key) { + assert(_curr_pos < _list.size()); + EXPECT_EQ(key, _list[_curr_pos]); + ++_curr_pos; + } + size_t curr_pos() const noexcept { return _curr_pos; } +}; + } TEST_F(BTreeTest, require_that_compaction_works) @@ -1557,7 +1589,9 @@ TEST_F(BTreeTest, require_that_compaction_works) GenerationHandler g; Tree t; std::vector<int> before_list; + std::vector<typename Tree::ConstIterator> before_iterators; std::vector<int> after_list; + std::vector<typename Tree::ConstIterator> after_iterators; for (uint32_t i = 1; i < 256; ++i) { t.insert(i, 101); } @@ -1565,14 +1599,39 @@ TEST_F(BTreeTest, require_that_compaction_works) t.remove(i); } inc_generation(g, t); + auto guard = g.takeGuard(); auto memory_usage_before = t.getAllocator().getMemoryUsage(); t.foreach_key([&before_list](int key) { before_list.emplace_back(key); }); - t.compact_worst(); + make_iterators(t, before_list, before_iterators); + for (int i = 0; i < 15; ++i) { + t.compact_worst(); + } inc_generation(g, t); auto memory_usage_after = t.getAllocator().getMemoryUsage(); t.foreach_key([&after_list](int key) { after_list.emplace_back(key); }); + make_iterators(t, after_list, after_iterators); EXPECT_LT(memory_usage_after.deadBytes(), memory_usage_before.deadBytes()); EXPECT_EQ(before_list, after_list); + EXPECT_EQ(before_iterators.size(), after_iterators.size()); + for (size_t i = 0; i < before_iterators.size(); ++i) { + for (size_t j = 0; j < after_iterators.size(); ++j) { + EXPECT_EQ(before_iterators[i] == after_iterators[j], i == j); + EXPECT_EQ(before_iterators[i] - after_iterators[j], static_cast<ssize_t>(i - j)); + EXPECT_EQ(after_iterators[j] - before_iterators[i], static_cast<ssize_t>(j - i)); + if (i <= j) { + KeyRangeValidator validate_keys(before_list, i, j); + EXPECT_EQ(i, validate_keys.curr_pos()); + before_iterators[i].foreach_key_range(after_iterators[j], [&validate_keys](int key) { validate_keys(key); }); + EXPECT_EQ(j, validate_keys.curr_pos()); + } + if (j <= i) { + KeyRangeValidator validate_keys(before_list, j, i); + EXPECT_EQ(j, validate_keys.curr_pos()); + after_iterators[j].foreach_key_range(before_iterators[i], [&validate_keys](int key) { validate_keys(key); }); + EXPECT_EQ(i, validate_keys.curr_pos()); + } + } + } } } diff --git a/vespalib/src/tests/datastore/fixed_size_hash_map/fixed_size_hash_map_test.cpp b/vespalib/src/tests/datastore/fixed_size_hash_map/fixed_size_hash_map_test.cpp index b929b248e33..b32a9baf90c 100644 --- a/vespalib/src/tests/datastore/fixed_size_hash_map/fixed_size_hash_map_test.cpp +++ b/vespalib/src/tests/datastore/fixed_size_hash_map/fixed_size_hash_map_test.cpp @@ -108,7 +108,7 @@ DataStoreFixedSizeHashTest::insert(uint32_t key) { MyCompare comp(_store, key); std::function<EntryRef(void)> insert_entry([this, key]() -> EntryRef { return _allocator.allocate(key); }); - auto& result = _hash_map->add(comp, EntryRef(), insert_entry); + auto& result = _hash_map->add(_hash_map->get_comp(comp), insert_entry); auto ref = result.first.load_relaxed(); auto &wrapped_entry = _allocator.get_wrapped(ref); EXPECT_EQ(key, wrapped_entry.value()); @@ -118,7 +118,7 @@ void DataStoreFixedSizeHashTest::remove(uint32_t key) { MyCompare comp(_store, key); - auto result = _hash_map->remove(comp, EntryRef()); + auto result = _hash_map->remove(_hash_map->get_comp(comp)); if (result != nullptr) { auto ref = result->first.load_relaxed(); auto &wrapped_entry = _allocator.get_wrapped(ref); @@ -131,7 +131,7 @@ bool DataStoreFixedSizeHashTest::has_key(uint32_t key) { MyCompare comp(_store, key); - auto result = _hash_map->find(comp, EntryRef()); + auto result = _hash_map->find(_hash_map->get_comp(comp)); if (result != nullptr) { auto ref = result->first.load_relaxed(); auto& wrapped_entry = _allocator.get_wrapped(ref); @@ -271,7 +271,7 @@ TEST_F(DataStoreFixedSizeHashTest, lookups_works_after_insert_and_remove) } for (auto &kv : expected) { MyCompare comp(_store, kv.first); - EXPECT_EQ(kv.second, _hash_map->find(comp, EntryRef()) != nullptr); + EXPECT_EQ(kv.second, _hash_map->find(_hash_map->get_comp(comp)) != nullptr); } } diff --git a/vespalib/src/tests/datastore/sharded_hash_map/sharded_hash_map_test.cpp b/vespalib/src/tests/datastore/sharded_hash_map/sharded_hash_map_test.cpp index 9f11b96e672..da9bc1284fa 100644 --- a/vespalib/src/tests/datastore/sharded_hash_map/sharded_hash_map_test.cpp +++ b/vespalib/src/tests/datastore/sharded_hash_map/sharded_hash_map_test.cpp @@ -50,6 +50,7 @@ struct DataStoreShardedHashTest : public ::testing::Test void read_work(uint32_t cnt); void read_work(); void write_work(uint32_t cnt); + void populate_sample_data(); }; @@ -173,6 +174,13 @@ DataStoreShardedHashTest::write_work(uint32_t cnt) LOG(info, "done %u write work", cnt); } +void +DataStoreShardedHashTest::populate_sample_data() +{ + for (uint32_t i = 0; i < 50; ++i) { + insert(i); + } +} TEST_F(DataStoreShardedHashTest, single_threaded_reader_without_updates) { @@ -216,4 +224,57 @@ TEST_F(DataStoreShardedHashTest, memory_usage_is_reported) EXPECT_LT(0, usage.allocatedBytesOnHold()); } +TEST_F(DataStoreShardedHashTest, foreach_key_works) +{ + populate_sample_data(); + std::vector<uint32_t> keys; + _hash_map.foreach_key([this, &keys](EntryRef ref) { keys.emplace_back(_allocator.get_wrapped(ref).value()); }); + std::sort(keys.begin(), keys.end()); + EXPECT_EQ(50, keys.size()); + for (uint32_t i = 0; i < 50; ++i) { + EXPECT_EQ(i, keys[i]); + } +} + +TEST_F(DataStoreShardedHashTest, move_keys_works) +{ + populate_sample_data(); + std::vector<EntryRef> refs; + _hash_map.foreach_key([&refs](EntryRef ref) { refs.emplace_back(ref); }); + std::vector<EntryRef> new_refs; + _hash_map.move_keys([this, &new_refs](EntryRef ref) { auto new_ref = _allocator.move(ref); _allocator.hold(ref); new_refs.emplace_back(new_ref); return new_ref; }); + std::vector<EntryRef> verify_new_refs; + _hash_map.foreach_key([&verify_new_refs](EntryRef ref) { verify_new_refs.emplace_back(ref); }); + EXPECT_EQ(50u, refs.size()); + EXPECT_NE(refs, new_refs); + EXPECT_EQ(new_refs, verify_new_refs); + for (uint32_t i = 0; i < 50; ++i) { + EXPECT_NE(refs[i], new_refs[i]); + auto value = _allocator.get_wrapped(refs[i]).value(); + auto new_value = _allocator.get_wrapped(refs[i]).value(); + EXPECT_EQ(value, new_value); + } +} + +TEST_F(DataStoreShardedHashTest, normalize_values_works) +{ + populate_sample_data(); + for (uint32_t i = 0; i < 50; ++i) { + MyCompare comp(_store, i); + auto result = _hash_map.find(comp, EntryRef()); + ASSERT_NE(result, nullptr); + EXPECT_EQ(i, _allocator.get_wrapped(result->first.load_relaxed()).value()); + result->second.store_relaxed(EntryRef(i + 200)); + } + _hash_map.normalize_values([](EntryRef ref) noexcept { return EntryRef(ref.ref() + 300); }); + for (uint32_t i = 0; i < 50; ++i) { + MyCompare comp(_store, i); + auto result = _hash_map.find(comp, EntryRef()); + ASSERT_NE(result, nullptr); + EXPECT_EQ(i, _allocator.get_wrapped(result->first.load_relaxed()).value()); + ASSERT_EQ(i + 500, result->second.load_relaxed().ref()); + result->second.store_relaxed(EntryRef()); + } +} + GTEST_MAIN_RUN_ALL_TESTS() diff --git a/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp b/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp index 3e8b144cf32..e61713e40d8 100644 --- a/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp +++ b/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp @@ -150,9 +150,19 @@ TestBase<UniqueStoreTypeAndDictionaryType>::TestBase() { switch (UniqueStoreTypeAndDictionaryType::dictionary_type) { case DictionaryType::BTREE: + EXPECT_TRUE(store.get_dictionary().get_has_btree_dictionary()); + EXPECT_FALSE(store.get_dictionary().get_has_hash_dictionary()); break; - default: + case DictionaryType::BTREE_AND_HASH: store.set_dictionary(std::make_unique<UniqueStoreDictionary<uniquestore::DefaultDictionary, IUniqueStoreDictionary, ShardedHashMap>>(std::make_unique<CompareType>(store.get_data_store()))); + EXPECT_TRUE(store.get_dictionary().get_has_btree_dictionary()); + EXPECT_TRUE(store.get_dictionary().get_has_hash_dictionary()); + break; + case DictionaryType::HASH: + default: + store.set_dictionary(std::make_unique<UniqueStoreDictionary<NoBTreeDictionary, IUniqueStoreDictionary, ShardedHashMap>>(std::make_unique<CompareType>(store.get_data_store()))); + EXPECT_FALSE(store.get_dictionary().get_has_btree_dictionary()); + EXPECT_TRUE(store.get_dictionary().get_has_hash_dictionary()); } } @@ -174,67 +184,97 @@ std::vector<const char *> TestBaseValues<CStringUniqueStore>::values{ "aa", "bbb template <> std::vector<double> TestBaseValues<DoubleUniqueStore>::values{ 10.0, 20.0, 30.0, 10.0 }; -struct OrderedNumberUniqueStore +struct BTreeNumberUniqueStore { using UniqueStoreType = NumberUniqueStore; static constexpr DictionaryType dictionary_type = DictionaryType::BTREE; }; -struct OrderedStringUniqueStore +struct BTreeStringUniqueStore { using UniqueStoreType = StringUniqueStore; static constexpr DictionaryType dictionary_type = DictionaryType::BTREE; }; -struct OrderedCStringUniqueStore +struct BTreeCStringUniqueStore { using UniqueStoreType = CStringUniqueStore; static constexpr DictionaryType dictionary_type = DictionaryType::BTREE; }; -struct OrderedDoubleUniqueStore +struct BTreeDoubleUniqueStore { using UniqueStoreType = DoubleUniqueStore; static constexpr DictionaryType dictionary_type = DictionaryType::BTREE; }; -struct OrderedSmallOffsetNumberUniqueStore +struct BTreeSmallOffsetNumberUniqueStore { using UniqueStoreType = SmallOffsetNumberUniqueStore; static constexpr DictionaryType dictionary_type = DictionaryType::BTREE; }; -struct UnorderedNumberUniqueStore +struct HybridNumberUniqueStore { using UniqueStoreType = NumberUniqueStore; static constexpr DictionaryType dictionary_type = DictionaryType::BTREE_AND_HASH; }; -struct UnorderedStringUniqueStore +struct HybridStringUniqueStore { using UniqueStoreType = StringUniqueStore; static constexpr DictionaryType dictionary_type = DictionaryType::BTREE_AND_HASH; }; -struct UnorderedCStringUniqueStore +struct HybridCStringUniqueStore { using UniqueStoreType = CStringUniqueStore; static constexpr DictionaryType dictionary_type = DictionaryType::BTREE_AND_HASH; }; -struct UnorderedDoubleUniqueStore +struct HybridDoubleUniqueStore { using UniqueStoreType = DoubleUniqueStore; static constexpr DictionaryType dictionary_type = DictionaryType::BTREE_AND_HASH; }; -struct UnorderedSmallOffsetNumberUniqueStore +struct HybridSmallOffsetNumberUniqueStore { using UniqueStoreType = SmallOffsetNumberUniqueStore; static constexpr DictionaryType dictionary_type = DictionaryType::BTREE_AND_HASH; }; -using UniqueStoreTestTypes = ::testing::Types<OrderedNumberUniqueStore, OrderedStringUniqueStore, OrderedCStringUniqueStore, OrderedDoubleUniqueStore, UnorderedNumberUniqueStore, UnorderedStringUniqueStore, UnorderedCStringUniqueStore, UnorderedDoubleUniqueStore>; +struct HashNumberUniqueStore +{ + using UniqueStoreType = NumberUniqueStore; + static constexpr DictionaryType dictionary_type = DictionaryType::HASH; +}; + +struct HashStringUniqueStore +{ + using UniqueStoreType = StringUniqueStore; + static constexpr DictionaryType dictionary_type = DictionaryType::HASH; +}; + +struct HashCStringUniqueStore +{ + using UniqueStoreType = CStringUniqueStore; + static constexpr DictionaryType dictionary_type = DictionaryType::HASH; +}; + +struct HashDoubleUniqueStore +{ + using UniqueStoreType = DoubleUniqueStore; + static constexpr DictionaryType dictionary_type = DictionaryType::HASH; +}; + +struct HashSmallOffsetNumberUniqueStore +{ + using UniqueStoreType = SmallOffsetNumberUniqueStore; + static constexpr DictionaryType dictionary_type = DictionaryType::HASH; +}; + +using UniqueStoreTestTypes = ::testing::Types<BTreeNumberUniqueStore, BTreeStringUniqueStore, BTreeCStringUniqueStore, BTreeDoubleUniqueStore, HybridNumberUniqueStore, HybridStringUniqueStore, HybridCStringUniqueStore, HybridDoubleUniqueStore, HashNumberUniqueStore, HashStringUniqueStore, HashCStringUniqueStore, HashDoubleUniqueStore>; VESPA_GTEST_TYPED_TEST_SUITE(TestBase, UniqueStoreTestTypes); // Disable warnings emitted by gtest generated files when using typed tests @@ -243,11 +283,11 @@ VESPA_GTEST_TYPED_TEST_SUITE(TestBase, UniqueStoreTestTypes); #pragma GCC diagnostic ignored "-Wsuggest-override" #endif -using NumberTest = TestBase<OrderedNumberUniqueStore>; -using StringTest = TestBase<OrderedStringUniqueStore>; -using CStringTest = TestBase<OrderedCStringUniqueStore>; -using DoubleTest = TestBase<OrderedDoubleUniqueStore>; -using SmallOffsetNumberTest = TestBase<OrderedSmallOffsetNumberUniqueStore>; +using NumberTest = TestBase<BTreeNumberUniqueStore>; +using StringTest = TestBase<BTreeStringUniqueStore>; +using CStringTest = TestBase<BTreeCStringUniqueStore>; +using DoubleTest = TestBase<BTreeDoubleUniqueStore>; +using SmallOffsetNumberTest = TestBase<BTreeSmallOffsetNumberUniqueStore>; TEST(UniqueStoreTest, trivial_and_non_trivial_types_are_tested) { diff --git a/vespalib/src/tests/datastore/unique_store_dictionary/unique_store_dictionary_test.cpp b/vespalib/src/tests/datastore/unique_store_dictionary/unique_store_dictionary_test.cpp index 2c1b4ad3cf1..ce1ebe395ce 100644 --- a/vespalib/src/tests/datastore/unique_store_dictionary/unique_store_dictionary_test.cpp +++ b/vespalib/src/tests/datastore/unique_store_dictionary/unique_store_dictionary_test.cpp @@ -41,7 +41,7 @@ public: template <typename UniqueStoreDictionaryType> struct DictionaryReadTest : public ::testing::Test { UniqueStoreDictionaryType dict; - IUniqueStoreDictionary::ReadSnapshot::UP snapshot; + std::unique_ptr<IUniqueStoreDictionaryReadSnapshot> snapshot; DictionaryReadTest() : dict(std::make_unique<Comparator>(0)), @@ -56,10 +56,12 @@ struct DictionaryReadTest : public ::testing::Test { void take_snapshot() { dict.freeze(); snapshot = dict.get_read_snapshot(); + snapshot->fill(); + snapshot->sort(); } }; -using DictionaryReadTestTypes = ::testing::Types<DefaultUniqueStoreDictionary, UniqueStoreDictionary<DefaultDictionary, IUniqueStoreDictionary, ShardedHashMap>>; +using DictionaryReadTestTypes = ::testing::Types<DefaultUniqueStoreDictionary, UniqueStoreDictionary<DefaultDictionary, IUniqueStoreDictionary, ShardedHashMap>, UniqueStoreDictionary<NoBTreeDictionary, IUniqueStoreDictionary, ShardedHashMap>>; VESPA_GTEST_TYPED_TEST_SUITE(DictionaryReadTest, DictionaryReadTestTypes); // Disable warnings emitted by gtest generated files when using typed tests @@ -79,6 +81,9 @@ TYPED_TEST(DictionaryReadTest, can_count_occurrences_of_a_key) TYPED_TEST(DictionaryReadTest, can_count_occurrences_of_keys_in_a_range) { + if (!this->dict.get_has_btree_dictionary()) { + return; + } this->add(3).add(5).add(7).add(9).take_snapshot(); EXPECT_EQ(1, this->snapshot->count_in_range(Comparator(3), Comparator(3))); EXPECT_EQ(1, this->snapshot->count_in_range(Comparator(3), Comparator(4))); diff --git a/vespalib/src/tests/require/require_test.cpp b/vespalib/src/tests/require/require_test.cpp index a97e7a362cc..65f4d049843 100644 --- a/vespalib/src/tests/require/require_test.cpp +++ b/vespalib/src/tests/require/require_test.cpp @@ -3,6 +3,8 @@ #include <vespa/vespalib/util/require.h> #include <vespa/vespalib/gtest/gtest.h> +using E = vespalib::RequireFailedException; + //----------------------------------------------------------------------------- void pass_require() { @@ -38,7 +40,6 @@ void fail_require_eq() { } TEST(RequireTest, require_can_fail) { - using E = vespalib::RequireFailedException; EXPECT_THROW( { try { fail_require(); } @@ -52,7 +53,6 @@ TEST(RequireTest, require_can_fail) { } TEST(RequireTest, require_eq_can_fail) { - using E = vespalib::RequireFailedException; EXPECT_THROW( { try { fail_require_eq(); } @@ -89,4 +89,71 @@ TEST(RequireTest, require_eq_can_be_constexpr) { //----------------------------------------------------------------------------- +TEST(RequireTest, require_eq_implicit_approx_for_double) { + double foo = 1.0; + double bar = 1.0 + 1e-9; + REQUIRE(foo != bar); + REQUIRE_EQ(foo, bar); +} + +//----------------------------------------------------------------------------- + +struct MyA { + int a; + int b; + template <typename T> + bool operator==(const T &rhs) const { + return (a == rhs.a) && (b == rhs.b); + } +}; +std::ostream &operator<<(std::ostream &out, const MyA &a) { + out << "MyA { a: " << a.a << ", b: " << a.b << " }"; + return out; +} + +struct MyB { + int a; + int b; +}; + +struct MyC { + char a; + ssize_t b; +}; + +TEST(RequireTest, explicit_compare_and_print) { + MyA x{5, 7}; + MyA y{5, 6}; + REQUIRE_EQ(x, x); + EXPECT_THROW(REQUIRE_EQ(x, y), E); +} + +TEST(RequireTest, implicit_compare_and_print) { + MyB x{5, 7}; + MyB y{5, 6}; + REQUIRE_EQ(x, x); + EXPECT_THROW(REQUIRE_EQ(x, y), E); +} + +TEST(RequireTest, comparable_but_unprintable) { + MyA x{5, 7}; + MyC y{5, 6}; + REQUIRE_EQ(x, x); + EXPECT_THROW(REQUIRE_EQ(x, y), E); +} + +// manual test for uncompilable code (uncomparable values) +TEST(RequireTest, uncomment_to_manually_check_uncompilable_code) { + MyA a{5, 7}; + MyB b{5, 7}; + MyC c{5, 7}; + (void) a; + (void) b; + (void) c; + // REQUIRE_EQ(b, a); + // REQUIRE_EQ(c, c); +} + +//----------------------------------------------------------------------------- + GTEST_MAIN_RUN_ALL_TESTS() diff --git a/vespalib/src/tests/util/bfloat16/bfloat16_test.cpp b/vespalib/src/tests/util/bfloat16/bfloat16_test.cpp index 5d70c95b1d9..d8bb93d7972 100644 --- a/vespalib/src/tests/util/bfloat16/bfloat16_test.cpp +++ b/vespalib/src/tests/util/bfloat16/bfloat16_test.cpp @@ -5,7 +5,7 @@ #include <vespa/vespalib/gtest/gtest.h> #include <stdio.h> #include <cmath> -#include <cmath> +#include <cstring> #include <vector> using namespace vespalib; @@ -165,4 +165,90 @@ TEST(BFloat16Test, check_special_values) { EXPECT_EQ(memcmp(&f_snan, &f_from_b_snan, sizeof(float)), 0); } +#include <onnxruntime/core/framework/endian.h> + +// extract from onnx-internal header file: +namespace onnxruntime { + +//BFloat16 +struct BFloat16 { + uint16_t val{0}; + explicit BFloat16() = default; + explicit BFloat16(uint16_t v) : val(v) {} + explicit BFloat16(float v) { + if (endian::native == endian::little) { + std::memcpy(&val, reinterpret_cast<char*>(&v) + sizeof(uint16_t), sizeof(uint16_t)); + } else { + std::memcpy(&val, &v, sizeof(uint16_t)); + } + } + + float ToFloat() const { + float result; + char* const first = reinterpret_cast<char*>(&result); + char* const second = first + sizeof(uint16_t); + if (endian::native == endian::little) { + std::memset(first, 0, sizeof(uint16_t)); + std::memcpy(second, &val, sizeof(uint16_t)); + } else { + std::memcpy(first, &val, sizeof(uint16_t)); + std::memset(second, 0, sizeof(uint16_t)); + } + return result; + } +}; + +} // namespace onnxruntime + +TEST(OnnxBFloat16Test, has_same_encoding) { + EXPECT_EQ(sizeof(vespalib::BFloat16), sizeof(onnxruntime::BFloat16)); + EXPECT_EQ(sizeof(vespalib::BFloat16), sizeof(uint16_t)); + EXPECT_EQ(sizeof(onnxruntime::BFloat16), sizeof(uint16_t)); + vespalib::BFloat16 our_value; + uint32_t ok_count = 0; + uint32_t nan_count = 0; + for (uint32_t i = 0; i < (1u << 16u); ++i) { + uint16_t bits = i; + our_value.assign_bits(bits); + onnxruntime::BFloat16 their_value(bits); + if (our_value.get_bits() != bits) { + printf("bad bits %04x -> %04x (vespalib)\n", bits, our_value.get_bits()); + printf("onnx converts -> %04x\n", their_value.val); + EXPECT_EQ(our_value.get_bits(), their_value.val); + continue; + } + EXPECT_EQ(their_value.val, bits); + EXPECT_EQ(memcmp(&our_value, &their_value, sizeof(our_value)), 0); + if (their_value.val != bits) { + printf("bad bits %04x -> %04x (onnx)\n", bits, their_value.val); + continue; + } + EXPECT_EQ(our_value.get_bits(), their_value.val); + if (our_value.get_bits() != their_value.val) { + printf("vespalib bits %04x != %04x onnx bits\n", our_value.get_bits(), their_value.val); + printf("corresponds to floats %g and %g\n", our_value.to_float(), their_value.ToFloat()); + continue; + } + float our_float = our_value.to_float(); + float their_float = their_value.ToFloat(); + EXPECT_EQ(std::isnan(our_float), std::isnan(their_float)); + if (std::isnan(our_float) && std::isnan(their_float)) { + ++nan_count; + continue; + } + if (our_float != their_float) { + printf("bits %04x as float differs: vespalib %g != %g onnx\n", bits, our_value.to_float(), their_value.ToFloat()); + } else { + ++ok_count; + } + EXPECT_EQ(our_float, their_float); + vespalib::BFloat16 our_back(our_float); + onnxruntime::BFloat16 their_back(their_float); + EXPECT_EQ(our_back.get_bits(), their_back.val); + } + printf("normal floats behave equally OK in both vespalib and onnx: %d (0x%04x)\n", ok_count, ok_count); + printf("floats that are NaN in both vespalib and onnx: %d (0x%04x)\n", nan_count, nan_count); + printf("total count (OK + NaN): %d (0x%04x)\n", ok_count + nan_count, ok_count + nan_count); +} + GTEST_MAIN_RUN_ALL_TESTS() diff --git a/vespalib/src/vespa/vespalib/btree/btreeiterator.h b/vespalib/src/vespa/vespalib/btree/btreeiterator.h index bbee8746d6f..21d232db31a 100644 --- a/vespalib/src/vespa/vespalib/btree/btreeiterator.h +++ b/vespalib/src/vespa/vespalib/btree/btreeiterator.h @@ -348,10 +348,20 @@ public: bool operator==(const BTreeIteratorBase & rhs) const { - if (_leaf.getNode() != rhs._leaf.getNode() || - _leaf.getIdx() != rhs._leaf.getIdx()) { + if (_leaf.getIdx() != rhs._leaf.getIdx()) { return false; } + if (_leaf.getNode() == rhs._leaf.getNode()) { + return true; + } + if ((_leaf.getNode() == nullptr) || (rhs._leaf.getNode() == nullptr) || (_pathSize != rhs._pathSize)) { + return false; + } + for (uint32_t level = 0; level < _pathSize; ++level) { + if (_path[level].getIdx() != rhs._path[level].getIdx()) { + return false; + } + } return true; } @@ -521,7 +531,6 @@ public: uint32_t eidx; do { --level; - assert(_path[level].getNode() == end_itr._path[level].getNode()); idx = _path[level].getIdx(); eidx = end_itr._path[level].getIdx(); if (idx > eidx) { @@ -545,7 +554,6 @@ public: return; } else { // Lowest shared node is a leaf node. - assert(_leaf.getNode() == end_itr._leaf.getNode()); } } uint32_t idx = _leaf.getIdx(); diff --git a/vespalib/src/vespa/vespalib/btree/btreeiterator.hpp b/vespalib/src/vespa/vespalib/btree/btreeiterator.hpp index 82864bff057..2102abbf10c 100644 --- a/vespalib/src/vespa/vespalib/btree/btreeiterator.hpp +++ b/vespalib/src/vespa/vespalib/btree/btreeiterator.hpp @@ -515,15 +515,12 @@ operator-(const BTreeIteratorBase &rhs) const if (_pathSize != 0) { uint32_t pidx = _pathSize; while (pidx > 0) { - assert(_path[pidx - 1].getNode() == rhs._path[pidx - 1].getNode()); if (_path[pidx - 1].getIdx() != rhs._path[pidx - 1].getIdx()) break; --pidx; } return position(pidx) - rhs.position(pidx); } else { - assert(_leaf.getNode() == nullptr || rhs._leaf.getNode() == nullptr || - _leaf.getNode() == rhs._leaf.getNode()); return position(0) - rhs.position(0); } } diff --git a/vespalib/src/vespa/vespalib/btree/btreenodestore.h b/vespalib/src/vespa/vespalib/btree/btreenodestore.h index 0b273b54c3a..81017447580 100644 --- a/vespalib/src/vespa/vespalib/btree/btreenodestore.h +++ b/vespalib/src/vespa/vespalib/btree/btreenodestore.h @@ -191,6 +191,10 @@ public: return _store.getCompacting(ref); } + bool has_held_buffers() const { + return _store.has_held_buffers(); + } + template <typename FunctionType> void foreach_key(EntryRef ref, FunctionType func) const { if (!ref.valid()) diff --git a/vespalib/src/vespa/vespalib/btree/btreeroot.h b/vespalib/src/vespa/vespalib/btree/btreeroot.h index 60eeb98e5a5..dc3507fc5d5 100644 --- a/vespalib/src/vespa/vespalib/btree/btreeroot.h +++ b/vespalib/src/vespa/vespalib/btree/btreeroot.h @@ -61,9 +61,10 @@ public: class FrozenView { private: BTreeNode::Ref _frozenRoot; - const NodeAllocatorType & _allocator; + const NodeAllocatorType *const _allocator; public: typedef ConstIterator Iterator; + FrozenView(); FrozenView(BTreeNode::Ref frozenRoot, const NodeAllocatorType & allocator); ConstIterator find(const KeyType& key, @@ -73,30 +74,30 @@ public: ConstIterator upperBound(const KeyType &key, CompareT comp = CompareT()) const; ConstIterator begin() const { - return ConstIterator(_frozenRoot, _allocator); + return ConstIterator(_frozenRoot, *_allocator); } void begin(std::vector<ConstIterator> &where) const { - where.emplace_back(_frozenRoot, _allocator); + where.emplace_back(_frozenRoot, *_allocator); } BTreeNode::Ref getRoot() const { return _frozenRoot; } size_t size() const; - const NodeAllocatorType &getAllocator() const { return _allocator; } + const NodeAllocatorType &getAllocator() const { return *_allocator; } const AggrT &getAggregated() const { - return _allocator.getAggregated(_frozenRoot); + return _allocator->getAggregated(_frozenRoot); } bool empty() const { return !_frozenRoot.valid(); } template <typename FunctionType> void foreach_key(FunctionType func) const { - _allocator.getNodeStore().foreach_key(_frozenRoot, func); + _allocator->getNodeStore().foreach_key(_frozenRoot, func); } template <typename FunctionType> void foreach(FunctionType func) const { - _allocator.getNodeStore().foreach(_frozenRoot, func); + _allocator->getNodeStore().foreach(_frozenRoot, func); } }; diff --git a/vespalib/src/vespa/vespalib/btree/btreeroot.hpp b/vespalib/src/vespa/vespalib/btree/btreeroot.hpp index dd271a16203..3cfd1cb49ea 100644 --- a/vespalib/src/vespa/vespalib/btree/btreeroot.hpp +++ b/vespalib/src/vespa/vespalib/btree/btreeroot.hpp @@ -170,10 +170,19 @@ upperBoundHelper(BTreeNode::Ref root, const KeyType & key, template <typename KeyT, typename DataT, typename AggrT, typename CompareT, typename TraitsT> BTreeRootT<KeyT, DataT, AggrT, CompareT, TraitsT>:: +FrozenView::FrozenView() + : _frozenRoot(BTreeNode::Ref()), + _allocator(nullptr) +{ +} + +template <typename KeyT, typename DataT, typename AggrT, typename CompareT, + typename TraitsT> +BTreeRootT<KeyT, DataT, AggrT, CompareT, TraitsT>:: FrozenView::FrozenView(BTreeNode::Ref frozenRoot, - const NodeAllocatorType & allocator) : - _frozenRoot(frozenRoot), - _allocator(allocator) + const NodeAllocatorType & allocator) + : _frozenRoot(frozenRoot), + _allocator(&allocator) { } @@ -184,7 +193,7 @@ BTreeRootT<KeyT, DataT, AggrT, CompareT, TraitsT>:: FrozenView::find(const KeyType & key, CompareT comp) const { - ConstIterator itr(BTreeNode::Ref(), _allocator); + ConstIterator itr(BTreeNode::Ref(), *_allocator); itr.lower_bound(_frozenRoot, key, comp); if (itr.valid() && comp(key, itr.getKey())) { itr.setupEnd(); @@ -199,7 +208,7 @@ BTreeRootT<KeyT, DataT, AggrT, CompareT, TraitsT>:: FrozenView::lowerBound(const KeyType & key, CompareT comp) const { - ConstIterator itr(BTreeNode::Ref(), _allocator); + ConstIterator itr(BTreeNode::Ref(), *_allocator); itr.lower_bound(_frozenRoot, key, comp); return itr; } @@ -211,7 +220,7 @@ BTreeRootT<KeyT, DataT, AggrT, CompareT, TraitsT>:: FrozenView::upperBound(const KeyType & key, CompareT comp) const { - ConstIterator itr(_frozenRoot, _allocator); + ConstIterator itr(_frozenRoot, *_allocator); if (itr.valid() && !comp(key, itr.getKey())) { itr.seekPast(key, comp); } @@ -225,7 +234,7 @@ BTreeRootT<KeyT, DataT, AggrT, CompareT, TraitsT>:: FrozenView::size() const { if (NodeAllocatorType::isValidRef(_frozenRoot)) { - return _allocator.validLeaves(_frozenRoot); + return _allocator->validLeaves(_frozenRoot); } return 0u; } diff --git a/vespalib/src/vespa/vespalib/datastore/datastorebase.cpp b/vespalib/src/vespa/vespalib/datastore/datastorebase.cpp index 2ec8cb3bda3..cce0b0695c2 100644 --- a/vespalib/src/vespa/vespalib/datastore/datastorebase.cpp +++ b/vespalib/src/vespa/vespalib/datastore/datastorebase.cpp @@ -66,6 +66,7 @@ public: _dsb(dsb), _bufferId(bufferId) { + _dsb.inc_hold_buffer_count(); } ~BufferHold() override @@ -85,6 +86,7 @@ DataStoreBase::DataStoreBase(uint32_t numBuffers, size_t maxArrays) _elemHold1List(), _elemHold2List(), _numBuffers(numBuffers), + _hold_buffer_count(0u), _maxArrays(maxArrays), _compaction_count(0u), _genHolder() @@ -187,6 +189,8 @@ DataStoreBase::transferHoldLists(generation_t generation) void DataStoreBase::doneHoldBuffer(uint32_t bufferId) { + assert(_hold_buffer_count > 0); + --_hold_buffer_count; _states[bufferId].onFree(_buffers[bufferId].getBuffer()); } @@ -515,5 +519,12 @@ DataStoreBase::startCompactWorstBuffers(bool compactMemory, bool compactAddressS return result; } +void +DataStoreBase::inc_hold_buffer_count() +{ + assert(_hold_buffer_count < std::numeric_limits<uint32_t>::max()); + ++_hold_buffer_count; +} + } diff --git a/vespalib/src/vespa/vespalib/datastore/datastorebase.h b/vespalib/src/vespa/vespalib/datastore/datastorebase.h index 2617973d239..13c242e85a7 100644 --- a/vespalib/src/vespa/vespalib/datastore/datastorebase.h +++ b/vespalib/src/vespa/vespalib/datastore/datastorebase.h @@ -156,6 +156,7 @@ protected: ElemHold2List _elemHold2List; const uint32_t _numBuffers; + uint32_t _hold_buffer_count; const size_t _maxArrays; mutable std::atomic<uint64_t> _compaction_count; @@ -350,6 +351,7 @@ private: */ void onActive(uint32_t bufferId, uint32_t typeId, size_t elemsNeeded); + void inc_hold_buffer_count(); public: uint32_t getTypeId(uint32_t bufferId) const { return _buffers[bufferId].getTypeId(); @@ -368,6 +370,7 @@ public: std::vector<uint32_t> startCompactWorstBuffers(bool compactMemory, bool compactAddressSpace); uint64_t get_compaction_count() const { return _compaction_count.load(std::memory_order_relaxed); } void inc_compaction_count() const { ++_compaction_count; } + bool has_held_buffers() const noexcept { return _hold_buffer_count != 0u; } }; } diff --git a/vespalib/src/vespa/vespalib/datastore/fixed_size_hash_map.cpp b/vespalib/src/vespa/vespalib/datastore/fixed_size_hash_map.cpp index cca61e3c852..18476dc64a5 100644 --- a/vespalib/src/vespa/vespalib/datastore/fixed_size_hash_map.cpp +++ b/vespalib/src/vespa/vespalib/datastore/fixed_size_hash_map.cpp @@ -52,8 +52,8 @@ FixedSizeHashMap::~FixedSizeHashMap() = default; void FixedSizeHashMap::force_add(const EntryComparator& comp, const KvType& kv) { - size_t hash_idx = comp.hash(kv.first.load_relaxed()) / _num_shards; - hash_idx %= _modulo; + ShardedHashComparator shardedComp(comp, kv.first.load_relaxed(), _num_shards); + uint32_t hash_idx = shardedComp.hash_idx() % _modulo; auto& chain_head = _chain_heads[hash_idx]; assert(_nodes.size() < _nodes.capacity()); uint32_t node_idx = _nodes.size(); @@ -63,15 +63,14 @@ FixedSizeHashMap::force_add(const EntryComparator& comp, const KvType& kv) } FixedSizeHashMap::KvType& -FixedSizeHashMap::add(const EntryComparator& comp, EntryRef key_ref, std::function<EntryRef(void)>& insert_entry) +FixedSizeHashMap::add(const ShardedHashComparator & comp, std::function<EntryRef(void)>& insert_entry) { - size_t hash_idx = comp.hash(key_ref) / _num_shards; - hash_idx %= _modulo; + uint32_t hash_idx = comp.hash_idx() % _modulo; auto& chain_head = _chain_heads[hash_idx]; uint32_t node_idx = chain_head.load_relaxed(); while (node_idx != no_node_idx) { auto& node = _nodes[node_idx]; - if (comp.equal(key_ref, node.get_kv().first.load_relaxed())) { + if (comp.equal(node.get_kv().first.load_relaxed())) { return node.get_kv(); } node_idx = node.get_next_node_idx().load(std::memory_order_relaxed); @@ -127,17 +126,16 @@ FixedSizeHashMap::trim_hold_lists_slow(generation_t first_used) } FixedSizeHashMap::KvType* -FixedSizeHashMap::remove(const EntryComparator& comp, EntryRef key_ref) +FixedSizeHashMap::remove(const ShardedHashComparator & comp) { - size_t hash_idx = comp.hash(key_ref) / _num_shards; - hash_idx %= _modulo; + uint32_t hash_idx = comp.hash_idx() % _modulo; auto& chain_head = _chain_heads[hash_idx]; uint32_t node_idx = chain_head.load_relaxed(); uint32_t prev_node_idx = no_node_idx; while (node_idx != no_node_idx) { auto &node = _nodes[node_idx]; uint32_t next_node_idx = node.get_next_node_idx().load(std::memory_order_relaxed); - if (comp.equal(key_ref, node.get_kv().first.load_relaxed())) { + if (comp.equal(node.get_kv().first.load_relaxed())) { if (prev_node_idx != no_node_idx) { _nodes[prev_node_idx].get_next_node_idx().store(next_node_idx, std::memory_order_release); } else { @@ -154,24 +152,6 @@ FixedSizeHashMap::remove(const EntryComparator& comp, EntryRef key_ref) return nullptr; } -FixedSizeHashMap::KvType* -FixedSizeHashMap::find(const EntryComparator& comp, EntryRef key_ref) -{ - size_t hash_idx = comp.hash(key_ref) / _num_shards; - hash_idx %= _modulo; - auto& chain_head = _chain_heads[hash_idx]; - uint32_t node_idx = chain_head.load_acquire(); - while (node_idx != no_node_idx) { - auto &node = _nodes[node_idx]; - EntryRef node_key_ref = node.get_kv().first.load_acquire(); - if (node_key_ref.valid() && comp.equal(key_ref, node_key_ref)) { - return &_nodes[node_idx].get_kv(); - } - node_idx = node.get_next_node_idx().load(std::memory_order_acquire); - } - return nullptr; -} - MemoryUsage FixedSizeHashMap::get_memory_usage() const { @@ -187,4 +167,54 @@ FixedSizeHashMap::get_memory_usage() const nodes_hold_size); } +void +FixedSizeHashMap::foreach_key(const std::function<void(EntryRef)>& callback) const +{ + for (auto& chain_head : _chain_heads) { + uint32_t node_idx = chain_head.load_relaxed(); + while (node_idx != no_node_idx) { + auto& node = _nodes[node_idx]; + callback(node.get_kv().first.load_relaxed()); + node_idx = node.get_next_node_idx().load(std::memory_order_relaxed); + } + } +} + +void +FixedSizeHashMap::move_keys(const std::function<EntryRef(EntryRef)>& callback) +{ + for (auto& chain_head : _chain_heads) { + uint32_t node_idx = chain_head.load_relaxed(); + while (node_idx != no_node_idx) { + auto& node = _nodes[node_idx]; + EntryRef old_ref = node.get_kv().first.load_relaxed(); + EntryRef new_ref = callback(old_ref); + if (new_ref != old_ref) { + node.get_kv().first.store_release(new_ref); + } + node_idx = node.get_next_node_idx().load(std::memory_order_relaxed); + } + } +} + +bool +FixedSizeHashMap::normalize_values(const std::function<EntryRef(EntryRef)>& normalize) +{ + bool changed = false; + for (auto& chain_head : _chain_heads) { + uint32_t node_idx = chain_head.load_relaxed(); + while (node_idx != no_node_idx) { + auto& node = _nodes[node_idx]; + EntryRef old_ref = node.get_kv().second.load_relaxed(); + EntryRef new_ref = normalize(old_ref); + if (new_ref != old_ref) { + node.get_kv().second.store_release(new_ref); + changed = true; + } + node_idx = node.get_next_node_idx().load(std::memory_order_relaxed); + } + } + return changed; +} + } diff --git a/vespalib/src/vespa/vespalib/datastore/fixed_size_hash_map.h b/vespalib/src/vespa/vespalib/datastore/fixed_size_hash_map.h index a0ca1bf0568..101dd1bead3 100644 --- a/vespalib/src/vespa/vespalib/datastore/fixed_size_hash_map.h +++ b/vespalib/src/vespa/vespalib/datastore/fixed_size_hash_map.h @@ -3,6 +3,7 @@ #pragma once #include "atomic_entry_ref.h" +#include "entry_comparator.h" #include <vespa/vespalib/util/array.h> #include <vespa/vespalib/util/arrayref.h> #include <vespa/vespalib/util/generationhandler.h> @@ -17,7 +18,27 @@ class MemoryUsage; } namespace vespalib::datastore { -class EntryComparator; +class ShardedHashComparator { +public: + ShardedHashComparator(const EntryComparator& comp, const EntryRef key_ref, uint32_t num_shards) + : _comp(comp), + _key_ref(key_ref) + { + size_t hash = comp.hash(key_ref); + _shard_idx = hash % num_shards; + _hash_idx = hash / num_shards; + } + uint32_t hash_idx() const { return _hash_idx; } + uint32_t shard_idx() const { return _shard_idx; } + bool equal(const EntryRef rhs) const { + return _comp.equal(_key_ref, rhs); + } +private: + const EntryComparator& _comp; + const EntryRef _key_ref; + uint32_t _shard_idx; + uint32_t _hash_idx; +}; /* * Fixed sized hash map over keys in data store, meant to support a faster @@ -42,7 +63,6 @@ public: using KvType = std::pair<AtomicEntryRef, AtomicEntryRef>; using generation_t = GenerationHandler::generation_t; using sgeneration_t = GenerationHandler::sgeneration_t; - private: class ChainHead { std::atomic<uint32_t> _node_idx; @@ -99,9 +119,26 @@ public: FixedSizeHashMap(uint32_t module, uint32_t capacity, uint32_t num_shards, const FixedSizeHashMap &orig, const EntryComparator& comp); ~FixedSizeHashMap(); - KvType& add(const EntryComparator& comp, EntryRef key_ref, std::function<EntryRef(void)>& insert_entry); - KvType* remove(const EntryComparator& comp, EntryRef key_ref); - KvType* find(const EntryComparator& comp, EntryRef key_ref); + ShardedHashComparator get_comp(const EntryComparator& comp) { + return ShardedHashComparator(comp, EntryRef(), _num_shards); + } + + KvType& add(const ShardedHashComparator & comp, std::function<EntryRef(void)>& insert_entry); + KvType* remove(const ShardedHashComparator & comp); + KvType* find(const ShardedHashComparator & comp) { + uint32_t hash_idx = comp.hash_idx() % _modulo; + auto& chain_head = _chain_heads[hash_idx]; + uint32_t node_idx = chain_head.load_acquire(); + while (node_idx != no_node_idx) { + auto &node = _nodes[node_idx]; + EntryRef node_key_ref = node.get_kv().first.load_acquire(); + if (node_key_ref.valid() && comp.equal(node_key_ref)) { + return &_nodes[node_idx].get_kv(); + } + node_idx = node.get_next_node_idx().load(std::memory_order_acquire); + } + return nullptr; + } void transfer_hold_lists(generation_t generation) { if (!_hold_1_list.empty()) { @@ -118,6 +155,9 @@ public: bool full() const noexcept { return _nodes.size() == _nodes.capacity() && _free_count == 0u; } size_t size() const noexcept { return _count; } MemoryUsage get_memory_usage() const; + void foreach_key(const std::function<void(EntryRef)>& callback) const; + void move_keys(const std::function<EntryRef(EntryRef)>& callback); + bool normalize_values(const std::function<EntryRef(EntryRef)>& normalize); }; } diff --git a/vespalib/src/vespa/vespalib/datastore/i_unique_store_dictionary.h b/vespalib/src/vespa/vespalib/datastore/i_unique_store_dictionary.h index d31139efc76..8493e8990e3 100644 --- a/vespalib/src/vespa/vespalib/datastore/i_unique_store_dictionary.h +++ b/vespalib/src/vespa/vespalib/datastore/i_unique_store_dictionary.h @@ -12,6 +12,7 @@ namespace vespalib::datastore { class EntryComparator; struct ICompactable; +class IUniqueStoreDictionaryReadSnapshot; class UniqueStoreAddResult; /** @@ -19,20 +20,6 @@ class UniqueStoreAddResult; */ class IUniqueStoreDictionary { public: - /** - * Class that provides a read snapshot of the dictionary. - * - * A generation guard that must be taken and held while the snapshot is considered valid. - */ - class ReadSnapshot { - public: - using UP = std::unique_ptr<ReadSnapshot>; - virtual ~ReadSnapshot() = default; - virtual size_t count(const EntryComparator& comp) const = 0; - virtual size_t count_in_range(const EntryComparator& low, const EntryComparator& high) const = 0; - virtual void foreach_key(std::function<void(EntryRef)> callback) const = 0; - }; - using generation_t = vespalib::GenerationHandler::generation_t; virtual ~IUniqueStoreDictionary() = default; virtual void freeze() = 0; @@ -47,7 +34,8 @@ public: virtual void build(vespalib::ConstArrayRef<EntryRef>, vespalib::ConstArrayRef<uint32_t> ref_counts, std::function<void(EntryRef)> hold) = 0; virtual void build(vespalib::ConstArrayRef<EntryRef> refs) = 0; virtual void build_with_payload(vespalib::ConstArrayRef<EntryRef> refs, vespalib::ConstArrayRef<uint32_t> payloads) = 0; - virtual std::unique_ptr<ReadSnapshot> get_read_snapshot() const = 0; + virtual std::unique_ptr<IUniqueStoreDictionaryReadSnapshot> get_read_snapshot() const = 0; + virtual bool get_has_btree_dictionary() const = 0; virtual bool get_has_hash_dictionary() const = 0; }; diff --git a/vespalib/src/vespa/vespalib/datastore/i_unique_store_dictionary_read_snapshot.h b/vespalib/src/vespa/vespalib/datastore/i_unique_store_dictionary_read_snapshot.h new file mode 100644 index 00000000000..b1ea05ffa4d --- /dev/null +++ b/vespalib/src/vespa/vespalib/datastore/i_unique_store_dictionary_read_snapshot.h @@ -0,0 +1,27 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "entryref.h" +#include <functional> + +namespace vespalib::datastore { + +class EntryComparator; + +/** + * Class that provides a read snapshot of the dictionary. + * + * A generation guard that must be taken and held while the snapshot is considered valid. + */ +class IUniqueStoreDictionaryReadSnapshot { +public: + virtual ~IUniqueStoreDictionaryReadSnapshot() = default; + virtual void fill() = 0; + virtual void sort() = 0; + virtual size_t count(const EntryComparator& comp) const = 0; + virtual size_t count_in_range(const EntryComparator& low, const EntryComparator& high) const = 0; + virtual void foreach_key(std::function<void(EntryRef)> callback) const = 0; +}; + +} diff --git a/vespalib/src/vespa/vespalib/datastore/sharded_hash_map.cpp b/vespalib/src/vespa/vespalib/datastore/sharded_hash_map.cpp index 7c609f89972..36d68873176 100644 --- a/vespalib/src/vespa/vespalib/datastore/sharded_hash_map.cpp +++ b/vespalib/src/vespa/vespalib/datastore/sharded_hash_map.cpp @@ -12,7 +12,7 @@ class ShardedHashMapShardHeld : public GenerationHeldBase std::unique_ptr<const FixedSizeHashMap> _data; public: ShardedHashMapShardHeld(size_t size, std::unique_ptr<const FixedSizeHashMap> data); - ~ShardedHashMapShardHeld(); + ~ShardedHashMapShardHeld() override; }; ShardedHashMapShardHeld::ShardedHashMapShardHeld(size_t size, std::unique_ptr<const FixedSizeHashMap> data) @@ -39,12 +39,6 @@ ShardedHashMap::~ShardedHashMap() } } -size_t -ShardedHashMap::get_shard_idx(const EntryComparator& comp, EntryRef key_ref) const -{ - return comp.hash(key_ref) % num_shards; -} - void ShardedHashMap::alloc_shard(size_t shard_idx) { @@ -70,46 +64,46 @@ ShardedHashMap::hold_shard(std::unique_ptr<const FixedSizeHashMap> map) ShardedHashMap::KvType& ShardedHashMap::add(const EntryComparator& comp, EntryRef key_ref, std::function<EntryRef(void)>& insert_entry) { - size_t shard_idx = get_shard_idx(comp, key_ref); - auto map = _maps[shard_idx].load(std::memory_order_relaxed); + ShardedHashComparator shardedComp(comp, key_ref, num_shards); + auto map = _maps[shardedComp.shard_idx()].load(std::memory_order_relaxed); if (map == nullptr || map->full()) { - alloc_shard(shard_idx); - map = _maps[shard_idx].load(std::memory_order_relaxed); + alloc_shard(shardedComp.shard_idx()); + map = _maps[shardedComp.shard_idx()].load(std::memory_order_relaxed); } - return map->add(comp, key_ref, insert_entry); + return map->add(shardedComp, insert_entry); } ShardedHashMap::KvType* ShardedHashMap::remove(const EntryComparator& comp, EntryRef key_ref) { - size_t shard_idx = get_shard_idx(comp, key_ref); - auto map = _maps[shard_idx].load(std::memory_order_relaxed); + ShardedHashComparator shardedComp(comp, key_ref, num_shards); + auto map = _maps[shardedComp.shard_idx()].load(std::memory_order_relaxed); if (map == nullptr) { return nullptr; } - return map->remove(comp, key_ref); + return map->remove(shardedComp); } ShardedHashMap::KvType* ShardedHashMap::find(const EntryComparator& comp, EntryRef key_ref) { - size_t shard_idx = get_shard_idx(comp, key_ref); - auto map = _maps[shard_idx].load(std::memory_order_relaxed); + ShardedHashComparator shardedComp(comp, key_ref, num_shards); + auto map = _maps[shardedComp.shard_idx()].load(std::memory_order_relaxed); if (map == nullptr) { return nullptr; } - return map->find(comp, key_ref); + return map->find(shardedComp); } const ShardedHashMap::KvType* ShardedHashMap::find(const EntryComparator& comp, EntryRef key_ref) const { - size_t shard_idx = get_shard_idx(comp, key_ref); - auto map = _maps[shard_idx].load(std::memory_order_relaxed); + ShardedHashComparator shardedComp(comp, key_ref, num_shards); + auto map = _maps[shardedComp.shard_idx()].load(std::memory_order_relaxed); if (map == nullptr) { return nullptr; } - return map->find(comp, key_ref); + return map->find(shardedComp); } void @@ -165,4 +159,39 @@ ShardedHashMap::get_memory_usage() const return memory_usage; } +void +ShardedHashMap::foreach_key(std::function<void(EntryRef)> callback) const +{ + for (size_t i = 0; i < num_shards; ++i) { + auto map = _maps[i].load(std::memory_order_relaxed); + if (map != nullptr) { + map->foreach_key(callback); + } + } +} + +void +ShardedHashMap::move_keys(std::function<EntryRef(EntryRef)> callback) +{ + for (size_t i = 0; i < num_shards; ++i) { + auto map = _maps[i].load(std::memory_order_relaxed); + if (map != nullptr) { + map->move_keys(callback); + } + } +} + +bool +ShardedHashMap::normalize_values(std::function<EntryRef(EntryRef)> normalize) +{ + bool changed = false; + for (size_t i = 0; i < num_shards; ++i) { + auto map = _maps[i].load(std::memory_order_relaxed); + if (map != nullptr) { + changed |= map->normalize_values(normalize); + } + } + return changed; +} + } diff --git a/vespalib/src/vespa/vespalib/datastore/sharded_hash_map.h b/vespalib/src/vespa/vespalib/datastore/sharded_hash_map.h index 7ea60646b06..aa787421634 100644 --- a/vespalib/src/vespa/vespalib/datastore/sharded_hash_map.h +++ b/vespalib/src/vespa/vespalib/datastore/sharded_hash_map.h @@ -41,7 +41,6 @@ private: std::atomic<FixedSizeHashMap *> _maps[num_shards]; std::unique_ptr<const EntryComparator> _comp; - size_t get_shard_idx(const EntryComparator& comp, EntryRef key_ref) const; void alloc_shard(size_t shard_idx); void hold_shard(std::unique_ptr<const FixedSizeHashMap> map); public: @@ -56,6 +55,9 @@ public: size_t size() const noexcept; const EntryComparator &get_default_comparator() const noexcept { return *_comp; } MemoryUsage get_memory_usage() const; + void foreach_key(std::function<void(EntryRef)> callback) const; + void move_keys(std::function<EntryRef(EntryRef)> callback); + bool normalize_values(std::function<EntryRef(EntryRef)> normalize); }; } diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_btree_dictionary_read_snapshot.h b/vespalib/src/vespa/vespalib/datastore/unique_store_btree_dictionary_read_snapshot.h new file mode 100644 index 00000000000..2f958744f5f --- /dev/null +++ b/vespalib/src/vespa/vespalib/datastore/unique_store_btree_dictionary_read_snapshot.h @@ -0,0 +1,30 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "i_unique_store_dictionary_read_snapshot.h" + +namespace vespalib::datastore { + +/** + * Class that provides a read snapshot of a unique store dictionary using a btree. + * + * A generation guard that must be taken and held while the snapshot is considered valid. + */ +template <typename BTreeDictionaryT> +class UniqueStoreBTreeDictionaryReadSnapshot : public IUniqueStoreDictionaryReadSnapshot { +private: + using BTreeDictionaryType = BTreeDictionaryT; + using FrozenView = typename BTreeDictionaryType::FrozenView; + FrozenView _frozen_view; + +public: + UniqueStoreBTreeDictionaryReadSnapshot(FrozenView frozen_view); + void fill() override; + void sort() override; + size_t count(const EntryComparator& comp) const override; + size_t count_in_range(const EntryComparator& low, const EntryComparator& high) const override; + void foreach_key(std::function<void(EntryRef)> callback) const override; +}; + +} diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_btree_dictionary_read_snapshot.hpp b/vespalib/src/vespa/vespalib/datastore/unique_store_btree_dictionary_read_snapshot.hpp new file mode 100644 index 00000000000..8fb542b2999 --- /dev/null +++ b/vespalib/src/vespa/vespalib/datastore/unique_store_btree_dictionary_read_snapshot.hpp @@ -0,0 +1,57 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "unique_store_btree_dictionary_read_snapshot.h" + +namespace vespalib::datastore { + +template <typename BTreeDictionaryT> +UniqueStoreBTreeDictionaryReadSnapshot<BTreeDictionaryT>::UniqueStoreBTreeDictionaryReadSnapshot(FrozenView frozen_view) + : _frozen_view(frozen_view) +{ +} + +template <typename BTreeDictionaryT> +void +UniqueStoreBTreeDictionaryReadSnapshot<BTreeDictionaryT>::fill() +{ +} + +template <typename BTreeDictionaryT> +void +UniqueStoreBTreeDictionaryReadSnapshot<BTreeDictionaryT>::sort() +{ +} + +template <typename BTreeDictionaryT> +size_t +UniqueStoreBTreeDictionaryReadSnapshot<BTreeDictionaryT>::count(const EntryComparator& comp) const +{ + auto itr = _frozen_view.lowerBound(EntryRef(), comp); + if (itr.valid() && !comp.less(EntryRef(), itr.getKey())) { + return 1u; + } + return 0u; +} + +template <typename BTreeDictionaryT> +size_t +UniqueStoreBTreeDictionaryReadSnapshot<BTreeDictionaryT>::count_in_range(const EntryComparator& low, const EntryComparator& high) const +{ + auto low_itr = _frozen_view.lowerBound(EntryRef(), low); + auto high_itr = low_itr; + if (high_itr.valid() && !high.less(EntryRef(), high_itr.getKey())) { + high_itr.seekPast(EntryRef(), high); + } + return high_itr - low_itr; +} + +template <typename BTreeDictionaryT> +void +UniqueStoreBTreeDictionaryReadSnapshot<BTreeDictionaryT>::foreach_key(std::function<void(EntryRef)> callback) const +{ + _frozen_view.foreach_key(callback); +} + +} diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.h b/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.h index 8fbff3f9d93..612188b4653 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.h +++ b/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.h @@ -8,7 +8,9 @@ namespace vespalib::datastore { class EntryComparatorWrapper; +class IUniqueStoreDictionaryReadSnapshot; +class NoBTreeDictionary { }; class NoHashDictionary; template <typename BTreeDictionaryT> @@ -24,6 +26,16 @@ public: } }; +template <> +class UniqueStoreBTreeDictionaryBase<NoBTreeDictionary> +{ +public: + static constexpr bool has_btree_dictionary = false; + UniqueStoreBTreeDictionaryBase() + { + } +}; + template <typename HashDictionaryT> class UniqueStoreHashDictionaryBase { @@ -54,22 +66,8 @@ template <typename BTreeDictionaryT, typename ParentT = IUniqueStoreDictionary, class UniqueStoreDictionary : public ParentT, public UniqueStoreBTreeDictionaryBase<BTreeDictionaryT>, public UniqueStoreHashDictionaryBase<HashDictionaryT> { protected: using BTreeDictionaryType = BTreeDictionaryT; - using DataType = typename BTreeDictionaryType::DataType; - using FrozenView = typename BTreeDictionaryType::FrozenView; - using ReadSnapshot = typename ParentT::ReadSnapshot; using generation_t = typename ParentT::generation_t; - class ReadSnapshotImpl : public ReadSnapshot { - private: - FrozenView _frozen_view; - - public: - ReadSnapshotImpl(FrozenView frozen_view); - size_t count(const EntryComparator& comp) const override; - size_t count_in_range(const EntryComparator& low, const EntryComparator& high) const override; - void foreach_key(std::function<void(EntryRef)> callback) const override; - }; - public: using UniqueStoreBTreeDictionaryBase<BTreeDictionaryT>::has_btree_dictionary; using UniqueStoreHashDictionaryBase<HashDictionaryT>::has_hash_dictionary; @@ -87,7 +85,8 @@ public: void build(vespalib::ConstArrayRef<EntryRef>, vespalib::ConstArrayRef<uint32_t> ref_counts, std::function<void(EntryRef)> hold) override; void build(vespalib::ConstArrayRef<EntryRef> refs) override; void build_with_payload(vespalib::ConstArrayRef<EntryRef>, vespalib::ConstArrayRef<uint32_t> payloads) override; - std::unique_ptr<ReadSnapshot> get_read_snapshot() const override; + std::unique_ptr<IUniqueStoreDictionaryReadSnapshot> get_read_snapshot() const override; + bool get_has_btree_dictionary() const override; bool get_has_hash_dictionary() const override; }; diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.hpp b/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.hpp index edacb306c6e..825db1a8cb5 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.hpp +++ b/vespalib/src/vespa/vespalib/datastore/unique_store_dictionary.hpp @@ -7,6 +7,8 @@ #include "i_compactable.h" #include "unique_store_add_result.h" #include "unique_store_dictionary.h" +#include "unique_store_btree_dictionary_read_snapshot.hpp" +#include "unique_store_hash_dictionary_read_snapshot.hpp" #include <vespa/vespalib/btree/btree.hpp> #include <vespa/vespalib/btree/btreebuilder.hpp> #include <vespa/vespalib/btree/btreeiterator.hpp> @@ -17,47 +19,6 @@ namespace vespalib::datastore { template <typename BTreeDictionaryT, typename ParentT, typename HashDictionaryT> -UniqueStoreDictionary<BTreeDictionaryT, ParentT, HashDictionaryT>:: -ReadSnapshotImpl::ReadSnapshotImpl(FrozenView frozen_view) - : _frozen_view(frozen_view) -{ -} - -template <typename BTreeDictionaryT, typename ParentT, typename HashDictionaryT> -size_t -UniqueStoreDictionary<BTreeDictionaryT, ParentT, HashDictionaryT>:: -ReadSnapshotImpl::count(const EntryComparator& comp) const -{ - auto itr = _frozen_view.lowerBound(EntryRef(), comp); - if (itr.valid() && !comp.less(EntryRef(), itr.getKey())) { - return 1u; - } - return 0u; -} - -template <typename BTreeDictionaryT, typename ParentT, typename HashDictionaryT> -size_t -UniqueStoreDictionary<BTreeDictionaryT, ParentT, HashDictionaryT>:: -ReadSnapshotImpl::count_in_range(const EntryComparator& low, - const EntryComparator& high) const -{ - auto low_itr = _frozen_view.lowerBound(EntryRef(), low); - auto high_itr = low_itr; - if (high_itr.valid() && !high.less(EntryRef(), high_itr.getKey())) { - high_itr.seekPast(EntryRef(), high); - } - return high_itr - low_itr; -} - -template <typename BTreeDictionaryT, typename ParentT, typename HashDictionaryT> -void -UniqueStoreDictionary<BTreeDictionaryT, ParentT, HashDictionaryT>:: -ReadSnapshotImpl::foreach_key(std::function<void(EntryRef)> callback) const -{ - _frozen_view.foreach_key(callback); -} - -template <typename BTreeDictionaryT, typename ParentT, typename HashDictionaryT> UniqueStoreDictionary<BTreeDictionaryT, ParentT, HashDictionaryT>::UniqueStoreDictionary(std::unique_ptr<EntryComparator> compare) : ParentT(), UniqueStoreBTreeDictionaryBase<BTreeDictionaryT>(), @@ -72,14 +33,18 @@ template <typename BTreeDictionaryT, typename ParentT, typename HashDictionaryT> void UniqueStoreDictionary<BTreeDictionaryT, ParentT, HashDictionaryT>::freeze() { - this->_btree_dict.getAllocator().freeze(); + if constexpr (has_btree_dictionary) { + this->_btree_dict.getAllocator().freeze(); + } } template <typename BTreeDictionaryT, typename ParentT, typename HashDictionaryT> void UniqueStoreDictionary<BTreeDictionaryT, ParentT, HashDictionaryT>::transfer_hold_lists(generation_t generation) { - this->_btree_dict.getAllocator().transferHoldLists(generation); + if constexpr (has_btree_dictionary) { + this->_btree_dict.getAllocator().transferHoldLists(generation); + } if constexpr (has_hash_dictionary) { this->_hash_dict.transfer_hold_lists(generation); } @@ -89,7 +54,9 @@ template <typename BTreeDictionaryT, typename ParentT, typename HashDictionaryT> void UniqueStoreDictionary<BTreeDictionaryT, ParentT, HashDictionaryT>::trim_hold_lists(generation_t firstUsed) { - this->_btree_dict.getAllocator().trimHoldLists(firstUsed); + if constexpr (has_btree_dictionary) { + this->_btree_dict.getAllocator().trimHoldLists(firstUsed); + } if constexpr (has_hash_dictionary) { this->_hash_dict.trim_hold_lists(firstUsed); } @@ -100,23 +67,32 @@ UniqueStoreAddResult UniqueStoreDictionary<BTreeDictionaryT, ParentT, HashDictionaryT>::add(const EntryComparator &comp, std::function<EntryRef(void)> insertEntry) { - auto itr = this->_btree_dict.lowerBound(EntryRef(), comp); - if (itr.valid() && !comp.less(EntryRef(), itr.getKey())) { - if constexpr (has_hash_dictionary) { - auto* result = this->_hash_dict.find(comp, EntryRef()); - assert(result != nullptr && result->first.load_relaxed() == itr.getKey()); + if constexpr (has_btree_dictionary) { + using DataType = typename BTreeDictionaryType::DataType; + auto itr = this->_btree_dict.lowerBound(EntryRef(), comp); + if (itr.valid() && !comp.less(EntryRef(), itr.getKey())) { + if constexpr (has_hash_dictionary) { + auto* result = this->_hash_dict.find(comp, EntryRef()); + assert(result != nullptr && result->first.load_relaxed() == itr.getKey()); + } + return UniqueStoreAddResult(itr.getKey(), false); + } else { + EntryRef newRef = insertEntry(); + this->_btree_dict.insert(itr, newRef, DataType()); + if constexpr (has_hash_dictionary) { + std::function<EntryRef(void)> insert_hash_entry([newRef]() noexcept -> EntryRef { return newRef; }); + auto& add_result = this->_hash_dict.add(comp, newRef, insert_hash_entry); + assert(add_result.first.load_relaxed() == newRef); + } + return UniqueStoreAddResult(newRef, true); } - return UniqueStoreAddResult(itr.getKey(), false); - } else { - EntryRef newRef = insertEntry(); - this->_btree_dict.insert(itr, newRef, DataType()); - if constexpr (has_hash_dictionary) { - std::function<EntryRef(void)> insert_hash_entry([newRef]() noexcept -> EntryRef { return newRef; }); - auto& add_result = this->_hash_dict.add(comp, newRef, insert_hash_entry); - assert(add_result.first.load_relaxed() == newRef); - } - return UniqueStoreAddResult(newRef, true); + bool inserted = false; + std::function<EntryRef(void)> insert_hash_entry([&inserted,&insertEntry]() { inserted = true; return insertEntry(); }); + auto& add_result = this->_hash_dict.add(comp, EntryRef(), insert_hash_entry); + EntryRef newRef = add_result.first.load_relaxed(); + assert(newRef.valid()); + return UniqueStoreAddResult(newRef, inserted); } } @@ -124,19 +100,24 @@ template <typename BTreeDictionaryT, typename ParentT, typename HashDictionaryT> EntryRef UniqueStoreDictionary<BTreeDictionaryT, ParentT, HashDictionaryT>::find(const EntryComparator &comp) { - auto itr = this->_btree_dict.lowerBound(EntryRef(), comp); - if (itr.valid() && !comp.less(EntryRef(), itr.getKey())) { - if constexpr (has_hash_dictionary) { - auto* result = this->_hash_dict.find(comp, EntryRef()); - assert(result != nullptr && result->first.load_relaxed() == itr.getKey()); + if constexpr (has_btree_dictionary) { + auto itr = this->_btree_dict.lowerBound(EntryRef(), comp); + if (itr.valid() && !comp.less(EntryRef(), itr.getKey())) { + if constexpr (has_hash_dictionary) { + auto* result = this->_hash_dict.find(comp, EntryRef()); + assert(result != nullptr && result->first.load_relaxed() == itr.getKey()); + } + return itr.getKey(); + } else { + if constexpr (has_hash_dictionary) { + auto* result = this->_hash_dict.find(comp, EntryRef()); + assert(result == nullptr); + } + return EntryRef(); } - return itr.getKey(); } else { - if constexpr (has_hash_dictionary) { - auto* result = this->_hash_dict.find(comp, EntryRef()); - assert(result == nullptr); - } - return EntryRef(); + auto* result = this->_hash_dict.find(comp, EntryRef()); + return (result == nullptr) ? EntryRef() : result->first.load_relaxed(); } } @@ -145,9 +126,11 @@ void UniqueStoreDictionary<BTreeDictionaryT, ParentT, HashDictionaryT>::remove(const EntryComparator &comp, EntryRef ref) { assert(ref.valid()); - auto itr = this->_btree_dict.lowerBound(ref, comp); - assert(itr.valid() && itr.getKey() == ref); - this->_btree_dict.remove(itr); + if constexpr (has_btree_dictionary) { + auto itr = this->_btree_dict.lowerBound(ref, comp); + assert(itr.valid() && itr.getKey() == ref); + this->_btree_dict.remove(itr); + } if constexpr (has_hash_dictionary) { auto *result = this->_hash_dict.remove(comp, ref); assert(result != nullptr && result->first.load_relaxed() == ref); @@ -158,20 +141,24 @@ template <typename BTreeDictionaryT, typename ParentT, typename HashDictionaryT> void UniqueStoreDictionary<BTreeDictionaryT, ParentT, HashDictionaryT>::move_entries(ICompactable &compactable) { - auto itr = this->_btree_dict.begin(); - while (itr.valid()) { - EntryRef oldRef(itr.getKey()); - EntryRef newRef(compactable.move(oldRef)); - if (newRef != oldRef) { - this->_btree_dict.thaw(itr); - itr.writeKey(newRef); - if constexpr (has_hash_dictionary) { - auto result = this->_hash_dict.find(this->_hash_dict.get_default_comparator(), oldRef); - assert(result != nullptr && result->first.load_relaxed() == oldRef); - result->first.store_release(newRef); + if constexpr (has_btree_dictionary) { + auto itr = this->_btree_dict.begin(); + while (itr.valid()) { + EntryRef oldRef(itr.getKey()); + EntryRef newRef(compactable.move(oldRef)); + if (newRef != oldRef) { + this->_btree_dict.thaw(itr); + itr.writeKey(newRef); + if constexpr (has_hash_dictionary) { + auto result = this->_hash_dict.find(this->_hash_dict.get_default_comparator(), oldRef); + assert(result != nullptr && result->first.load_relaxed() == oldRef); + result->first.store_release(newRef); + } } + ++itr; } - ++itr; + } else { + this->_hash_dict.move_keys([&compactable](EntryRef old_ref) { return compactable.move(old_ref); }); } } @@ -179,7 +166,11 @@ template <typename BTreeDictionaryT, typename ParentT, typename HashDictionaryT> uint32_t UniqueStoreDictionary<BTreeDictionaryT, ParentT, HashDictionaryT>::get_num_uniques() const { - return this->_btree_dict.size(); + if constexpr (has_btree_dictionary) { + return this->_btree_dict.size(); + } else { + return this->_hash_dict.size(); + } } template <typename BTreeDictionaryT, typename ParentT, typename HashDictionaryT> @@ -204,15 +195,18 @@ UniqueStoreDictionary<BTreeDictionaryT, ParentT, HashDictionaryT>::build(vespali { assert(refs.size() == ref_counts.size()); assert(!refs.empty()); - typename BTreeDictionaryType::Builder builder(this->_btree_dict.getAllocator()); - for (size_t i = 1; i < refs.size(); ++i) { - if (ref_counts[i] != 0u) { - builder.insert(refs[i], DataType()); - } else { - hold(refs[i]); + if constexpr (has_btree_dictionary) { + using DataType = typename BTreeDictionaryType::DataType; + typename BTreeDictionaryType::Builder builder(this->_btree_dict.getAllocator()); + for (size_t i = 1; i < refs.size(); ++i) { + if (ref_counts[i] != 0u) { + builder.insert(refs[i], DataType()); + } else { + hold(refs[i]); + } } + this->_btree_dict.assign(builder); } - this->_btree_dict.assign(builder); if constexpr (has_hash_dictionary) { for (size_t i = 1; i < refs.size(); ++i) { if (ref_counts[i] != 0u) { @@ -220,6 +214,8 @@ UniqueStoreDictionary<BTreeDictionaryT, ParentT, HashDictionaryT>::build(vespali std::function<EntryRef(void)> insert_hash_entry([ref]() noexcept -> EntryRef { return ref; }); auto& add_result = this->_hash_dict.add(this->_hash_dict.get_default_comparator(), ref, insert_hash_entry); assert(add_result.first.load_relaxed() == ref); + } else if constexpr (!has_btree_dictionary) { + hold(refs[i]); } } } @@ -229,11 +225,14 @@ template <typename BTreeDictionaryT, typename ParentT, typename HashDictionaryT> void UniqueStoreDictionary<BTreeDictionaryT, ParentT, HashDictionaryT>::build(vespalib::ConstArrayRef<EntryRef> refs) { - typename BTreeDictionaryType::Builder builder(this->_btree_dict.getAllocator()); - for (const auto& ref : refs) { - builder.insert(ref, DataType()); + if constexpr (has_btree_dictionary) { + using DataType = typename BTreeDictionaryType::DataType; + typename BTreeDictionaryType::Builder builder(this->_btree_dict.getAllocator()); + for (const auto& ref : refs) { + builder.insert(ref, DataType()); + } + this->_btree_dict.assign(builder); } - this->_btree_dict.assign(builder); if constexpr (has_hash_dictionary) { for (const auto& ref : refs) { std::function<EntryRef(void)> insert_hash_entry([ref]() noexcept -> EntryRef { return ref; }); @@ -249,33 +248,47 @@ UniqueStoreDictionary<BTreeDictionaryT, ParentT, HashDictionaryT>::build_with_pa vespalib::ConstArrayRef<uint32_t> payloads) { assert(refs.size() == payloads.size()); - typename BTreeDictionaryType::Builder builder(this->_btree_dict.getAllocator()); - for (size_t i = 0; i < refs.size(); ++i) { - if constexpr (std::is_same_v<DataType, uint32_t>) { - builder.insert(refs[i], payloads[i]); - } else { - builder.insert(refs[i], DataType()); + if constexpr (has_btree_dictionary) { + using DataType = typename BTreeDictionaryType::DataType; + typename BTreeDictionaryType::Builder builder(this->_btree_dict.getAllocator()); + for (size_t i = 0; i < refs.size(); ++i) { + if constexpr (std::is_same_v<DataType, uint32_t>) { + builder.insert(refs[i], payloads[i]); + } else { + builder.insert(refs[i], DataType()); + } } + this->_btree_dict.assign(builder); } - this->_btree_dict.assign(builder); if constexpr (has_hash_dictionary) { for (size_t i = 0; i < refs.size(); ++i) { EntryRef ref = refs[i]; std::function<EntryRef(void)> insert_hash_entry([ref]() noexcept -> EntryRef { return ref; }); auto& add_result = this->_hash_dict.add(this->_hash_dict.get_default_comparator(), ref, insert_hash_entry); assert(add_result.first.load_relaxed() == refs[i]); - if constexpr (std::is_same_v<DataType, uint32_t>) { - add_result.second.store_relaxed(EntryRef(payloads[i])); - } + add_result.second.store_relaxed(EntryRef(payloads[i])); } } } template <typename BTreeDictionaryT, typename ParentT, typename HashDictionaryT> -std::unique_ptr<typename ParentT::ReadSnapshot> +std::unique_ptr<IUniqueStoreDictionaryReadSnapshot> UniqueStoreDictionary<BTreeDictionaryT, ParentT, HashDictionaryT>::get_read_snapshot() const { - return std::make_unique<ReadSnapshotImpl>(this->_btree_dict.getFrozenView()); + if constexpr (has_btree_dictionary) { + return std::make_unique<UniqueStoreBTreeDictionaryReadSnapshot<BTreeDictionaryT>>(this->_btree_dict.getFrozenView()); + } + if constexpr (has_hash_dictionary) { + return std::make_unique<UniqueStoreHashDictionaryReadSnapshot<HashDictionaryT>>(this->_hash_dict); + } + return std::unique_ptr<IUniqueStoreDictionaryReadSnapshot>(); +} + +template <typename BTreeDictionaryT, typename ParentT, typename HashDictionaryT> +bool +UniqueStoreDictionary<BTreeDictionaryT, ParentT, HashDictionaryT>::get_has_btree_dictionary() const +{ + return has_btree_dictionary; } template <typename BTreeDictionaryT, typename ParentT, typename HashDictionaryT> diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_enumerator.h b/vespalib/src/vespa/vespalib/datastore/unique_store_enumerator.h index 78597a53dc8..be591649310 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store_enumerator.h +++ b/vespalib/src/vespa/vespalib/datastore/unique_store_enumerator.h @@ -3,6 +3,7 @@ #pragma once #include "i_unique_store_dictionary.h" +#include "i_unique_store_dictionary_read_snapshot.h" #include <vespa/vespalib/stllike/allocator.h> namespace vespalib::datastore { @@ -23,7 +24,7 @@ public: private: using UInt32Vector = std::vector<uint32_t, vespalib::allocator_large<uint32_t>>; using EnumValues = std::vector<UInt32Vector>; - IUniqueStoreDictionary::ReadSnapshot::UP _dict_snapshot; + std::unique_ptr<IUniqueStoreDictionaryReadSnapshot> _dict_snapshot; const DataStoreBase &_store; EnumValues _enumValues; uint32_t _next_enum_val; diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_enumerator.hpp b/vespalib/src/vespa/vespalib/datastore/unique_store_enumerator.hpp index dbcb2c10b86..7a43b16e66a 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store_enumerator.hpp +++ b/vespalib/src/vespa/vespalib/datastore/unique_store_enumerator.hpp @@ -15,6 +15,8 @@ UniqueStoreEnumerator<RefT>::UniqueStoreEnumerator(const IUniqueStoreDictionary _enumValues(), _next_enum_val(1) { + _dict_snapshot->fill(); + _dict_snapshot->sort(); allocate_enum_values(); } diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_hash_dictionary_read_snapshot.h b/vespalib/src/vespa/vespalib/datastore/unique_store_hash_dictionary_read_snapshot.h new file mode 100644 index 00000000000..947d8e00f95 --- /dev/null +++ b/vespalib/src/vespa/vespalib/datastore/unique_store_hash_dictionary_read_snapshot.h @@ -0,0 +1,34 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "i_unique_store_dictionary_read_snapshot.h" + +namespace vespalib::datastore { + +/** + * Class that provides a read snapshot of a unique store dictionary using a hash. + * + * A generation guard that must be taken and held while the snapshot is considered valid. + * + * fill() must be called by the writer thread. + * sort() must be called if order of refs should correspond to sorted dictionary order. + * + */ +template <typename HashDictionaryT> +class UniqueStoreHashDictionaryReadSnapshot : public IUniqueStoreDictionaryReadSnapshot { +private: + using HashDictionaryType = HashDictionaryT; + const HashDictionaryType& _hash; + std::vector<EntryRef> _refs; + +public: + UniqueStoreHashDictionaryReadSnapshot(const HashDictionaryType &hash); + void fill() override; + void sort() override; + size_t count(const EntryComparator& comp) const override; + size_t count_in_range(const EntryComparator& low, const EntryComparator& high) const override; + void foreach_key(std::function<void(EntryRef)> callback) const override; +}; + +} diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_hash_dictionary_read_snapshot.hpp b/vespalib/src/vespa/vespalib/datastore/unique_store_hash_dictionary_read_snapshot.hpp new file mode 100644 index 00000000000..4353b415aa4 --- /dev/null +++ b/vespalib/src/vespa/vespalib/datastore/unique_store_hash_dictionary_read_snapshot.hpp @@ -0,0 +1,55 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "unique_store_hash_dictionary_read_snapshot.h" + +namespace vespalib::datastore { + +template <typename HashDictionaryT> +UniqueStoreHashDictionaryReadSnapshot<HashDictionaryT>::UniqueStoreHashDictionaryReadSnapshot(const HashDictionaryType &hash) + : _hash(hash), + _refs() +{ +} + +template <typename HashDictionaryT> +void +UniqueStoreHashDictionaryReadSnapshot<HashDictionaryT>::fill() +{ + _hash.foreach_key([this](EntryRef ref) { _refs.push_back(ref); }); +} + +template <typename HashDictionaryT> +void +UniqueStoreHashDictionaryReadSnapshot<HashDictionaryT>::sort() +{ + auto& comp = _hash.get_default_comparator(); + std::sort(_refs.begin(), _refs.end(), [&comp](EntryRef lhs, EntryRef rhs) { return comp.less(lhs, rhs); }); +} + +template <typename HashDictionaryT> +size_t +UniqueStoreHashDictionaryReadSnapshot<HashDictionaryT>::count(const EntryComparator& comp) const +{ + auto result = _hash.find(comp, EntryRef()); + return ((result != nullptr) ? 1u : 0u); +} + +template <typename HashDictionaryT> +size_t +UniqueStoreHashDictionaryReadSnapshot<HashDictionaryT>::count_in_range(const EntryComparator&, const EntryComparator&) const +{ + return 1u; +} + +template <typename HashDictionaryT> +void +UniqueStoreHashDictionaryReadSnapshot<HashDictionaryT>::foreach_key(std::function<void(EntryRef)> callback) const +{ + for (auto ref : _refs) { + callback(ref); + } +} + +} diff --git a/vespalib/src/vespa/vespalib/stllike/hashtable.h b/vespalib/src/vespa/vespalib/stllike/hashtable.h index a5bea37b707..1a982b97208 100644 --- a/vespalib/src/vespa/vespalib/stllike/hashtable.h +++ b/vespalib/src/vespa/vespalib/stllike/hashtable.h @@ -1,9 +1,10 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <iterator> #include <vespa/vespalib/util/array.h> +#include <vespa/vespalib/util/traits.h> #include <algorithm> +#include <iterator> namespace vespalib { @@ -103,26 +104,75 @@ public: enum {npos=-1u, invalid=-2u}; hash_node() : _node(), _next(invalid) {} hash_node(const V & node, next_t next=npos) - : _node(node), _next(next) {} + : _next(next) + { + new (_node) V(node); + } hash_node(V &&node, next_t next=npos) - : _node(std::move(node)), _next(next) {} - hash_node(hash_node &&) noexcept = default; - hash_node &operator=(hash_node &&) noexcept = default; - hash_node(const hash_node &) = default; // These will not be created - hash_node &operator=(const hash_node &) = default; // if V is non-copyable. + : _next(next) + { + new (_node) V(std::move(node)); + } + hash_node(hash_node && rhs) noexcept + : _next(rhs._next) + { + if (rhs.valid()) { + new (_node) V(std::move(rhs.getValue())); + } + } + hash_node &operator=(hash_node && rhs) noexcept { + destruct(); + if (rhs.valid()) { + new (_node) V(std::move(rhs.getValue())); + _next = rhs._next; + } else { + _next = invalid; + } + return *this; + } + hash_node(const hash_node & rhs) + : _next(rhs._next) + { + if (rhs.valid()) { + new (_node) V(rhs.getValue()); + } + } + hash_node &operator=(const hash_node & rhs) { + destruct(); + if (rhs.valid()) { + new (_node) V(rhs.getValue()); + _next = rhs._next; + } else { + _next = invalid; + } + return *this; + } + ~hash_node() { + destruct(); + } bool operator == (const hash_node & rhs) const { - return (_next == rhs._next) && (_node == rhs._node); + return (_next == rhs._next) && (!valid() || (getValue() == rhs.getValue())); } - V & getValue() { return _node; } - const V & getValue() const { return _node; } + V & getValue() { return *reinterpret_cast<V *>(_node); } + const V & getValue() const { return *reinterpret_cast<const V *>(_node); } next_t getNext() const { return _next; } void setNext(next_t next) { _next = next; } - void invalidate() { _next = invalid; _node = V(); } + void invalidate() { + destruct(); + _next = invalid; + } void terminate() { _next = npos; } bool valid() const { return _next != invalid; } bool hasNext() const { return valid() && (_next != npos); } private: - V _node; + void destruct() { + if constexpr (!can_skip_destruction<V>::value) { + if (valid()) { + getValue().~V(); + } + } + } + char _node[sizeof(V)] alignas(V); next_t _next; }; diff --git a/vespalib/src/vespa/vespalib/util/alloc.cpp b/vespalib/src/vespa/vespalib/util/alloc.cpp index d86d35be4c7..5f1faa785ba 100644 --- a/vespalib/src/vespa/vespalib/util/alloc.cpp +++ b/vespalib/src/vespa/vespalib/util/alloc.cpp @@ -467,34 +467,6 @@ MemoryAllocator::select_allocator(size_t mmapLimit, size_t alignment) { return & AutoAllocator::getAllocator(mmapLimit, alignment); } -Alloc::Alloc(const MemoryAllocator * allocator, size_t sz) noexcept - : _alloc(allocator->alloc(sz)), - _allocator(allocator) -{ -} - -Alloc::~Alloc() -{ - if (_alloc.first != nullptr) { - _allocator->free(_alloc); - _alloc.first = nullptr; - } -} - -Alloc& -Alloc::operator=(Alloc && rhs) noexcept -{ - if (this != & rhs) { - if (_alloc.first != nullptr) { - _allocator->free(_alloc); - } - _alloc = rhs._alloc; - _allocator = rhs._allocator; - rhs.clear(); - } - return *this; -} - Alloc Alloc::allocHeap(size_t sz) { diff --git a/vespalib/src/vespa/vespalib/util/alloc.h b/vespalib/src/vespa/vespalib/util/alloc.h index a1bb6af6514..f608a244035 100644 --- a/vespalib/src/vespa/vespalib/util/alloc.h +++ b/vespalib/src/vespa/vespalib/util/alloc.h @@ -1,13 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/vespalib/util/optimized.h> +#include "optimized.h" +#include "memory_allocator.h" #include <memory> namespace vespalib::alloc { -class MemoryAllocator; - /** * This represents an allocation. * It can be created, moved, swapped. @@ -41,9 +40,24 @@ public: { rhs.clear(); } - Alloc & operator=(Alloc && rhs) noexcept; + Alloc & operator=(Alloc && rhs) noexcept { + if (this != & rhs) { + if (_alloc.first != nullptr) { + _allocator->free(_alloc); + } + _alloc = rhs._alloc; + _allocator = rhs._allocator; + rhs.clear(); + } + return *this; + } Alloc() noexcept : _alloc(nullptr, 0), _allocator(nullptr) { } - ~Alloc(); + ~Alloc() { + if (_alloc.first != nullptr) { + _allocator->free(_alloc); + _alloc.first = nullptr; + } + } void swap(Alloc & rhs) noexcept { std::swap(_alloc, rhs._alloc); std::swap(_allocator, rhs._allocator); @@ -65,8 +79,15 @@ public: static Alloc alloc() noexcept; static Alloc alloc_with_allocator(const MemoryAllocator* allocator) noexcept; private: - Alloc(const MemoryAllocator * allocator, size_t sz) noexcept; - Alloc(const MemoryAllocator * allocator) noexcept : _alloc(nullptr, 0), _allocator(allocator) { } + Alloc(const MemoryAllocator * allocator, size_t sz) noexcept + : _alloc(allocator->alloc(sz)), + _allocator(allocator) + { + } + Alloc(const MemoryAllocator * allocator) noexcept + : _alloc(nullptr, 0), + _allocator(allocator) + { } void clear() { _alloc.first = nullptr; _alloc.second = 0; diff --git a/vespalib/src/vespa/vespalib/util/approx.h b/vespalib/src/vespa/vespalib/util/approx.h index 6afe8041476..8c6baa648ea 100644 --- a/vespalib/src/vespa/vespalib/util/approx.h +++ b/vespalib/src/vespa/vespalib/util/approx.h @@ -12,7 +12,7 @@ namespace vespalib { * step 1 unit in the last place towards the other number. This means the * two numbers must be equal to 23 bits precision. **/ -inline bool approx_equal(double a, double b) +constexpr bool approx_equal(double a, double b) { if (a == b) return true; if (a > 1.0 || a < -1.0) { diff --git a/vespalib/src/vespa/vespalib/util/compress.h b/vespalib/src/vespa/vespalib/util/compress.h index f4190c999c3..2bab2083b34 100644 --- a/vespalib/src/vespa/vespalib/util/compress.h +++ b/vespalib/src/vespa/vespalib/util/compress.h @@ -42,7 +42,7 @@ public: const uint8_t * src = static_cast<const uint8_t *>(srcv); const uint8_t c = src[0]; size_t numbytes; - if (c & 0x40) { + if (__builtin_expect(c & 0x40, false)) { if (c & 0x20) { n = ((c & 0x1f) << 24) + (src[1] << 16) + (src[2] << 8) + src[3]; numbytes = 4; diff --git a/vespalib/src/vespa/vespalib/util/require.h b/vespalib/src/vespa/vespalib/util/require.h index c666a06c8d6..a4283520314 100644 --- a/vespalib/src/vespa/vespalib/util/require.h +++ b/vespalib/src/vespa/vespalib/util/require.h @@ -3,11 +3,87 @@ #pragma once #include "macro.h" +#include "approx.h" +#include "classname.h" #include <iostream> +#include <type_traits> #include <vespa/vespalib/util/exception.h> +#include <vespa/vespalib/objects/hexdump.h> namespace vespalib { +namespace require_impl { + +//----------------------------------------------------------------------------- + +template<typename A, typename B> +using comparability = decltype(std::declval<const A &>() == std::declval<const B &>()); + +template<typename A, typename B, class = void> +struct are_comparable : std::false_type {}; + +template<typename A, typename B> +struct are_comparable<A,B,std::void_t<comparability<A,B>>> : std::true_type{}; + +//----------------------------------------------------------------------------- + +template<typename S, typename V> +using streamability = decltype(std::declval<S &>() << std::declval<const V &>()); + +template<typename S, typename V, class = void> +struct is_streamable : std::false_type {}; + +template<typename S, typename V> +struct is_streamable<S,V,std::void_t<streamability<S,V>>> : std::true_type{}; + +//----------------------------------------------------------------------------- + +// memcmp is not constexpr, so we need an alternative +constexpr bool mem_eq(const char *a, const char *b, size_t len) { + for (size_t i = 0; i < len; ++i) { + if (a[i] != b[i]) { + return false; + } + } + return true; +} + +template <class A, class B> +constexpr bool eq(const A &a, const B &b) { + if constexpr (are_comparable<A,B>()) { + return (a == b); + } else if constexpr (std::is_same_v<A,B> && + std::has_unique_object_representations_v<A>) + { + static_assert(sizeof(A) == sizeof(B)); + return mem_eq(reinterpret_cast<const char *>(std::addressof(a)), + reinterpret_cast<const char *>(std::addressof(b)), + sizeof(A)); + } else { + static_assert(are_comparable<A,B>(), "values are not comparable"); + return false; + } +} + +constexpr bool eq(double a, double b) { return approx_equal(a, b); } + +//----------------------------------------------------------------------------- + +template <typename S, typename V> +void print(S &os, const V &value) { + if constexpr (is_streamable<S,V>()) { + os << value; + } else if constexpr (std::has_unique_object_representations_v<V>) { + os << "(" << getClassName<V>() << ") " << HexDump(std::addressof(value), sizeof(V)); + } else { + os << "not printable (type: " << getClassName<V>() << ")"; + } +} + +//----------------------------------------------------------------------------- + +} // namespace require_impl + VESPA_DEFINE_EXCEPTION(RequireFailedException, Exception); constexpr void handle_require_success() {} @@ -22,8 +98,12 @@ void handle_require_eq_failure [[noreturn]] (const A& a, const B& b, const char { std::cerr << file << ":" << line << ": error: "; std::cerr << "expected (" << a_desc << " == " << b_desc << ")\n"; - std::cerr << " lhs (" << a_desc << ") is: " << a << "\n"; - std::cerr << " rhs (" << b_desc << ") is: " << b << "\n"; + std::cerr << " lhs (" << a_desc << ") is: "; + require_impl::print(std::cerr, a); + std::cerr << "\n"; + std::cerr << " rhs (" << b_desc << ") is: "; + require_impl::print(std::cerr, b); + std::cerr << "\n"; throw_require_failed(description, file, line); } @@ -45,7 +125,7 @@ void handle_require_eq_failure [[noreturn]] (const A& a, const B& b, const char * for the value types. **/ #define REQUIRE_EQ(a, b) \ - (a == b) ? vespalib::handle_require_success() : \ + vespalib::require_impl::eq(a, b) ? vespalib::handle_require_success() : \ vespalib::handle_require_eq_failure(a, b, \ VESPA_STRINGIZE(a), VESPA_STRINGIZE(b), \ VESPA_STRINGIZE(a) " == " VESPA_STRINGIZE(b), \ diff --git a/vespamalloc/src/vespamalloc/malloc/malloc.cpp b/vespamalloc/src/vespamalloc/malloc/malloc.cpp index 9653f0ae645..380e2844f42 100644 --- a/vespamalloc/src/vespamalloc/malloc/malloc.cpp +++ b/vespamalloc/src/vespamalloc/malloc/malloc.cpp @@ -3,6 +3,7 @@ #include <vespamalloc/malloc/memorywatcher.h> #include <vespamalloc/malloc/memblock.h> #include <vespamalloc/malloc/stat.h> +#include <vespamalloc/malloc/threadpool.hpp> namespace vespamalloc { diff --git a/vespamalloc/src/vespamalloc/malloc/threadpool.h b/vespamalloc/src/vespamalloc/malloc/threadpool.h index f719df026fe..1846bbdab4d 100644 --- a/vespamalloc/src/vespamalloc/malloc/threadpool.h +++ b/vespamalloc/src/vespamalloc/malloc/threadpool.h @@ -19,8 +19,8 @@ public: void setPool(AllocPool & pool) { _allocPool = & pool; } - void malloc(size_t sz, MemBlockPtrT & mem) __attribute__((noinline)); - void free(MemBlockPtrT mem, SizeClassT sc) __attribute__((noinline)); + void malloc(size_t sz, MemBlockPtrT & mem); + void free(MemBlockPtrT mem, SizeClassT sc); void info(FILE * os, size_t level, const DataSegment<MemBlockPtrT> & ds) const __attribute__((noinline)); /** diff --git a/zookeeper-client-common/src/main/java/com/yahoo/vespa/zookeeper/client/ZkClientConfigBuilder.java b/zookeeper-client-common/src/main/java/com/yahoo/vespa/zookeeper/client/ZkClientConfigBuilder.java index 4f96bba22b0..8236fdedc71 100644 --- a/zookeeper-client-common/src/main/java/com/yahoo/vespa/zookeeper/client/ZkClientConfigBuilder.java +++ b/zookeeper-client-common/src/main/java/com/yahoo/vespa/zookeeper/client/ZkClientConfigBuilder.java @@ -46,7 +46,7 @@ public class ZkClientConfigBuilder { public ZKClientConfig toConfig(Path configFile) throws IOException, QuorumPeerConfig.ConfigException { String configString = toConfigString(); Files.createDirectories(configFile.getParent()); - Path tempFile = configFile.resolveSibling("." + configFile.getFileName() + ".tmp"); + Path tempFile = Files.createTempFile(configFile.toAbsolutePath().getParent(), "." + configFile.getFileName(), ".tmp"); Files.writeString(tempFile, configString); Files.move(tempFile, configFile, StandardCopyOption.ATOMIC_MOVE); return new ZKClientConfig(configFile.toString()); |