diff options
39 files changed, 419 insertions, 291 deletions
diff --git a/client/go/go.mod b/client/go/go.mod index 8d46118dea6..b69a64f87f7 100644 --- a/client/go/go.mod +++ b/client/go/go.mod @@ -11,7 +11,7 @@ require ( github.com/klauspost/compress v1.17.4 github.com/mattn/go-colorable v0.1.13 github.com/mattn/go-isatty v0.0.20 - github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 + github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c github.com/spf13/cobra v1.8.0 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.8.4 diff --git a/client/go/go.sum b/client/go/go.sum index c4dd9837194..6af9dbf84c9 100644 --- a/client/go/go.sum +++ b/client/go/go.sum @@ -32,6 +32,8 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= @@ -53,6 +55,7 @@ golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= diff --git a/config-model-api/abi-spec.json b/config-model-api/abi-spec.json index c3e20e534ff..363ab3918d3 100644 --- a/config-model-api/abi-spec.json +++ b/config-model-api/abi-spec.json @@ -771,7 +771,9 @@ "attributes" : [ "public" ], - "methods" : [ ], + "methods" : [ + "public java.util.Map messagesById()" + ], "fields" : [ ] }, "com.yahoo.config.application.api.ValidationOverrides" : { @@ -1457,9 +1459,9 @@ ], "methods" : [ "public abstract long aggregatedModelCostInBytes()", - "public abstract void registerModel(com.yahoo.config.application.api.ApplicationFile)", + "public void registerModel(com.yahoo.config.application.api.ApplicationFile)", "public abstract void registerModel(com.yahoo.config.application.api.ApplicationFile, com.yahoo.config.model.api.OnnxModelOptions)", - "public abstract void registerModel(java.net.URI)", + "public void registerModel(java.net.URI)", "public abstract void registerModel(java.net.URI, com.yahoo.config.model.api.OnnxModelOptions)", "public abstract java.util.Map models()", "public abstract void setRestartOnDeploy()", @@ -1481,9 +1483,7 @@ "public void <init>()", "public com.yahoo.config.model.api.OnnxModelCost$Calculator newCalculator(com.yahoo.config.application.api.ApplicationPackage, com.yahoo.config.provision.ApplicationId)", "public long aggregatedModelCostInBytes()", - "public void registerModel(com.yahoo.config.application.api.ApplicationFile)", "public void registerModel(com.yahoo.config.application.api.ApplicationFile, com.yahoo.config.model.api.OnnxModelOptions)", - "public void registerModel(java.net.URI)", "public void registerModel(java.net.URI, com.yahoo.config.model.api.OnnxModelOptions)", "public java.util.Map models()", "public void setRestartOnDeploy()", @@ -1501,7 +1501,9 @@ "record" ], "methods" : [ + "public void <init>(java.lang.String, long, long, com.yahoo.config.model.api.OnnxModelOptions)", "public void <init>(java.lang.String, long, long, java.util.Optional)", + "public com.yahoo.config.model.api.OnnxModelOptions options()", "public final java.lang.String toString()", "public final int hashCode()", "public final boolean equals(java.lang.Object)", diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/ValidationOverrides.java b/config-model-api/src/main/java/com/yahoo/config/application/api/ValidationOverrides.java index 1edddb63e52..7b52d825473 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/ValidationOverrides.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/ValidationOverrides.java @@ -15,6 +15,7 @@ import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -36,7 +37,7 @@ public class ValidationOverrides { private final String xmlForm; - /** Creates a validation overrides which does not have an xml form */ + /** Creates a validation overrides which does not have an XML form */ public ValidationOverrides(List<Allow> overrides) { this(overrides, null); } @@ -163,10 +164,13 @@ public class ValidationOverrides { */ public static class ValidationException extends IllegalArgumentException { + private final Map<ValidationId, Collection<String>> messagesById = new LinkedHashMap<>(); + static final long serialVersionUID = 789984668; private ValidationException(ValidationId validationId, String message) { super(validationId + ": " + message + ". " + toAllowMessage(validationId)); + messagesById.put(validationId, List.of(message)); } private ValidationException(Map<ValidationId, Collection<String>> messagesById) { @@ -175,8 +179,11 @@ public class ValidationOverrides { String.join("\n\t", messages.getValue()) + "\n" + toAllowMessage(messages.getKey())) .collect(Collectors.joining("\n"))); + messagesById.forEach((id, messages) -> this.messagesById.put(id, List.copyOf(messages))); } + public Map<ValidationId, Collection<String>> messagesById() { return Map.copyOf(messagesById); } + } } diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/OnnxModelCost.java b/config-model-api/src/main/java/com/yahoo/config/model/api/OnnxModelCost.java index d70b751eba0..69f2b6b6dce 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/OnnxModelCost.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/OnnxModelCost.java @@ -18,9 +18,15 @@ public interface OnnxModelCost { interface Calculator { long aggregatedModelCostInBytes(); - void registerModel(ApplicationFile path); + // TODO: Unused, remove when 8.263.7 is oldest model in use + default void registerModel(ApplicationFile path) { + registerModel(path, OnnxModelOptions.empty()); + } void registerModel(ApplicationFile path, OnnxModelOptions onnxModelOptions); - void registerModel(URI uri); + // TODO: Unused, remove when 8.263.7 is oldest model in use + default void registerModel(URI uri) { + registerModel(uri, OnnxModelOptions.empty()); + } void registerModel(URI uri, OnnxModelOptions onnxModelOptions); Map<String, ModelInfo> models(); void setRestartOnDeploy(); @@ -28,16 +34,24 @@ public interface OnnxModelCost { void store(); } - record ModelInfo(String modelId, long estimatedCost, long hash, Optional<OnnxModelOptions> onnxModelOptions) {} + record ModelInfo(String modelId, long estimatedCost, long hash, Optional<OnnxModelOptions> onnxModelOptions) { + + public ModelInfo(String modelId, long estimatedCost, long hash, OnnxModelOptions onnxModelOptions) { + this(modelId, estimatedCost, hash, Optional.of(onnxModelOptions)); + } + + public OnnxModelOptions options() { + return onnxModelOptions.orElseThrow(() -> new IllegalStateException("No onnxModelOptions exist")); + } + + } static OnnxModelCost disabled() { return new DisabledOnnxModelCost(); } class DisabledOnnxModelCost implements OnnxModelCost, Calculator { @Override public Calculator newCalculator(ApplicationPackage appPkg, ApplicationId applicationId) { return this; } @Override public long aggregatedModelCostInBytes() {return 0;} - @Override public void registerModel(ApplicationFile path) {} @Override public void registerModel(ApplicationFile path, OnnxModelOptions onnxModelOptions) {} - @Override public void registerModel(URI uri) {} @Override public void registerModel(URI uri, OnnxModelOptions onnxModelOptions) {} @Override public Map<String, ModelInfo> models() { return Map.of(); } @Override public void setRestartOnDeploy() {} diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java index 33dfee58d1a..f19341098f4 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java @@ -172,7 +172,7 @@ public class DeployState implements ConfigDefinitionStore { /** Get the global rank profile registry for this application. */ public final RankProfileRegistry rankProfileRegistry() { return rankProfileRegistry; } - /** Returns the validation overrides of this. This is never null */ + /** Returns the validation overrides of this. This is never null. */ public ValidationOverrides validationOverrides() { return validationOverrides; } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java index 903f1c06024..65572d07cc2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java @@ -217,13 +217,6 @@ public class VespaModelFactory implements ModelFactory { private List<ConfigChangeAction> validateModel(VespaModel model, DeployState deployState, ValidationParameters validationParameters) { try { return new Validation(additionalValidators).validate(model, validationParameters, deployState); - } catch (ValidationOverrides.ValidationException e) { - if (deployState.isHosted() && zone.environment().isManuallyDeployed()) - deployState.getDeployLogger().logApplicationPackage(Level.WARNING, - "Auto-overriding validation which would be disallowed in production: " + - Exceptions.toMessageString(e)); - else - rethrowUnlessIgnoreErrors(e, validationParameters.ignoreValidationErrors()); } catch (IllegalArgumentException | TransientException | QuotaExceededException e) { rethrowUnlessIgnoreErrors(e, validationParameters.ignoreValidationErrors()); } catch (Exception e) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java index 56277345515..0f7a415c33a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java @@ -3,8 +3,8 @@ package com.yahoo.vespa.model.application.validation; import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.application.api.ValidationOverrides; +import com.yahoo.config.application.api.ValidationOverrides.ValidationException; import com.yahoo.config.model.api.ConfigChangeAction; -import com.yahoo.config.model.api.Model; import com.yahoo.config.model.api.ValidationParameters; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.provision.ClusterSpec; @@ -25,21 +25,17 @@ import com.yahoo.vespa.model.application.validation.change.RestartOnDeployForOnn import com.yahoo.vespa.model.application.validation.change.StartupCommandChangeValidator; import com.yahoo.vespa.model.application.validation.change.StreamingSearchClusterChangeValidator; import com.yahoo.vespa.model.application.validation.first.RedundancyValidator; +import com.yahoo.yolean.Exceptions; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; +import java.util.ArrayList; +import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Set; +import java.util.logging.Level; import java.util.stream.Collectors; -import static java.util.stream.Collectors.groupingBy; -import static java.util.stream.Collectors.mapping; -import static java.util.stream.Collectors.toCollection; - /** * Executor of validators. This defines the right order of validator execution. * @@ -51,7 +47,7 @@ public class Validation { public Validation() { this(List.of()); } - /** Create instance taking additional validators (e.g for cloud applications) */ + /** Create instance taking additional validators (e.g., for cloud applications) */ public Validation(List<Validator> additionalValidators) { this.additionalValidators = additionalValidators; } /** @@ -62,85 +58,80 @@ public class Validation { * @throws ValidationOverrides.ValidationException if the change fails validation */ public List<ConfigChangeAction> validate(VespaModel model, ValidationParameters validationParameters, DeployState deployState) { + Execution execution = new Execution(model, deployState); if (validationParameters.checkRouting()) { - new RoutingValidator().validate(model, deployState); - new RoutingSelectorValidator().validate(model, deployState); + validateRouting(execution); } - new SchemasDirValidator().validate(model, deployState); - new BundleValidator().validate(model, deployState); - new PublicApiBundleValidator().validate(model, deployState); - new SearchDataTypeValidator().validate(model, deployState); - new ComplexFieldsWithStructFieldAttributesValidator().validate(model, deployState); - new ComplexFieldsWithStructFieldIndexesValidator().validate(model, deployState); - new StreamingValidator().validate(model, deployState); - new RankSetupValidator(validationParameters.ignoreValidationErrors()).validate(model, deployState); - new NoPrefixForIndexes().validate(model, deployState); - new ContainerInCloudValidator().validate(model, deployState); - new DeploymentSpecValidator().validate(model, deployState); - new ValidationOverridesValidator().validate(model, deployState); - new ConstantValidator().validate(model, deployState); - new SecretStoreValidator().validate(model, deployState); - new EndpointCertificateSecretsValidator().validate(model, deployState); - new AccessControlFilterValidator().validate(model, deployState); - new QuotaValidator().validate(model, deployState); - new UriBindingsValidator().validate(model, deployState); - new CloudDataPlaneFilterValidator().validate(model, deployState); - new AccessControlFilterExcludeValidator().validate(model, deployState); - new CloudUserFilterValidator().validate(model, deployState); - new CloudHttpConnectorValidator().validate(model, deployState); - new UrlConfigValidator().validate(model, deployState); - new JvmHeapSizeValidator().validate(model, deployState); - - additionalValidators.forEach(v -> v.validate(model, deployState)); - - List<ConfigChangeAction> result = Collections.emptyList(); + + validateModel(validationParameters, execution); + + additionalValidators.forEach(execution::run); + if (deployState.getProperties().isFirstTimeDeployment()) { - validateFirstTimeDeployment(model, deployState); - } else { - Optional<Model> currentActiveModel = deployState.getPreviousModel(); - if (currentActiveModel.isPresent() && (currentActiveModel.get() instanceof VespaModel)) { - result = validateChanges((VespaModel) currentActiveModel.get(), model, deployState); - deferConfigChangesForClustersToBeRestarted(result, model); - } + validateFirstTimeDeployment(execution); + } + else if (deployState.getPreviousModel().isPresent() && (deployState.getPreviousModel().get() instanceof VespaModel vespaModel)) { + validateChanges(vespaModel, execution); + // TODO: Why is this done here? It won't be done on more than one config server? + deferConfigChangesForClustersToBeRestarted(execution.actions, model); } - return result; + + execution.throwIfFailed(); + return execution.actions; } - private static List<ConfigChangeAction> validateChanges(VespaModel currentModel, VespaModel nextModel, - DeployState deployState) { - ChangeValidator[] validators = new ChangeValidator[] { - new IndexingModeChangeValidator(), - new GlobalDocumentChangeValidator(), - new IndexedSearchClusterChangeValidator(), - new StreamingSearchClusterChangeValidator(), - new ConfigValueChangeValidator(), - new StartupCommandChangeValidator(), - new ContentTypeRemovalValidator(), - new ContentClusterRemovalValidator(), - new ResourcesReductionValidator(), - new ResourcesReductionValidator(), - new ContainerRestartValidator(), - new NodeResourceChangeValidator(), - new RedundancyIncreaseValidator(), - new CertificateRemovalChangeValidator(), - new RedundancyValidator(), - new RestartOnDeployForOnnxModelChangesValidator(), - }; - List<ConfigChangeAction> actions = Arrays.stream(validators) - .flatMap(v -> v.validate(currentModel, nextModel, deployState).stream()) - .toList(); - - Map<ValidationId, Collection<String>> disallowableActions = actions.stream() - .filter(action -> action.validationId().isPresent()) - .collect(groupingBy(action -> action.validationId().orElseThrow(), - mapping(ConfigChangeAction::getMessage, - toCollection(LinkedHashSet::new)))); - deployState.validationOverrides().invalid(disallowableActions, deployState.now()); - return actions; + private static void validateRouting(Execution execution) { + execution.run(new RoutingValidator()); + execution.run(new RoutingSelectorValidator()); } - private static void validateFirstTimeDeployment(VespaModel model, DeployState deployState) { - new RedundancyValidator().validate(model, deployState); + private static void validateModel(ValidationParameters validationParameters, Execution execution) { + execution.run(new SchemasDirValidator()); + execution.run(new BundleValidator()); + execution.run(new PublicApiBundleValidator()); + execution.run(new SearchDataTypeValidator()); + execution.run(new ComplexFieldsWithStructFieldAttributesValidator()); + execution.run(new ComplexFieldsWithStructFieldIndexesValidator()); + execution.run(new StreamingValidator()); + execution.run(new RankSetupValidator(validationParameters.ignoreValidationErrors())); + execution.run(new NoPrefixForIndexes()); + execution.run(new ContainerInCloudValidator()); + execution.run(new DeploymentSpecValidator()); + execution.run(new ValidationOverridesValidator()); + execution.run(new ConstantValidator()); + execution.run(new SecretStoreValidator()); + execution.run(new EndpointCertificateSecretsValidator()); + execution.run(new AccessControlFilterValidator()); + execution.run(new QuotaValidator()); + execution.run(new UriBindingsValidator()); + execution.run(new CloudDataPlaneFilterValidator()); + execution.run(new AccessControlFilterExcludeValidator()); + execution.run(new CloudUserFilterValidator()); + execution.run(new CloudHttpConnectorValidator()); + execution.run(new UrlConfigValidator()); + execution.run(new JvmHeapSizeValidator()); + } + + private static void validateFirstTimeDeployment(Execution execution) { + execution.run(new RedundancyValidator()); + } + + private static void validateChanges(VespaModel currentModel, Execution execution) { + execution.run(new IndexingModeChangeValidator(), currentModel); + execution.run(new GlobalDocumentChangeValidator(), currentModel); + execution.run(new IndexedSearchClusterChangeValidator(), currentModel); + execution.run(new StreamingSearchClusterChangeValidator(), currentModel); + execution.run(new ConfigValueChangeValidator(), currentModel); + execution.run(new StartupCommandChangeValidator(), currentModel); + execution.run(new ContentTypeRemovalValidator(), currentModel); + execution.run(new ContentClusterRemovalValidator(), currentModel); + execution.run(new ResourcesReductionValidator(), currentModel); + execution.run(new ContainerRestartValidator(), currentModel); + execution.run(new NodeResourceChangeValidator(), currentModel); + execution.run(new RedundancyIncreaseValidator(), currentModel); + execution.run(new CertificateRemovalChangeValidator(), currentModel); + execution.run(new RedundancyValidator(), currentModel); + execution.run(new RestartOnDeployForOnnxModelChangesValidator(), currentModel); } private static void deferConfigChangesForClustersToBeRestarted(List<ConfigChangeAction> actions, VespaModel model) { @@ -159,4 +150,61 @@ public class Validation { } } + + private static class Execution { + + private final Map<ValidationId, List<String>> failures = new LinkedHashMap<>(); + private final VespaModel model; + private final DeployState deployState; + private final List<ConfigChangeAction> actions = new ArrayList<>(); + + private Execution(VespaModel model, DeployState deployState) { + this.model = model; + this.deployState = deployState; + } + + private void run(Validator validator) { + try { + validator.validate(model, deployState); + } + catch (ValidationException e) { + e.messagesById().forEach((id, messages) -> failures.computeIfAbsent(id, __ -> new ArrayList<>()).addAll(messages)); + } + } + + private void run(ChangeValidator validator, VespaModel previousModel) { + try { + // Some change validators throw, while some return a list of changes that may again be disallowed. + for (ConfigChangeAction action : validator.validate(previousModel, model, deployState)) { + actions.add(action); + if (action.validationId().isPresent()) run(new Validator() { // Changes without a validation ID are always allowed. + @Override public void validate(VespaModel model, DeployState deployState) { + deployState.validationOverrides().invalid(action.validationId().get(), action.getMessage(), deployState.now()); + } + }); + } + } + catch (ValidationException e) { + e.messagesById().forEach((id, messages) -> failures.computeIfAbsent(id, __ -> new ArrayList<>()).addAll(messages)); + } + } + + private void throwIfFailed() { + try { + if (failures.size() == 1 && failures.values().iterator().next().size() == 1) // Retain single-form exception message when possible. + deployState.validationOverrides().invalid(failures.keySet().iterator().next(), failures.values().iterator().next().get(0), deployState.now()); + else + deployState.validationOverrides().invalid(failures, deployState.now()); + } + catch (ValidationException e) { + if (deployState.isHosted() && deployState.zone().environment().isManuallyDeployed()) + deployState.getDeployLogger().logApplicationPackage(Level.WARNING, + "Auto-overriding validation which would be disallowed in production: " + + Exceptions.toMessageString(e)); + else throw e; + } + } + + } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidator.java index fb07f65b5f4..e2dd3aca0b9 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidator.java @@ -40,7 +40,7 @@ public class IndexingScriptChangeValidator { String fieldName = nextField.getName(); ImmutableSDField currentField = currentSchema.getConcreteField(fieldName); if (currentField != null) { - validateScripts(currentField, nextField).ifPresent(r -> result.add(r)); + validateScripts(currentField, nextField).ifPresent(result::add); } else if (nextField.isExtraField()) { result.add(VespaReindexAction.of(id, diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidatorTest.java index 0b32194e257..31e4c661151 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidatorTest.java @@ -126,16 +126,9 @@ class JvmHeapSizeValidatorTest { @Override public boolean restartOnDeploy() { return false;} @Override public void store() {} @Override public long aggregatedModelCostInBytes() { return totalCost.get(); } - @Override public void registerModel(ApplicationFile path) {} @Override public void registerModel(ApplicationFile path, OnnxModelOptions onnxModelOptions) {} @Override - public void registerModel(URI uri) { - assertEquals("https://my/url/model.onnx", uri.toString()); - totalCost.addAndGet(modelCost); - } - - @Override public void registerModel(URI uri, OnnxModelOptions onnxModelOptions) { assertEquals("https://my/url/model.onnx", uri.toString()); totalCost.addAndGet(modelCost); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/CertificateRemovalChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/CertificateRemovalChangeValidatorTest.java index 6b7df8871aa..bc36b800bfb 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/CertificateRemovalChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/CertificateRemovalChangeValidatorTest.java @@ -23,9 +23,11 @@ import static org.junit.jupiter.api.Assertions.assertThrows; public class CertificateRemovalChangeValidatorTest { private static final String validationOverrides = - "<validation-overrides>\n" + - " <allow until='2000-01-14' comment='test override'>certificate-removal</allow>\n" + - "</validation-overrides>\n"; + """ + <validation-overrides> + <allow until='2000-01-14' comment='test override'>certificate-removal</allow> + </validation-overrides> + """; @Test void validate() { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexingModeChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexingModeChangeValidatorTest.java index 3fd3180b37e..9e0eab9aba7 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexingModeChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexingModeChangeValidatorTest.java @@ -22,19 +22,28 @@ import static org.junit.jupiter.api.Assertions.fail; public class IndexingModeChangeValidatorTest { @Test + void testChangingIndexModeFromIndexedToStreamingWhenDisallowedButInDev() { + ValidationTester tester = new ValidationTester(); + + VespaModel oldModel = + tester.deploy(null, getServices("index"), Environment.dev, "<validation-overrides />").getFirst(); + List<ConfigChangeAction> actions = tester.deploy(oldModel, getServices("streaming"), Environment.dev, "<calidation-overrides />").getSecond(); + assertReindexingChange("Document type 'music' in cluster 'default-content' changed indexing mode from 'indexed' to 'streaming'", actions); + } + + @Test void testChangingIndexModeFromIndexedToStreamingWhenDisallowed() { ValidationTester tester = new ValidationTester(); VespaModel oldModel = tester.deploy(null, getServices("index"), Environment.prod, "<validation-overrides />").getFirst(); try { - List<ConfigChangeAction> changeActions = - tester.deploy(oldModel, getServices("streaming"), Environment.prod, "<calidation-overrides />").getSecond(); + tester.deploy(oldModel, getServices("streaming"), Environment.prod, "<calidation-overrides />").getSecond(); fail("Should throw on disallowed config change action"); } catch (ValidationException e) { - assertEquals("indexing-mode-change:\n" + - "\tDocument type 'music' in cluster 'default-content' changed indexing mode from 'indexed' to 'streaming'\n" + + assertEquals("indexing-mode-change: " + + "Document type 'music' in cluster 'default-content' changed indexing mode from 'indexed' to 'streaming'. " + "To allow this add <allow until='yyyy-mm-dd'>indexing-mode-change</allow> to validation-overrides.xml, see https://docs.vespa.ai/en/reference/validation-overrides.html", e.getMessage()); } @@ -94,8 +103,10 @@ public class IndexingModeChangeValidatorTest { } private static final String validationOverrides = - "<validation-overrides>\n" + - " <allow until='2000-01-14' comment='test override'>indexing-mode-change</allow>\n" + - "</validation-overrides>\n"; + """ + <validation-overrides> + <allow until='2000-01-14' comment='test override'>indexing-mode-change</allow> + </validation-overrides> + """; } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidatorTest.java index f9c0d00ac2c..b7e612127c1 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidatorTest.java @@ -29,6 +29,14 @@ public class ResourcesReductionValidatorTest { private final ValidationTester tester = new ValidationTester(provisioner); @Test + void ok_when_reduction_by_over_50_percent_in_hosted_dev() { + var fromResources = new NodeResources(8, 64, 800, 1); + var toResources = new NodeResources(8, 16, 800, 1); + VespaModel previous = tester.deploy(null, contentServices(6, fromResources), Environment.dev, null, CONTAINER_CLUSTER).getFirst(); + tester.deploy(previous, contentServices(6, toResources), Environment.dev, null, CONTAINER_CLUSTER); + } + + @Test void fail_when_reduction_by_over_50_percent() { var fromResources = new NodeResources(8, 64, 800, 1); var toResources = new NodeResources(8, 16, 800, 1); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForOnnxModelChangesValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForOnnxModelChangesValidatorTest.java index adcf58785fa..13389689de5 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForOnnxModelChangesValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForOnnxModelChangesValidatorTest.java @@ -15,7 +15,6 @@ import java.net.URI; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Optional; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -89,17 +88,11 @@ public class RestartOnDeployForOnnxModelChangesValidatorTest { public long aggregatedModelCostInBytes() { return estimatedCost; } @Override - public void registerModel(ApplicationFile path) {} - - @Override public void registerModel(ApplicationFile path, OnnxModelOptions onnxModelOptions) {} @Override - public void registerModel(URI uri) {} - - @Override public void registerModel(URI uri, OnnxModelOptions onnxModelOptions) { - models.put(uri.toString(), new OnnxModelCost.ModelInfo(uri.toString(), estimatedCost, hash, Optional.ofNullable(onnxModelOptions))); + models.put(uri.toString(), new OnnxModelCost.ModelInfo(uri.toString(), estimatedCost, hash, onnxModelOptions)); } @Override diff --git a/dependency-versions/pom.xml b/dependency-versions/pom.xml index 308b7842c18..fe2448fa174 100644 --- a/dependency-versions/pom.xml +++ b/dependency-versions/pom.xml @@ -62,7 +62,7 @@ <apache.httpcore5.vespa.version>5.2.4</apache.httpcore5.vespa.version> <apiguardian.vespa.version>1.1.2</apiguardian.vespa.version> <asm.vespa.version>9.6</asm.vespa.version> - <assertj.vespa.version>3.24.2</assertj.vespa.version> + <assertj.vespa.version>3.25.0</assertj.vespa.version> <!-- Athenz dependencies. Make sure these dependencies match those in Vespa's internal repositories --> <athenz.vespa.version>1.11.48</athenz.vespa.version> diff --git a/searchlib/src/tests/attribute/stringattribute/stringattribute_test.cpp b/searchlib/src/tests/attribute/stringattribute/stringattribute_test.cpp index 1beb2b1e501..1bfb9fb41f9 100644 --- a/searchlib/src/tests/attribute/stringattribute/stringattribute_test.cpp +++ b/searchlib/src/tests/attribute/stringattribute/stringattribute_test.cpp @@ -1,5 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/testapp.h> +#include <vespa/searchcommon/attribute/config.h> #include <vespa/searchlib/attribute/enumstore.h> #include <vespa/searchlib/attribute/singlestringattribute.h> #include <vespa/searchlib/attribute/singlestringpostattribute.h> @@ -8,7 +9,6 @@ #include <vespa/searchlib/attribute/enumstore.hpp> #include <vespa/searchlib/attribute/single_string_enum_search_context.h> -#include <vespa/searchlib/attribute/multistringpostattribute.hpp> #include <vespa/log/log.h> LOG_SETUP("stringattribute_test"); diff --git a/searchlib/src/tests/query/streaming_query_test.cpp b/searchlib/src/tests/query/streaming_query_test.cpp index 08705fa837b..0a751e96222 100644 --- a/searchlib/src/tests/query/streaming_query_test.cpp +++ b/searchlib/src/tests/query/streaming_query_test.cpp @@ -287,7 +287,10 @@ TEST(StreamingQueryTest, test_query_language) class AllowRewrite : public QueryNodeResultFactory { public: - virtual bool getRewriteFloatTerms() const override { return true; } + explicit AllowRewrite(vespalib::stringref index) noexcept : _allowedIndex(index) {} + bool getRewriteFloatTerms(vespalib::stringref index) const noexcept override { return index == _allowedIndex; } +private: + vespalib::string _allowedIndex; }; const char TERM_UNIQ = static_cast<char>(ParseItem::ITEM_TERM) | static_cast<char>(ParseItem::IF_UNIQUEID); @@ -297,12 +300,12 @@ TEST(StreamingQueryTest, e_is_not_rewritten_even_if_allowed) const char term[6] = {TERM_UNIQ, 3, 1, 'c', 1, 'e'}; vespalib::stringref stackDump(term, sizeof(term)); EXPECT_EQ(6u, stackDump.size()); - AllowRewrite allowRewrite; + AllowRewrite allowRewrite("c"); const Query q(allowRewrite, stackDump); EXPECT_TRUE(q.valid()); const QueryNode & root = q.getRoot(); EXPECT_TRUE(dynamic_cast<const QueryTerm *>(&root) != nullptr); - const QueryTerm & qt = static_cast<const QueryTerm &>(root); + const auto & qt = static_cast<const QueryTerm &>(root); EXPECT_EQ("c", qt.index()); EXPECT_EQ(vespalib::stringref("e"), qt.getTerm()); EXPECT_EQ(3u, qt.uniqueId()); @@ -313,12 +316,12 @@ TEST(StreamingQueryTest, onedot0e_is_not_rewritten_by_default) const char term[9] = {TERM_UNIQ, 3, 1, 'c', 4, '1', '.', '0', 'e'}; vespalib::stringref stackDump(term, sizeof(term)); EXPECT_EQ(9u, stackDump.size()); - QueryNodeResultFactory empty; + AllowRewrite empty("nix"); const Query q(empty, stackDump); EXPECT_TRUE(q.valid()); const QueryNode & root = q.getRoot(); EXPECT_TRUE(dynamic_cast<const QueryTerm *>(&root) != nullptr); - const QueryTerm & qt = static_cast<const QueryTerm &>(root); + const auto & qt = static_cast<const QueryTerm &>(root); EXPECT_EQ("c", qt.index()); EXPECT_EQ(vespalib::stringref("1.0e"), qt.getTerm()); EXPECT_EQ(3u, qt.uniqueId()); @@ -329,34 +332,34 @@ TEST(StreamingQueryTest, onedot0e_is_rewritten_if_allowed_too) const char term[9] = {TERM_UNIQ, 3, 1, 'c', 4, '1', '.', '0', 'e'}; vespalib::stringref stackDump(term, sizeof(term)); EXPECT_EQ(9u, stackDump.size()); - AllowRewrite empty; + AllowRewrite empty("c"); const Query q(empty, stackDump); EXPECT_TRUE(q.valid()); const QueryNode & root = q.getRoot(); EXPECT_TRUE(dynamic_cast<const EquivQueryNode *>(&root) != nullptr); - const EquivQueryNode & equiv = static_cast<const EquivQueryNode &>(root); + const auto & equiv = static_cast<const EquivQueryNode &>(root); EXPECT_EQ(2u, equiv.size()); EXPECT_TRUE(dynamic_cast<const QueryTerm *>(equiv[0].get()) != nullptr); { - const QueryTerm & qt = static_cast<const QueryTerm &>(*equiv[0]); + const auto & qt = static_cast<const QueryTerm &>(*equiv[0]); EXPECT_EQ("c", qt.index()); EXPECT_EQ(vespalib::stringref("1.0e"), qt.getTerm()); EXPECT_EQ(3u, qt.uniqueId()); } EXPECT_TRUE(dynamic_cast<const PhraseQueryNode *>(equiv[1].get()) != nullptr); { - const PhraseQueryNode & phrase = static_cast<const PhraseQueryNode &>(*equiv[1]); + const auto & phrase = static_cast<const PhraseQueryNode &>(*equiv[1]); EXPECT_EQ(2u, phrase.size()); EXPECT_TRUE(dynamic_cast<const QueryTerm *>(phrase[0].get()) != nullptr); { - const QueryTerm & qt = static_cast<const QueryTerm &>(*phrase[0]); + const auto & qt = static_cast<const QueryTerm &>(*phrase[0]); EXPECT_EQ("c", qt.index()); EXPECT_EQ(vespalib::stringref("1"), qt.getTerm()); EXPECT_EQ(0u, qt.uniqueId()); } EXPECT_TRUE(dynamic_cast<const QueryTerm *>(phrase[1].get()) != nullptr); { - const QueryTerm & qt = static_cast<const QueryTerm &>(*phrase[1]); + const auto & qt = static_cast<const QueryTerm &>(*phrase[1]); EXPECT_EQ("c", qt.index()); EXPECT_EQ(vespalib::stringref("0e"), qt.getTerm()); EXPECT_EQ(0u, qt.uniqueId()); @@ -460,7 +463,7 @@ TEST(StreamingQueryTest, test_phrase_evaluate) terms[1]->add(1, 5, 0, 1); terms[2]->add(0, 5, 0, 1); HitList hits; - PhraseQueryNode * p = static_cast<PhraseQueryNode *>(phrases[0]); + auto * p = static_cast<PhraseQueryNode *>(phrases[0]); p->evaluateHits(hits); ASSERT_EQ(3u, hits.size()); EXPECT_EQ(hits[0].wordpos(), 2u); @@ -750,7 +753,7 @@ TEST(StreamingQueryTest, require_that_incorrectly_specified_diversity_can_be_par TEST(StreamingQueryTest, require_that_we_do_not_break_the_stack_on_bad_query) { - QueryTermSimple term("<form><iframe+	 +src=\\\"javascript:alert(1)\\\" 	;>", TermType::WORD); + QueryTermSimple term(R"(<form><iframe+	 +src=\"javascript:alert(1)\" 	;>)", TermType::WORD); EXPECT_FALSE(term.isValid()); } @@ -759,7 +762,7 @@ TEST(StreamingQueryTest, a_unhandled_sameElement_stack) const char * stack = "\022\002\026xyz_abcdefghij_xyzxyzxQ\001\vxxxxxx_name\034xxxxxx_xxxx_xxxxxxx_xxxxxxxxE\002\005delta\b<0.00393"; vespalib::stringref stackDump(stack); EXPECT_EQ(85u, stackDump.size()); - AllowRewrite empty; + AllowRewrite empty(""); const Query q(empty, stackDump); EXPECT_TRUE(q.valid()); const QueryNode & root = q.getRoot(); @@ -793,7 +796,7 @@ TEST(StreamingQueryTest, test_same_element_evaluate) vespalib::string stackDump = StackDumpCreator::create(*node); QueryNodeResultFactory empty; Query q(empty, stackDump); - SameElementQueryNode * sameElem = dynamic_cast<SameElementQueryNode *>(&q.getRoot()); + auto * sameElem = dynamic_cast<SameElementQueryNode *>(&q.getRoot()); EXPECT_TRUE(sameElem != nullptr); EXPECT_EQ("field", sameElem->getIndex()); EXPECT_EQ(3u, sameElem->size()); diff --git a/searchlib/src/vespa/searchlib/query/query_term_simple.cpp b/searchlib/src/vespa/searchlib/query/query_term_simple.cpp index 8df7764c183..fb2be58c536 100644 --- a/searchlib/src/vespa/searchlib/query/query_term_simple.cpp +++ b/searchlib/src/vespa/searchlib/query/query_term_simple.cpp @@ -7,6 +7,7 @@ #include <vespa/vespalib/locale/c.h> #include <cmath> #include <limits> +#include <charconv> namespace { @@ -161,24 +162,35 @@ QueryTermSimple::getRange() const return getIntegerRange<int64_t>(); } -template <int B> struct IntDecoder { - static int64_t fromstr(const char * v, char ** end) { return strtoll(v, end, B); } - static int64_t nearestDownwd(int64_t n, int64_t min) { return (n > min ? n - 1 : n); } - static int64_t nearestUpward(int64_t n, int64_t max) { return (n < max ? n + 1 : n); } + static int64_t fromstr(const char * q, const char * qend, char ** end) noexcept { + int64_t v(0); + for (;q < qend && isspace(*q); q++); + std::from_chars_result err = std::from_chars(q, qend, v, 10); + if (err.ec == std::errc::result_out_of_range) { + v = (*q == '-') ? std::numeric_limits<int64_t>::min() : std::numeric_limits<int64_t>::max(); + } + *end = const_cast<char *>(err.ptr); + return v; + } + static int64_t nearestDownwd(int64_t n, int64_t min) noexcept { return (n > min ? n - 1 : n); } + static int64_t nearestUpward(int64_t n, int64_t max) noexcept { return (n < max ? n + 1 : n); } }; struct DoubleDecoder { - static double fromstr(const char * v, char ** end) { return vespalib::locale::c::strtod(v, end); } - static double nearestDownwd(double n, double min) { return std::nextafterf(n, min); } - static double nearestUpward(double n, double max) { return std::nextafterf(n, max); } + static double fromstr(const char * q, const char * qend, char ** end) { + (void) qend; + return vespalib::locale::c::strtod(q, end); + } + static double nearestDownwd(double n, double min) noexcept { return std::nextafterf(n, min); } + static double nearestUpward(double n, double max) noexcept { return std::nextafterf(n, max); } }; bool QueryTermSimple::getAsIntegerTerm(int64_t & lower, int64_t & upper) const { lower = std::numeric_limits<int64_t>::min(); upper = std::numeric_limits<int64_t>::max(); - return getAsNumericTerm(lower, upper, IntDecoder<10>()); + return getAsNumericTerm(lower, upper, IntDecoder()); } bool QueryTermSimple::getAsDoubleTerm(double & lower, double & upper) const @@ -266,11 +278,12 @@ QueryTermSimple::getAsNumericTerm(T & lower, T & upper, D d) const T low(lower); T high(upper); const char * q = _term.c_str(); + const char * qend = q + sz; const char first(q[0]); const char last(q[sz-1]); bool isRange = (first == '<') || (first == '>') || (first == '['); q += isRange ? 1 : 0; - T ll = d.fromstr(q, &err); + T ll = d.fromstr(q, qend, &err); bool valid = isValid() && ((*err == 0) || (*err == ';')); if (!valid) return false; @@ -289,7 +302,7 @@ QueryTermSimple::getAsNumericTerm(T & lower, T & upper, D d) const low = (first == '[') ? ll : d.nearestUpward(ll, upper); } q = err + 1; - T hh = d.fromstr(q, &err); + T hh = d.fromstr(q, qend, &err); bool hasUpperLimit(q != err); if (*err == ';') { err = const_cast<char *>(_term.end() - 1); diff --git a/searchlib/src/vespa/searchlib/query/streaming/query.cpp b/searchlib/src/vespa/searchlib/query/streaming/query.cpp index d2eee5d345f..3079ec31e8f 100644 --- a/searchlib/src/vespa/searchlib/query/streaming/query.cpp +++ b/searchlib/src/vespa/searchlib/query/streaming/query.cpp @@ -12,7 +12,7 @@ QueryConnector::visitMembers(vespalib::ObjectVisitor &visitor) const visit(visitor, "Operator", _opName); } -QueryConnector::QueryConnector(const char * opName) +QueryConnector::QueryConnector(const char * opName) noexcept : QueryNode(), _opName(opName), _index(), @@ -31,7 +31,7 @@ const HitList & QueryConnector::evaluateHits(HitList & hl) const { if (evaluate()) { - hl.push_back(Hit(1, 0, 0, 1)); + hl.emplace_back(1, 0, 0, 1); } return hl; } @@ -105,10 +105,10 @@ QueryConnector::create(ParseItem::ItemType type) { switch (type) { case search::ParseItem::ITEM_AND: return std::make_unique<AndQueryNode>(); - case search::ParseItem::ITEM_OR: return std::make_unique<OrQueryNode>(); + case search::ParseItem::ITEM_OR: case search::ParseItem::ITEM_WEAK_AND: return std::make_unique<OrQueryNode>(); + case search::ParseItem::ITEM_WEIGHTED_SET: case search::ParseItem::ITEM_EQUIV: return std::make_unique<EquivQueryNode>(); - case search::ParseItem::ITEM_WEIGHTED_SET: return std::make_unique<EquivQueryNode>(); case search::ParseItem::ITEM_WAND: return std::make_unique<OrQueryNode>(); case search::ParseItem::ITEM_NOT: return std::make_unique<AndNotQueryNode>(); case search::ParseItem::ITEM_PHRASE: return std::make_unique<PhraseQueryNode>(); @@ -340,7 +340,7 @@ Query::Query(const QueryNodeResultFactory & factory, vespalib::stringref queryRe bool Query::evaluate() const { - return valid() ? _root->evaluate() : false; + return valid() && _root->evaluate(); } bool diff --git a/searchlib/src/vespa/searchlib/query/streaming/query.h b/searchlib/src/vespa/searchlib/query/streaming/query.h index 42c3b94002c..3904f743d26 100644 --- a/searchlib/src/vespa/searchlib/query/streaming/query.h +++ b/searchlib/src/vespa/searchlib/query/streaming/query.h @@ -13,8 +13,8 @@ namespace search::streaming { class QueryConnector : public QueryNode { public: - QueryConnector(const char * opName); - ~QueryConnector(); + explicit QueryConnector(const char * opName) noexcept; + ~QueryConnector() override; const HitList & evaluateHits(HitList & hl) const override; void reset() override; void getLeaves(QueryTermList & tl) override; @@ -44,7 +44,7 @@ private: class TrueNode : public QueryConnector { public: - TrueNode() : QueryConnector("AND") { } + TrueNode() noexcept : QueryConnector("AND") { } bool evaluate() const override; }; @@ -52,7 +52,7 @@ public: class FalseNode : public QueryConnector { public: - FalseNode() : QueryConnector("AND") { } + FalseNode() noexcept : QueryConnector("AND") { } bool evaluate() const override; }; @@ -62,8 +62,8 @@ public: class AndQueryNode : public QueryConnector { public: - AndQueryNode() : QueryConnector("AND") { } - AndQueryNode(const char * opName) : QueryConnector(opName) { } + AndQueryNode() noexcept : QueryConnector("AND") { } + explicit AndQueryNode(const char * opName) noexcept : QueryConnector(opName) { } bool evaluate() const override; bool isFlattenable(ParseItem::ItemType type) const override { return type == ParseItem::ITEM_AND; } }; @@ -74,7 +74,7 @@ public: class AndNotQueryNode : public QueryConnector { public: - AndNotQueryNode() : QueryConnector("ANDNOT") { } + AndNotQueryNode() noexcept : QueryConnector("ANDNOT") { } bool evaluate() const override; bool isFlattenable(ParseItem::ItemType type) const override { return type == ParseItem::ITEM_NOT; } }; @@ -85,8 +85,8 @@ public: class OrQueryNode : public QueryConnector { public: - OrQueryNode() : QueryConnector("OR") { } - OrQueryNode(const char * opName) : QueryConnector(opName) { } + OrQueryNode() noexcept : QueryConnector("OR") { } + explicit OrQueryNode(const char * opName) noexcept : QueryConnector(opName) { } bool evaluate() const override; bool isFlattenable(ParseItem::ItemType type) const override { return (type == ParseItem::ITEM_OR) || @@ -102,7 +102,7 @@ public: class EquivQueryNode : public OrQueryNode { public: - EquivQueryNode() : OrQueryNode("EQUIV") { } + EquivQueryNode() noexcept : OrQueryNode("EQUIV") { } bool evaluate() const override; bool isFlattenable(ParseItem::ItemType type) const override { return (type == ParseItem::ITEM_EQUIV) || @@ -117,7 +117,7 @@ public: class PhraseQueryNode : public AndQueryNode { public: - PhraseQueryNode() : AndQueryNode("PHRASE"), _fieldInfo(32) { } + PhraseQueryNode() noexcept : AndQueryNode("PHRASE"), _fieldInfo(32) { } bool evaluate() const override; const HitList & evaluateHits(HitList & hl) const override; void getPhrases(QueryNodeRefList & tl) override; @@ -138,7 +138,7 @@ private: class SameElementQueryNode : public AndQueryNode { public: - SameElementQueryNode() : AndQueryNode("SAME_ELEMENT") { } + SameElementQueryNode() noexcept : AndQueryNode("SAME_ELEMENT") { } bool evaluate() const override; const HitList & evaluateHits(HitList & hl) const override; bool isFlattenable(ParseItem::ItemType type) const override { return type == ParseItem::ITEM_NOT; } @@ -151,8 +151,8 @@ public: class NearQueryNode : public AndQueryNode { public: - NearQueryNode() : AndQueryNode("NEAR"), _distance(0) { } - NearQueryNode(const char * opName) : AndQueryNode(opName), _distance(0) { } + NearQueryNode() noexcept : AndQueryNode("NEAR"), _distance(0) { } + explicit NearQueryNode(const char * opName) noexcept : AndQueryNode(opName), _distance(0) { } bool evaluate() const override; void distance(size_t dist) { _distance = dist; } size_t distance() const { return _distance; } @@ -169,8 +169,7 @@ private: class ONearQueryNode : public NearQueryNode { public: - ONearQueryNode() : NearQueryNode("ONEAR") { } - ~ONearQueryNode() { } + ONearQueryNode() noexcept : NearQueryNode("ONEAR") { } bool evaluate() const override; }; diff --git a/searchlib/src/vespa/searchlib/query/streaming/querynode.cpp b/searchlib/src/vespa/searchlib/query/streaming/querynode.cpp index db0fbd5b98e..6a17d95ebf2 100644 --- a/searchlib/src/vespa/searchlib/query/streaming/querynode.cpp +++ b/searchlib/src/vespa/searchlib/query/streaming/querynode.cpp @@ -13,12 +13,18 @@ LOG_SETUP(".vsm.querynode"); namespace search::streaming { namespace { - vespalib::stringref DEFAULT("default"); - bool disableRewrite(const QueryNode * qn) { - return dynamic_cast<const NearQueryNode *> (qn) || - dynamic_cast<const PhraseQueryNode *> (qn) || - dynamic_cast<const SameElementQueryNode *>(qn); - } + +vespalib::stringref DEFAULT("default"); +bool disableRewrite(const QueryNode * qn) { + return dynamic_cast<const NearQueryNode *> (qn) || + dynamic_cast<const PhraseQueryNode *> (qn) || + dynamic_cast<const SameElementQueryNode *>(qn); +} + +bool possibleFloat(const QueryTerm & qt, const QueryTerm::string & term) { + return !qt.encoding().isBase10Integer() && qt.encoding().isFloat() && (term.find('.') != QueryTerm::string::npos); +} + } QueryNode::UP @@ -43,8 +49,8 @@ QueryNode::Build(const QueryNode * parent, const QueryNodeResultFactory & factor { qn = QueryConnector::create(type); if (qn) { - QueryConnector * qc = dynamic_cast<QueryConnector *> (qn.get()); - NearQueryNode * nqn = dynamic_cast<NearQueryNode *> (qc); + auto * qc = dynamic_cast<QueryConnector *> (qn.get()); + auto * nqn = dynamic_cast<NearQueryNode *> (qc); if (nqn) { nqn->distance(queryRep.getNearDistance()); } @@ -150,21 +156,17 @@ QueryNode::Build(const QueryNode * parent, const QueryNodeResultFactory & factor qt->setFuzzyMaxEditDistance(queryRep.getFuzzyMaxEditDistance()); qt->setFuzzyPrefixLength(queryRep.getFuzzyPrefixLength()); } - if (qt->encoding().isBase10Integer() || - ! qt->encoding().isFloat() || - ! factory.getRewriteFloatTerms() || - ! allowRewrite || - (ssTerm.find('.') == vespalib::string::npos)) - { - qn = std::move(qt); - } else { + if (possibleFloat(*qt, ssTerm) && factory.getRewriteFloatTerms(ssIndex) && allowRewrite) { auto phrase = std::make_unique<PhraseQueryNode>(); - phrase->addChild(std::make_unique<QueryTerm>(factory.create(), ssTerm.substr(0, ssTerm.find('.')), ssIndex, TermType::WORD)); - phrase->addChild(std::make_unique<QueryTerm>(factory.create(), ssTerm.substr(ssTerm.find('.') + 1), ssIndex, TermType::WORD)); + auto dotPos = ssTerm.find('.'); + phrase->addChild(std::make_unique<QueryTerm>(factory.create(), ssTerm.substr(0, dotPos), ssIndex, TermType::WORD)); + phrase->addChild(std::make_unique<QueryTerm>(factory.create(), ssTerm.substr(dotPos + 1), ssIndex, TermType::WORD)); auto orqn = std::make_unique<EquivQueryNode>(); orqn->addChild(std::move(qt)); orqn->addChild(std::move(phrase)); qn = std::move(orqn); + } else { + qn = std::move(qt); } } } diff --git a/searchlib/src/vespa/searchlib/query/streaming/querynode.h b/searchlib/src/vespa/searchlib/query/streaming/querynode.h index 09c44d951d3..bfc840e4603 100644 --- a/searchlib/src/vespa/searchlib/query/streaming/querynode.h +++ b/searchlib/src/vespa/searchlib/query/streaming/querynode.h @@ -36,7 +36,7 @@ class QueryNode public: using UP = std::unique_ptr<QueryNode>; - virtual ~QueryNode() { } + virtual ~QueryNode() = default; /// This evalutes if the subtree starting here evaluates to true. virtual bool evaluate() const = 0; /// This return the hitList for this subtree. Does only give meaning in a diff --git a/searchlib/src/vespa/searchlib/query/streaming/querynoderesultbase.h b/searchlib/src/vespa/searchlib/query/streaming/querynoderesultbase.h index 62fc32a4575..d7704fb60e1 100644 --- a/searchlib/src/vespa/searchlib/query/streaming/querynoderesultbase.h +++ b/searchlib/src/vespa/searchlib/query/streaming/querynoderesultbase.h @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once +#include <vespa/vespalib/stllike/string.h> #include <memory> namespace search::streaming { @@ -20,8 +21,8 @@ public: class QueryNodeResultFactory { public: virtual ~QueryNodeResultFactory() = default; - virtual bool getRewriteFloatTerms() const { return false; } - virtual std::unique_ptr<QueryNodeResultBase> create() const { return std::unique_ptr<QueryNodeResultBase>(); } + virtual bool getRewriteFloatTerms(vespalib::stringref index) const noexcept { (void) index; return false; } + virtual std::unique_ptr<QueryNodeResultBase> create() const { return {}; } }; } diff --git a/searchlib/src/vespa/searchlib/query/streaming/queryterm.cpp b/searchlib/src/vespa/searchlib/query/streaming/queryterm.cpp index 9c45427d07d..a658ff5f3d6 100644 --- a/searchlib/src/vespa/searchlib/query/streaming/queryterm.cpp +++ b/searchlib/src/vespa/searchlib/query/streaming/queryterm.cpp @@ -15,6 +15,7 @@ private: }; CharInfo::CharInfo() + : _charInfo() { // XXX: Should refactor to reduce number of magic constants. memset(_charInfo, 0x01, 128); // All 7 bits are ascii7bit @@ -33,7 +34,7 @@ CharInfo::CharInfo() _charInfo[uint8_t('E')] = 0x05; } -static CharInfo _G_charTable; +CharInfo _G_charTable; } @@ -65,10 +66,10 @@ QueryTerm::QueryTerm(std::unique_ptr<QueryNodeResultBase> org, const string & te { if (!termS.empty()) { uint8_t enc(0xff); - for (size_t i(0), m(termS.size()); i < m; i++) { - enc &= _G_charTable.get(termS[i]); + for (char c : termS) { + enc &= _G_charTable.get(c); } - _encoding = enc; + _encoding = EncodingBitMap(enc); } } diff --git a/searchlib/src/vespa/searchlib/query/streaming/queryterm.h b/searchlib/src/vespa/searchlib/query/streaming/queryterm.h index 6e91437b1f9..2d1156a9c51 100644 --- a/searchlib/src/vespa/searchlib/query/streaming/queryterm.h +++ b/searchlib/src/vespa/searchlib/query/streaming/queryterm.h @@ -27,7 +27,7 @@ public: class EncodingBitMap { public: - EncodingBitMap(uint8_t bm=0) : _enc(bm) { } + explicit EncodingBitMap(uint8_t bm) : _enc(bm) { } bool isFloat() const { return _enc & Float; } bool isBase10Integer() const { return _enc & Base10Integer; } bool isAscii7Bit() const { return _enc & Ascii7Bit; } diff --git a/storage/src/vespa/storage/visiting/countvisitor.cpp b/storage/src/vespa/storage/visiting/countvisitor.cpp index b8b415402d6..fe1569f84da 100644 --- a/storage/src/vespa/storage/visiting/countvisitor.cpp +++ b/storage/src/vespa/storage/visiting/countvisitor.cpp @@ -26,10 +26,9 @@ CountVisitor::handleDocuments(const document::BucketId& /*bucketId*/, DocEntryList& entries, HitCounter& hitCounter) { - for (size_t i = 0; i < entries.size(); ++i) { - const spi::DocEntry& entry(*entries[i]); - if (!entry.isRemove()) { - const document::Document* doc = entry.getDocument(); + for (const auto & entry : entries) { + if (!entry->isRemove()) { + const document::Document* doc = entry->getDocument(); if (doc) { const document::IdString& idString = doc->getId().getScheme(); @@ -57,33 +56,25 @@ CountVisitor::handleDocuments(const document::BucketId& /*bucketId*/, } void CountVisitor::completedVisiting(HitCounter&) { - documentapi::MapVisitorMessage* cmd(new documentapi::MapVisitorMessage()); + auto cmd = std::make_unique<documentapi::MapVisitorMessage>(); - for (std::map<std::string, int>::iterator iter = _schemeCount.begin(); - iter != _schemeCount.end(); - iter++) { - cmd->getData().set(vespalib::make_string("scheme.%s", iter->first.c_str()), iter->second); + for (const auto & count : _schemeCount) { + cmd->getData().set(vespalib::make_string("scheme.%s", count.first.c_str()), count.second); } - for (NamespaceCountMap::const_iterator iter = _namespaceCount.begin(); - iter != _namespaceCount.end(); - iter++) { - cmd->getData().set(vespalib::make_string("namespace.%s", iter->first.c_str()), iter->second); + for (const auto & count : _namespaceCount) { + cmd->getData().set(vespalib::make_string("namespace.%s", count.first.c_str()), count.second); } - for (GroupCountMap::const_iterator iter = _groupCount.begin(); - iter != _groupCount.end(); - iter++) { - cmd->getData().set(vespalib::make_string("group.%s", iter->first.c_str()), iter->second); + for (const auto & count : _groupCount) { + cmd->getData().set(vespalib::make_string("group.%s", count.first.c_str()), count.second); } - for (std::map<uint64_t, int>::iterator iter = _userCount.begin(); - iter != _userCount.end(); - iter++) { - cmd->getData().set(vespalib::make_string("user.%" PRIu64, iter->first), iter->second); + for (const auto & count : _userCount) { + cmd->getData().set(vespalib::make_string("user.%" PRIu64, count.first), count.second); } - sendMessage(documentapi::DocumentMessage::UP(cmd)); + sendMessage(std::move(cmd)); } } diff --git a/storage/src/vespa/storageapi/mbusprot/protocolserialization7.cpp b/storage/src/vespa/storageapi/mbusprot/protocolserialization7.cpp index efbe8c9b42d..57047be6037 100644 --- a/storage/src/vespa/storageapi/mbusprot/protocolserialization7.cpp +++ b/storage/src/vespa/storageapi/mbusprot/protocolserialization7.cpp @@ -56,8 +56,8 @@ void set_bucket_info(protobuf::BucketInfo& dest, const api::BucketInfo& src) { } document::Bucket get_bucket(const protobuf::Bucket& src) { - return document::Bucket(document::BucketSpace(src.space_id()), - document::BucketId(src.raw_bucket_id())); + return {document::BucketSpace(src.space_id()), + document::BucketId(src.raw_bucket_id())}; } api::BucketInfo get_bucket_info(const protobuf::BucketInfo& src) { @@ -953,11 +953,11 @@ void fill_api_apply_diff_vector(std::vector<api::ApplyBucketDiffCommand::Entry>& dest._docName = proto_entry.document_id(); // TODO consider making buffers std::strings instead to avoid explicit zeroing-on-resize overhead dest._headerBlob.resize(proto_entry.header_blob().size()); - if (proto_entry.header_blob().size() > 0) { + if (!proto_entry.header_blob().empty()) { memcpy(dest._headerBlob.data(), proto_entry.header_blob().data(), proto_entry.header_blob().size()); } dest._bodyBlob.resize(proto_entry.body_blob().size()); - if (proto_entry.body_blob().size() > 0) { + if (!proto_entry.body_blob().empty()) { memcpy(dest._bodyBlob.data(), proto_entry.body_blob().data(), proto_entry.body_blob().size()); } } diff --git a/storage/src/vespa/storageapi/message/datagram.cpp b/storage/src/vespa/storageapi/message/datagram.cpp index d2ced1d4b7b..103b7ead08c 100644 --- a/storage/src/vespa/storageapi/message/datagram.cpp +++ b/storage/src/vespa/storageapi/message/datagram.cpp @@ -5,8 +5,7 @@ using document::BucketSpace; -namespace storage { -namespace api { +namespace storage::api { IMPLEMENT_COMMAND(MapVisitorCommand, MapVisitorReply) IMPLEMENT_REPLY(MapVisitorReply) @@ -24,11 +23,9 @@ MapVisitorCommand::print(std::ostream& out, bool verbose, { out << "MapVisitor(" << _statistics.size() << " entries"; if (verbose) { - for (vdslib::Parameters::ParametersMap::const_iterator it - = _statistics.begin(); it != _statistics.end(); ++it) - { - out << ",\n" << indent << " " << it->first << ": " - << vespalib::stringref(it->second.c_str(), it->second.length()); + for (const auto & stat : _statistics) { + out << ",\n" << indent << " " << stat.first << ": " + << vespalib::stringref(stat.second.c_str(), stat.second.length()); } out << ") : "; StorageCommand::print(out, verbose, indent); @@ -66,9 +63,9 @@ EmptyBucketsCommand::print(std::ostream& out, bool verbose, { out << "EmptyBuckets("; if (verbose) { - for (uint32_t i=0; i<_buckets.size(); ++i) { + for (const auto & bucket : _buckets) { out << "\n" << indent << " "; - out << _buckets[i]; + out << bucket; } } else { out << _buckets.size() << " buckets"; @@ -96,5 +93,4 @@ EmptyBucketsReply::print(std::ostream& out, bool verbose, } } -} // api -} // storage +} diff --git a/storage/src/vespa/storageapi/message/visitor.h b/storage/src/vespa/storageapi/message/visitor.h index fddb7604eff..979b8064bd8 100644 --- a/storage/src/vespa/storageapi/message/visitor.h +++ b/storage/src/vespa/storageapi/message/visitor.h @@ -58,7 +58,7 @@ public: /** Create another command with similar visitor settings. */ CreateVisitorCommand(const CreateVisitorCommand& template_); - ~CreateVisitorCommand(); + ~CreateVisitorCommand() override; void setVisitorCmdId(uint32_t id) { _visitorCmdId = id; } void setControlDestination(vespalib::stringref d) { _controlDestination = d; } @@ -211,7 +211,7 @@ public: void setErrorCode(ReturnCode && code) { _error = std::move(code); } void setCompleted() { _completed = true; } void setBucketCompleted(const document::BucketId& id, Timestamp lastVisited) { - _bucketsCompleted.push_back(BucketTimestampPair(id, lastVisited)); + _bucketsCompleted.emplace_back(id, lastVisited); } void setBucketsCompleted(const std::vector<BucketTimestampPair>& bc) { _bucketsCompleted = bc; @@ -234,7 +234,7 @@ class VisitorInfoReply : public StorageReply { bool _completed; public: - VisitorInfoReply(const VisitorInfoCommand& cmd); + explicit VisitorInfoReply(const VisitorInfoCommand& cmd); bool visitorCompleted() const { return _completed; } void print(std::ostream& out, bool verbose, const std::string& indent) const override; diff --git a/streamingvisitors/src/tests/rank_processor/rank_processor_test.cpp b/streamingvisitors/src/tests/rank_processor/rank_processor_test.cpp index 2d138d1d336..93e35e4c6d2 100644 --- a/streamingvisitors/src/tests/rank_processor/rank_processor_test.cpp +++ b/streamingvisitors/src/tests/rank_processor/rank_processor_test.cpp @@ -40,7 +40,7 @@ protected: RankProcessorTest::RankProcessorTest() : testing::Test(), - _factory(), + _factory(nullptr), _query(), _query_wrapper() { diff --git a/streamingvisitors/src/vespa/searchvisitor/querytermdata.h b/streamingvisitors/src/vespa/searchvisitor/querytermdata.h index 8c1c3771917..36176f70d1d 100644 --- a/streamingvisitors/src/vespa/searchvisitor/querytermdata.h +++ b/streamingvisitors/src/vespa/searchvisitor/querytermdata.h @@ -17,15 +17,26 @@ private: search::fef::SimpleTermData _termData; public: QueryTermData * clone() const override { return new QueryTermData(); } - search::fef::SimpleTermData &getTermData() { return _termData; } + search::fef::SimpleTermData &getTermData() noexcept { return _termData; } +}; + +class SearchMethodInfo { +public: + virtual ~SearchMethodInfo() = default; + virtual bool is_text_matching(vespalib::stringref index) const noexcept = 0; }; class QueryTermDataFactory final : public search::streaming::QueryNodeResultFactory { public: + QueryTermDataFactory(const SearchMethodInfo * searchMethodInfo) noexcept : _searchMethodInfo(searchMethodInfo) {} std::unique_ptr<search::streaming::QueryNodeResultBase> create() const override { return std::make_unique<QueryTermData>(); } - bool getRewriteFloatTerms() const override { return true; } + bool getRewriteFloatTerms(vespalib::stringref index ) const noexcept override { + return _searchMethodInfo && _searchMethodInfo->is_text_matching(index); + } +private: + const SearchMethodInfo * _searchMethodInfo; }; diff --git a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp index 4d31c71c0a0..bd22ba65816 100644 --- a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp +++ b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp @@ -238,14 +238,16 @@ SearchVisitor::SummaryGenerator::fillSummary(AttributeVector::DocId lid, const H return {}; } -void SearchVisitor::HitsResultPreparator::execute(vespalib::Identifiable & obj) +void +SearchVisitor::HitsResultPreparator::execute(vespalib::Identifiable & obj) { auto & hitsAggr(static_cast<HitsAggregationResult &>(obj)); hitsAggr.setSummaryGenerator(_summaryGenerator); _numHitsAggregators++; } -bool SearchVisitor::HitsResultPreparator::check(const vespalib::Identifiable & obj) const +bool +SearchVisitor::HitsResultPreparator::check(const vespalib::Identifiable & obj) const { return obj.getClass().inherits(HitsAggregationResult::classId); } @@ -259,7 +261,8 @@ SearchVisitor::GroupingEntry::GroupingEntry(Grouping * grouping) : SearchVisitor::GroupingEntry::~GroupingEntry() = default; -void SearchVisitor::GroupingEntry::aggregate(const document::Document & doc, search::HitRank rank) +void +SearchVisitor::GroupingEntry::aggregate(const document::Document & doc, search::HitRank rank) { if (_count < _limit) { _grouping->aggregate(doc, rank); @@ -310,7 +313,15 @@ SearchVisitor::SearchVisitor(StorageComponent& component, LOG(debug, "Created SearchVisitor"); } -void SearchVisitor::init(const Parameters & params) +bool +SearchVisitor::is_text_matching(vespalib::stringref index) const noexcept { + vsm::FieldIdT fId = _fieldSearchSpecMap.nameIdMap().fieldNo(index); + auto found = _fieldSearchSpecMap.specMap().find(fId); + return (found != _fieldSearchSpecMap.specMap().end()) && found->second.uses_string_search_method(); +} + +void +SearchVisitor::init(const Parameters & params) { VISITOR_TRACE(6, "About to lazily init VSM adapter"); _attrMan.add(_documentIdAttributeBacking); @@ -397,7 +408,12 @@ void SearchVisitor::init(const Parameters & params) if ( params.lookup("query", queryBlob) ) { LOG(spam, "Received query blob of %zu bytes", queryBlob.size()); VISITOR_TRACE(9, vespalib::make_string("Setting up for query blob of %zu bytes", queryBlob.size())); - QueryTermDataFactory addOnFactory; + + // Create mapping from field name to field id, from field id to search spec, + // and from index name to list of field ids + _fieldSearchSpecMap.buildFromConfig(_env->get_vsm_fields_config()); + + QueryTermDataFactory addOnFactory(this); _query = Query(addOnFactory, vespalib::stringref(queryBlob.data(), queryBlob.size())); _searchBuffer->reserve(0x10000); @@ -414,7 +430,6 @@ void SearchVisitor::init(const Parameters & params) StringFieldIdTMap fieldsInQuery; setupFieldSearchers(additionalFields, fieldsInQuery); - setupScratchDocument(fieldsInQuery); _syntheticFieldsController.setup(_fieldSearchSpecMap.nameIdMap(), fieldsInQuery); @@ -754,9 +769,6 @@ void SearchVisitor::setupFieldSearchers(const std::vector<vespalib::string> & additionalFields, StringFieldIdTMap & fieldsInQuery) { - // Create mapping from field name to field id, from field id to search spec, - // and from index name to list of field ids - _fieldSearchSpecMap.buildFromConfig(_env->get_vsm_fields_config()); // Add extra elements to mapping from field name to field id _fieldSearchSpecMap.buildFromConfig(additionalFields); @@ -1145,7 +1157,8 @@ SearchVisitor::fillSortBuffer() return pos; } -void SearchVisitor::completedBucket(const document::BucketId&, HitCounter&) +void +SearchVisitor::completedBucket(const document::BucketId&, HitCounter&) { LOG(debug, "Completed bucket"); } @@ -1157,7 +1170,8 @@ SearchVisitor::generate_query_result(HitCounter& counter) return std::move(_queryResult); } -void SearchVisitor::completedVisitingInternal(HitCounter& hitCounter) +void +SearchVisitor::completedVisitingInternal(HitCounter& hitCounter) { if (!_init_called) { init(_params); diff --git a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h index ef7a41f23a5..76b2016e2e2 100644 --- a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h +++ b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h @@ -8,6 +8,7 @@ #include "rankmanager.h" #include "rankprocessor.h" #include "searchenvironment.h" +#include "querytermdata.h" #include <vespa/vsm/common/docsum.h> #include <vespa/vsm/common/documenttypemapping.h> #include <vespa/vsm/common/storagedocument.h> @@ -42,7 +43,8 @@ class SearchEnvironmentSnapshot; * @brief Visitor that applies a search query to visitor data and * converts them to a QueryResultCommand. **/ -class SearchVisitor : public storage::Visitor { +class SearchVisitor : public storage::Visitor, + public SearchMethodInfo { public: SearchVisitor(storage::StorageComponent&, storage::VisitorEnvironment& vEnv, const vdslib::Parameters & params); @@ -488,6 +490,7 @@ private: vsm::StringFieldIdTMapT _fieldsUnion; void setupAttributeVector(const vsm::FieldPath &fieldPath); + bool is_text_matching(vespalib::stringref index) const noexcept override; }; class SearchVisitorFactory : public storage::VisitorFactory { diff --git a/streamingvisitors/src/vespa/vsm/searcher/floatfieldsearcher.cpp b/streamingvisitors/src/vespa/vsm/searcher/floatfieldsearcher.cpp index 7dd40348f47..8558522003f 100644 --- a/streamingvisitors/src/vespa/vsm/searcher/floatfieldsearcher.cpp +++ b/streamingvisitors/src/vespa/vsm/searcher/floatfieldsearcher.cpp @@ -37,7 +37,7 @@ void FloatFieldSearcherT<T>::prepare(search::streaming::QueryTermList& qtl, _floatTerm.clear(); FieldSearcher::prepare(qtl, buf, field_paths, query_env); for (auto qt : qtl) { - size_t sz(qt->termLen()); + size_t sz(qt->termLen()); if (sz) { auto range = qt->getRange<T>(); _floatTerm.emplace_back(range.low, range.high, range.valid); diff --git a/streamingvisitors/src/vespa/vsm/vsm/fieldsearchspec.h b/streamingvisitors/src/vespa/vsm/vsm/fieldsearchspec.h index b0154a82dae..f7ca07b4dc5 100644 --- a/streamingvisitors/src/vespa/vsm/vsm/fieldsearchspec.h +++ b/streamingvisitors/src/vespa/vsm/vsm/fieldsearchspec.h @@ -23,6 +23,11 @@ public: bool valid() const { return static_cast<bool>(_searcher); } size_t maxLength() const { return _maxLength; } bool uses_nearest_neighbor_search_method() const noexcept { return _searchMethod == VsmfieldsConfig::Fieldspec::Searchmethod::NEAREST_NEIGHBOR; } + bool uses_string_search_method() const noexcept { + return (_searchMethod == VsmfieldsConfig::Fieldspec::Searchmethod::UTF8) || + (_searchMethod == VsmfieldsConfig::Fieldspec::Searchmethod::AUTOUTF8) || + (_searchMethod == VsmfieldsConfig::Fieldspec::Searchmethod::SSE2UTF8); + } const vespalib::string& get_arg1() const noexcept { return _arg1; } /** diff --git a/vdslib/src/vespa/vdslib/container/parameters.cpp b/vdslib/src/vespa/vdslib/container/parameters.cpp index 298f4f6c0d8..236b4970396 100644 --- a/vdslib/src/vespa/vdslib/container/parameters.cpp +++ b/vdslib/src/vespa/vdslib/container/parameters.cpp @@ -7,6 +7,7 @@ #include <vespa/vespalib/util/xmlstream.h> #include <vespa/vespalib/util/growablebytebuffer.h> #include <ostream> +#include <charconv> using namespace vdslib; @@ -137,13 +138,35 @@ vespalib::string Parameters::toString() const return ret; } -template void vdslib::Parameters::set(vespalib::stringref , int32_t); -template void vdslib::Parameters::set(vespalib::stringref , int64_t); -template void vdslib::Parameters::set(vespalib::stringref , uint64_t); -template void vdslib::Parameters::set(vespalib::stringref , double); -template void vdslib::Parameters::set(vespalib::stringref , const char *); -template void vdslib::Parameters::set(vespalib::stringref , vespalib::string); -template void vdslib::Parameters::set(vespalib::stringref , std::string); +void +Parameters::set(KeyT id, int32_t value) { + char tmp[16]; + auto res = std::to_chars(tmp, tmp + sizeof(tmp), value, 10); + _parameters[id] = Value(tmp, size_t(res.ptr - tmp)); +} + +void +Parameters::set(KeyT id, int64_t value) { + char tmp[32]; + auto res = std::to_chars(tmp, tmp + sizeof(tmp), value, 10); + _parameters[id] = Value(tmp, size_t(res.ptr - tmp)); +} + +void +Parameters::set(KeyT id, uint64_t value) { + char tmp[32]; + auto res = std::to_chars(tmp, tmp + sizeof(tmp), value, 10); + _parameters[id] = Value(tmp, size_t(res.ptr - tmp)); +} + +void +Parameters::set(KeyT id, double value) { + vespalib::asciistream ost; + ost << value; + _parameters[id] = Value(ost.str()); +} + + template int32_t vdslib::Parameters::get(vespalib::stringref , int32_t) const; template int64_t vdslib::Parameters::get(vespalib::stringref , int64_t) const; template uint64_t vdslib::Parameters::get(vespalib::stringref , uint64_t) const; diff --git a/vdslib/src/vespa/vdslib/container/parameters.h b/vdslib/src/vespa/vdslib/container/parameters.h index 854aec4be20..d28e2cd9890 100644 --- a/vdslib/src/vespa/vdslib/container/parameters.h +++ b/vdslib/src/vespa/vdslib/container/parameters.h @@ -28,11 +28,11 @@ public: class Value : public vespalib::string { public: - Value() { } - Value(vespalib::stringref s) : vespalib::string(s) { } - Value(const vespalib::string & s) : vespalib::string(s) { } - Value(const void *v, size_t sz) : vespalib::string(v, sz) { } - size_t length() const { return size() - 1; } + Value() = default; + explicit Value(vespalib::stringref s) noexcept : vespalib::string(s) { } + explicit Value(const vespalib::string & s) noexcept : vespalib::string(s) { } + Value(const void *v, size_t sz) noexcept : vespalib::string(v, sz) { } + size_t length() const noexcept { return size() - 1; } }; using ValueRef = vespalib::stringref; using ParametersMap = vespalib::hash_map<vespalib::string, Value>; @@ -43,8 +43,8 @@ private: public: Parameters(); - Parameters(document::ByteBuffer& buffer); - ~Parameters(); + explicit Parameters(document::ByteBuffer& buffer); + ~Parameters() override; bool operator==(const Parameters &other) const; @@ -53,7 +53,9 @@ public: bool hasValue(KeyT id) const { return (_parameters.find(id) != _parameters.end()); } unsigned int size() const { return _parameters.size(); } bool lookup(KeyT id, ValueRef & v) const; - void set(KeyT id, const void * v, size_t sz) { _parameters[id] = Value(v, sz); } + void set(KeyT id, const void * v, size_t sz) { + _parameters[id] = Value(v, sz); + } void print(std::ostream& out, bool verbose, const std::string& indent) const; void serialize(vespalib::GrowableByteBuffer& buffer) const; @@ -63,17 +65,15 @@ public: ParametersMap::const_iterator begin() const { return _parameters.begin(); } ParametersMap::const_iterator end() const { return _parameters.end(); } /// Convenience from earlier use. - void set(KeyT id, vespalib::stringref value) { _parameters[id] = Value(value.data(), value.size()); } + void set(KeyT id, vespalib::stringref value) { + _parameters[id] = Value(value.data(), value.size()); + } + void set(KeyT id, int32_t value); + void set(KeyT id, int64_t value); + void set(KeyT id, uint64_t value); + void set(KeyT id, double value); vespalib::stringref get(KeyT id, vespalib::stringref def = "") const; - /** - * Set the value identified by the id given. This requires the type to be - * convertible by stringstreams. - * - * @param id The value to get. - * @param t The value to save. Will be converted to a string. - */ - template<typename T> - void set(KeyT id, T t); + /** * Get the value identified by the id given, as the same type as the default diff --git a/vdslib/src/vespa/vdslib/container/parameters.hpp b/vdslib/src/vespa/vdslib/container/parameters.hpp index ae5706046d4..d91612bc094 100644 --- a/vdslib/src/vespa/vdslib/container/parameters.hpp +++ b/vdslib/src/vespa/vdslib/container/parameters.hpp @@ -7,14 +7,6 @@ namespace vdslib { template<typename T> -void -Parameters::set(KeyT id, T t) { - vespalib::asciistream ost; - ost << t; - _parameters[id] = ost.str(); -} - -template<typename T> T Parameters::get(KeyT id, T def) const { vespalib::stringref ref; diff --git a/vespalib/src/vespa/vespalib/locale/locale.h b/vespalib/src/vespa/vespalib/locale/locale.h index 10ac6144fe7..16aa7a77d98 100644 --- a/vespalib/src/vespa/vespalib/locale/locale.h +++ b/vespalib/src/vespa/vespalib/locale/locale.h @@ -14,7 +14,7 @@ public: Locale(); // Standard C locale, NOT default locale. Locale(int category, const char *locale); ~Locale(); - locale_t get() const { return _locale; } + locale_t get() const noexcept { return _locale; } private: locale_t _locale; }; |