diff options
21 files changed, 183 insertions, 255 deletions
diff --git a/client/js/app/yarn.lock b/client/js/app/yarn.lock index c06265edaa1..17707aef2c8 100644 --- a/client/js/app/yarn.lock +++ b/client/js/app/yarn.lock @@ -1306,10 +1306,10 @@ dependencies: "@babel/runtime" "^7.13.10" -"@remix-run/router@1.14.0": - version "1.14.0" - resolved "https://registry.yarnpkg.com/@remix-run/router/-/router-1.14.0.tgz#9bc39a5a3a71b81bdb310eba6def5bc3966695b7" - integrity sha512-WOHih+ClN7N8oHk9N4JUiMxQJmRVaOxcg8w7F/oHUXzJt920ekASLI/7cYX8XkntDWRhLZtsk6LbGrkgOAvi5A== +"@remix-run/router@1.14.1": + version "1.14.1" + resolved "https://registry.yarnpkg.com/@remix-run/router/-/router-1.14.1.tgz#6d2dd03d52e604279c38911afc1079d58c50a755" + integrity sha512-Qg4DMQsfPNAs88rb2xkdk03N3bjK4jgX5fR24eHCTR9q6PrhZQZ4UJBPzCHJkIpTRN1UKxx2DzjZmnC+7Lj0Ow== "@rollup/rollup-android-arm-eabi@4.9.0": version "4.9.0" @@ -2564,9 +2564,9 @@ eslint-plugin-import@^2: tsconfig-paths "^3.15.0" eslint-plugin-prettier@^5: - version "5.1.0" - resolved "https://registry.yarnpkg.com/eslint-plugin-prettier/-/eslint-plugin-prettier-5.1.0.tgz#f14bb2b18756ad54f1ad3dc4c989cb73dfa326a3" - integrity sha512-hQc+2zbnMeXcIkg+pKZtVa+3Yqx4WY7SMkn1PLZ4VbBEU7jJIpVn9347P8BBhTbz6ne85aXvQf30kvexcqBeWw== + version "5.1.1" + resolved "https://registry.yarnpkg.com/eslint-plugin-prettier/-/eslint-plugin-prettier-5.1.1.tgz#ab7d9823788b557ff7ccdd50a5849d7760cb8bef" + integrity sha512-WQpV3mSmIobb77s4qiCZu3dBrZZ0rj8ckSfBtRrgNK9Wnh2s3eiaxNTWloz1LJ1WtvqZES/PAI7PLvsrGt/CEA== dependencies: prettier-linter-helpers "^1.0.0" synckit "^0.8.5" @@ -4466,9 +4466,9 @@ npm-run-path@^4.0.0, npm-run-path@^4.0.1: path-key "^3.0.0" npm-run-path@^5.1.0: - version "5.1.0" - resolved "https://registry.yarnpkg.com/npm-run-path/-/npm-run-path-5.1.0.tgz#bc62f7f3f6952d9894bd08944ba011a6ee7b7e00" - integrity sha512-sJOdmRGrY2sjNTRMbSvluQqg+8X7ZK61yvzBEIDhz4f8z1TZFYABsqjjCBd/0PUNE9M6QDgHJXQkGUEm7Q+l9Q== + version "5.2.0" + resolved "https://registry.yarnpkg.com/npm-run-path/-/npm-run-path-5.2.0.tgz#224cdd22c755560253dd71b83a1ef2f758b2e955" + integrity sha512-W4/tgAXFqFA0iL7fk0+uQ3g7wkL8xJmx3XdK0VGb4cHW//eZTtKGvFBBoRKVTpY7n6ze4NL9ly7rgXcHufqXKg== dependencies: path-key "^4.0.0" @@ -4850,19 +4850,19 @@ react-refresh@^0.14.0: integrity sha512-wViHqhAd8OHeLS/IRMJjTSDHF3U9eWi62F/MledQGPdJGDhodXJ9PBLNGr6WWL7qlH12Mt3TyTpbS+hGXMjCzQ== react-router-dom@^6: - version "6.21.0" - resolved "https://registry.yarnpkg.com/react-router-dom/-/react-router-dom-6.21.0.tgz#aa4c6bc046a8e8723095bc09b3c0ab2254532712" - integrity sha512-1dUdVj3cwc1npzJaf23gulB562ESNvxf7E4x8upNJycqyUm5BRRZ6dd3LrlzhtLaMrwOCO8R0zoiYxdaJx4LlQ== + version "6.21.1" + resolved "https://registry.yarnpkg.com/react-router-dom/-/react-router-dom-6.21.1.tgz#58b459d2fe1841388c95bb068f85128c45e27349" + integrity sha512-QCNrtjtDPwHDO+AO21MJd7yIcr41UetYt5jzaB9Y1UYaPTCnVuJq6S748g1dE11OQlCFIQg+RtAA1SEZIyiBeA== dependencies: - "@remix-run/router" "1.14.0" - react-router "6.21.0" + "@remix-run/router" "1.14.1" + react-router "6.21.1" -react-router@6.21.0: - version "6.21.0" - resolved "https://registry.yarnpkg.com/react-router/-/react-router-6.21.0.tgz#6fe3e59877aca3dccceec1801d26991ddf42d12b" - integrity sha512-hGZ0HXbwz3zw52pLZV3j3+ec+m/PQ9cTpBvqjFQmy2XVUWGn5MD+31oXHb6dVTxYzmAeaiUBYjkoNz66n3RGCg== +react-router@6.21.1: + version "6.21.1" + resolved "https://registry.yarnpkg.com/react-router/-/react-router-6.21.1.tgz#8db7ee8d7cfc36513c9a66b44e0897208c33be34" + integrity sha512-W0l13YlMTm1YrpVIOpjCADJqEUpz1vm+CMo47RuFX4Ftegwm6KOYsL5G3eiE52jnJpKvzm6uB/vTKTPKM8dmkA== dependencies: - "@remix-run/router" "1.14.0" + "@remix-run/router" "1.14.1" react-textarea-autosize@8.3.4: version "8.3.4" diff --git a/config-model-api/abi-spec.json b/config-model-api/abi-spec.json index 10c5662678e..c3e20e534ff 100644 --- a/config-model-api/abi-spec.json +++ b/config-model-api/abi-spec.json @@ -771,9 +771,7 @@ "attributes" : [ "public" ], - "methods" : [ - "public java.util.Map messagesById()" - ], + "methods" : [ ], "fields" : [ ] }, "com.yahoo.config.application.api.ValidationOverrides" : { diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/ValidationOverrides.java b/config-model-api/src/main/java/com/yahoo/config/application/api/ValidationOverrides.java index 7b52d825473..1edddb63e52 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/ValidationOverrides.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/ValidationOverrides.java @@ -15,7 +15,6 @@ import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -37,7 +36,7 @@ public class ValidationOverrides { private final String xmlForm; - /** Creates a validation overrides which does not have an XML form */ + /** Creates a validation overrides which does not have an xml form */ public ValidationOverrides(List<Allow> overrides) { this(overrides, null); } @@ -164,13 +163,10 @@ public class ValidationOverrides { */ public static class ValidationException extends IllegalArgumentException { - private final Map<ValidationId, Collection<String>> messagesById = new LinkedHashMap<>(); - static final long serialVersionUID = 789984668; private ValidationException(ValidationId validationId, String message) { super(validationId + ": " + message + ". " + toAllowMessage(validationId)); - messagesById.put(validationId, List.of(message)); } private ValidationException(Map<ValidationId, Collection<String>> messagesById) { @@ -179,11 +175,8 @@ public class ValidationOverrides { String.join("\n\t", messages.getValue()) + "\n" + toAllowMessage(messages.getKey())) .collect(Collectors.joining("\n"))); - messagesById.forEach((id, messages) -> this.messagesById.put(id, List.copyOf(messages))); } - public Map<ValidationId, Collection<String>> messagesById() { return Map.copyOf(messagesById); } - } } diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java index f19341098f4..33dfee58d1a 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java @@ -172,7 +172,7 @@ public class DeployState implements ConfigDefinitionStore { /** Get the global rank profile registry for this application. */ public final RankProfileRegistry rankProfileRegistry() { return rankProfileRegistry; } - /** Returns the validation overrides of this. This is never null. */ + /** Returns the validation overrides of this. This is never null */ public ValidationOverrides validationOverrides() { return validationOverrides; } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java index 65572d07cc2..903f1c06024 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java @@ -217,6 +217,13 @@ public class VespaModelFactory implements ModelFactory { private List<ConfigChangeAction> validateModel(VespaModel model, DeployState deployState, ValidationParameters validationParameters) { try { return new Validation(additionalValidators).validate(model, validationParameters, deployState); + } catch (ValidationOverrides.ValidationException e) { + if (deployState.isHosted() && zone.environment().isManuallyDeployed()) + deployState.getDeployLogger().logApplicationPackage(Level.WARNING, + "Auto-overriding validation which would be disallowed in production: " + + Exceptions.toMessageString(e)); + else + rethrowUnlessIgnoreErrors(e, validationParameters.ignoreValidationErrors()); } catch (IllegalArgumentException | TransientException | QuotaExceededException e) { rethrowUnlessIgnoreErrors(e, validationParameters.ignoreValidationErrors()); } catch (Exception e) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java index b9995d290cc..56277345515 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.model.application.validation; import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.application.api.ValidationOverrides; -import com.yahoo.config.application.api.ValidationOverrides.ValidationException; import com.yahoo.config.model.api.ConfigChangeAction; import com.yahoo.config.model.api.Model; import com.yahoo.config.model.api.ValidationParameters; @@ -26,19 +25,15 @@ import com.yahoo.vespa.model.application.validation.change.RestartOnDeployForOnn import com.yahoo.vespa.model.application.validation.change.StartupCommandChangeValidator; import com.yahoo.vespa.model.application.validation.change.StreamingSearchClusterChangeValidator; import com.yahoo.vespa.model.application.validation.first.RedundancyValidator; -import com.yahoo.yolean.Exceptions; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; -import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.logging.Level; import java.util.stream.Collectors; import static java.util.stream.Collectors.groupingBy; @@ -56,7 +51,7 @@ public class Validation { public Validation() { this(List.of()); } - /** Create instance taking additional validators (e.g., for cloud applications) */ + /** Create instance taking additional validators (e.g for cloud applications) */ public Validation(List<Validator> additionalValidators) { this.additionalValidators = additionalValidators; } /** @@ -67,53 +62,52 @@ public class Validation { * @throws ValidationOverrides.ValidationException if the change fails validation */ public List<ConfigChangeAction> validate(VespaModel model, ValidationParameters validationParameters, DeployState deployState) { - Execution execution = new Execution(model, deployState); if (validationParameters.checkRouting()) { - execution.run(new RoutingValidator()); - execution.run(new RoutingSelectorValidator()); + new RoutingValidator().validate(model, deployState); + new RoutingSelectorValidator().validate(model, deployState); } - execution.run(new SchemasDirValidator()); - execution.run(new BundleValidator()); - execution.run(new PublicApiBundleValidator()); - execution.run(new SearchDataTypeValidator()); - execution.run(new ComplexFieldsWithStructFieldAttributesValidator()); - execution.run(new ComplexFieldsWithStructFieldIndexesValidator()); - execution.run(new StreamingValidator()); - execution.run(new RankSetupValidator(validationParameters.ignoreValidationErrors())); - execution.run(new NoPrefixForIndexes()); - execution.run(new ContainerInCloudValidator()); - execution.run(new DeploymentSpecValidator()); - execution.run(new ValidationOverridesValidator()); - execution.run(new ConstantValidator()); - execution.run(new SecretStoreValidator()); - execution.run(new EndpointCertificateSecretsValidator()); - execution.run(new AccessControlFilterValidator()); - execution.run(new QuotaValidator()); - execution.run(new UriBindingsValidator()); - execution.run(new CloudDataPlaneFilterValidator()); - execution.run(new AccessControlFilterExcludeValidator()); - execution.run(new CloudUserFilterValidator()); - execution.run(new CloudHttpConnectorValidator()); - execution.run(new UrlConfigValidator()); - execution.run(new JvmHeapSizeValidator()); - - additionalValidators.forEach(execution::run); + new SchemasDirValidator().validate(model, deployState); + new BundleValidator().validate(model, deployState); + new PublicApiBundleValidator().validate(model, deployState); + new SearchDataTypeValidator().validate(model, deployState); + new ComplexFieldsWithStructFieldAttributesValidator().validate(model, deployState); + new ComplexFieldsWithStructFieldIndexesValidator().validate(model, deployState); + new StreamingValidator().validate(model, deployState); + new RankSetupValidator(validationParameters.ignoreValidationErrors()).validate(model, deployState); + new NoPrefixForIndexes().validate(model, deployState); + new ContainerInCloudValidator().validate(model, deployState); + new DeploymentSpecValidator().validate(model, deployState); + new ValidationOverridesValidator().validate(model, deployState); + new ConstantValidator().validate(model, deployState); + new SecretStoreValidator().validate(model, deployState); + new EndpointCertificateSecretsValidator().validate(model, deployState); + new AccessControlFilterValidator().validate(model, deployState); + new QuotaValidator().validate(model, deployState); + new UriBindingsValidator().validate(model, deployState); + new CloudDataPlaneFilterValidator().validate(model, deployState); + new AccessControlFilterExcludeValidator().validate(model, deployState); + new CloudUserFilterValidator().validate(model, deployState); + new CloudHttpConnectorValidator().validate(model, deployState); + new UrlConfigValidator().validate(model, deployState); + new JvmHeapSizeValidator().validate(model, deployState); + + additionalValidators.forEach(v -> v.validate(model, deployState)); List<ConfigChangeAction> result = Collections.emptyList(); if (deployState.getProperties().isFirstTimeDeployment()) { - validateFirstTimeDeployment(execution); + validateFirstTimeDeployment(model, deployState); } else { Optional<Model> currentActiveModel = deployState.getPreviousModel(); if (currentActiveModel.isPresent() && (currentActiveModel.get() instanceof VespaModel)) { - result = validateChanges((VespaModel) currentActiveModel.get(), execution); + result = validateChanges((VespaModel) currentActiveModel.get(), model, deployState); deferConfigChangesForClustersToBeRestarted(result, model); } } - execution.throwIfFailed(); return result; } - private static List<ConfigChangeAction> validateChanges(VespaModel currentModel, Execution execution) { + private static List<ConfigChangeAction> validateChanges(VespaModel currentModel, VespaModel nextModel, + DeployState deployState) { ChangeValidator[] validators = new ChangeValidator[] { new IndexingModeChangeValidator(), new GlobalDocumentChangeValidator(), @@ -133,15 +127,20 @@ public class Validation { new RestartOnDeployForOnnxModelChangesValidator(), }; List<ConfigChangeAction> actions = Arrays.stream(validators) - .flatMap(v -> v.validate(currentModel, execution.model, execution.deployState).stream()) + .flatMap(v -> v.validate(currentModel, nextModel, deployState).stream()) .toList(); - execution.runChanges(actions); + Map<ValidationId, Collection<String>> disallowableActions = actions.stream() + .filter(action -> action.validationId().isPresent()) + .collect(groupingBy(action -> action.validationId().orElseThrow(), + mapping(ConfigChangeAction::getMessage, + toCollection(LinkedHashSet::new)))); + deployState.validationOverrides().invalid(disallowableActions, deployState.now()); return actions; } - private static void validateFirstTimeDeployment(Execution execution) { - execution.run(new RedundancyValidator()); + private static void validateFirstTimeDeployment(VespaModel model, DeployState deployState) { + new RedundancyValidator().validate(model, deployState); } private static void deferConfigChangesForClustersToBeRestarted(List<ConfigChangeAction> actions, VespaModel model) { @@ -160,53 +159,4 @@ public class Validation { } } - - private static class Execution { - - private final Map<ValidationId, List<String>> failures = new LinkedHashMap<>(); - private final VespaModel model; - private final DeployState deployState; - - private Execution(VespaModel model, DeployState deployState) { - this.model = model; - this.deployState = deployState; - } - - private void run(Validator validator) { - try { - validator.validate(model, deployState); - } - catch (ValidationException e) { - e.messagesById().forEach((id, messages) -> failures.computeIfAbsent(id, __ -> new ArrayList<>()).addAll(messages)); - } - } - - private void runChanges(List<ConfigChangeAction> actions) { - for (ConfigChangeAction action : actions) { - if (action.validationId().isPresent()) run(new Validator() { // Changes without a validation ID are always allowed. - @Override public void validate(VespaModel model, DeployState deployState) { - deployState.validationOverrides().invalid(action.validationId().get(), action.getMessage(), deployState.now()); - } - }); - } - } - - private void throwIfFailed() { - try { - if (failures.size() == 1 && failures.values().iterator().next().size() == 1) // Retain single-form exception message when possible. - deployState.validationOverrides().invalid(failures.keySet().iterator().next(), failures.values().iterator().next().get(0), deployState.now()); - else - deployState.validationOverrides().invalid(failures, deployState.now()); - } - catch (ValidationException e) { - if (deployState.isHosted() && deployState.zone().environment().isManuallyDeployed()) - deployState.getDeployLogger().logApplicationPackage(Level.WARNING, - "Auto-overriding validation which would be disallowed in production: " + - Exceptions.toMessageString(e)); - else throw e; - } - } - - } - } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidator.java index e2dd3aca0b9..fb07f65b5f4 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidator.java @@ -40,7 +40,7 @@ public class IndexingScriptChangeValidator { String fieldName = nextField.getName(); ImmutableSDField currentField = currentSchema.getConcreteField(fieldName); if (currentField != null) { - validateScripts(currentField, nextField).ifPresent(result::add); + validateScripts(currentField, nextField).ifPresent(r -> result.add(r)); } else if (nextField.isExtraField()) { result.add(VespaReindexAction.of(id, diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/CertificateRemovalChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/CertificateRemovalChangeValidatorTest.java index bc36b800bfb..6b7df8871aa 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/CertificateRemovalChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/CertificateRemovalChangeValidatorTest.java @@ -23,11 +23,9 @@ import static org.junit.jupiter.api.Assertions.assertThrows; public class CertificateRemovalChangeValidatorTest { private static final String validationOverrides = - """ - <validation-overrides> - <allow until='2000-01-14' comment='test override'>certificate-removal</allow> - </validation-overrides> - """; + "<validation-overrides>\n" + + " <allow until='2000-01-14' comment='test override'>certificate-removal</allow>\n" + + "</validation-overrides>\n"; @Test void validate() { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexingModeChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexingModeChangeValidatorTest.java index 9e0eab9aba7..3fd3180b37e 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexingModeChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexingModeChangeValidatorTest.java @@ -22,28 +22,19 @@ import static org.junit.jupiter.api.Assertions.fail; public class IndexingModeChangeValidatorTest { @Test - void testChangingIndexModeFromIndexedToStreamingWhenDisallowedButInDev() { - ValidationTester tester = new ValidationTester(); - - VespaModel oldModel = - tester.deploy(null, getServices("index"), Environment.dev, "<validation-overrides />").getFirst(); - List<ConfigChangeAction> actions = tester.deploy(oldModel, getServices("streaming"), Environment.dev, "<calidation-overrides />").getSecond(); - assertReindexingChange("Document type 'music' in cluster 'default-content' changed indexing mode from 'indexed' to 'streaming'", actions); - } - - @Test void testChangingIndexModeFromIndexedToStreamingWhenDisallowed() { ValidationTester tester = new ValidationTester(); VespaModel oldModel = tester.deploy(null, getServices("index"), Environment.prod, "<validation-overrides />").getFirst(); try { - tester.deploy(oldModel, getServices("streaming"), Environment.prod, "<calidation-overrides />").getSecond(); + List<ConfigChangeAction> changeActions = + tester.deploy(oldModel, getServices("streaming"), Environment.prod, "<calidation-overrides />").getSecond(); fail("Should throw on disallowed config change action"); } catch (ValidationException e) { - assertEquals("indexing-mode-change: " + - "Document type 'music' in cluster 'default-content' changed indexing mode from 'indexed' to 'streaming'. " + + assertEquals("indexing-mode-change:\n" + + "\tDocument type 'music' in cluster 'default-content' changed indexing mode from 'indexed' to 'streaming'\n" + "To allow this add <allow until='yyyy-mm-dd'>indexing-mode-change</allow> to validation-overrides.xml, see https://docs.vespa.ai/en/reference/validation-overrides.html", e.getMessage()); } @@ -103,10 +94,8 @@ public class IndexingModeChangeValidatorTest { } private static final String validationOverrides = - """ - <validation-overrides> - <allow until='2000-01-14' comment='test override'>indexing-mode-change</allow> - </validation-overrides> - """; + "<validation-overrides>\n" + + " <allow until='2000-01-14' comment='test override'>indexing-mode-change</allow>\n" + + "</validation-overrides>\n"; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantHandler.java index 79a55466021..e9b1c8c83f8 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/TenantHandler.java @@ -6,6 +6,7 @@ import com.yahoo.component.annotation.Inject; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.restapi.ErrorResponse; +import com.yahoo.restapi.MessageResponse; import com.yahoo.restapi.RestApi; import com.yahoo.restapi.RestApiException; import com.yahoo.restapi.RestApiRequestHandler; @@ -63,7 +64,7 @@ public class TenantHandler extends RestApiRequestHandler<TenantHandler> { private HttpResponse putTenant(RestApi.RequestContext context) { TenantName name = TenantName.from(context.pathParameters().getStringOrThrow("tenant")); if (tenantRepository.checkThatTenantExists(name)) - throw new RestApiException.BadRequest("There already exists a tenant '" + name + "'"); + return new MessageResponse("Tenant '" + name + "' already exists"); if ( ! name.value().matches(TENANT_NAME_REGEXP)) throw new RestApiException.BadRequest("Illegal tenant name: " + name); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TenantHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TenantHandlerTest.java index 49e6ebbfcff..fbebd463569 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TenantHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TenantHandlerTest.java @@ -108,7 +108,7 @@ public class TenantHandlerTest { "{\"message\":\"Tenant a created.\"}"); assertEquals(tenantRepository.getTenant(a).getName(), a); assertResponse(PUT, "/application/v2/tenant/a", - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"There already exists a tenant 'a'\"}"); + "{\"message\":\"Tenant 'a' already exists\"}"); } @Test diff --git a/dependency-versions/pom.xml b/dependency-versions/pom.xml index 31bcb6bdc91..59fca668ebb 100644 --- a/dependency-versions/pom.xml +++ b/dependency-versions/pom.xml @@ -33,7 +33,7 @@ <!-- DO NOT UPGRADE THESE TO A NEW MAJOR VERSION WITHOUT CHECKING FOR BINARY COMPATIBILITY --> <aopalliance.vespa.version>1.0</aopalliance.vespa.version> - <error-prone-annotations.vespa.version>2.23.0</error-prone-annotations.vespa.version> + <error-prone-annotations.vespa.version>2.24.0</error-prone-annotations.vespa.version> <guava.vespa.version>33.0.0-jre</guava.vespa.version> <guice.vespa.version>6.0.0</guice.vespa.version> <jackson2.vespa.version>2.16.0</jackson2.vespa.version> @@ -66,7 +66,7 @@ <!-- Athenz dependencies. Make sure these dependencies match those in Vespa's internal repositories --> <athenz.vespa.version>1.11.48</athenz.vespa.version> - <aws-sdk.vespa.version>1.12.607</aws-sdk.vespa.version> + <aws-sdk.vespa.version>1.12.623</aws-sdk.vespa.version> <!-- Athenz END --> <!-- WARNING: If you change curator version, you also need to update @@ -76,7 +76,7 @@ xargs perl -pi -e 's/major = [0-9]+, minor = [0-9]+, micro = [0-9]+/major = 5, minor = 3, micro = 0/g' --> <bouncycastle.vespa.version>1.76</bouncycastle.vespa.version> - <byte-buddy.vespa.version>1.14.10</byte-buddy.vespa.version> + <byte-buddy.vespa.version>1.14.11</byte-buddy.vespa.version> <checker-qual.vespa.version>3.38.0</checker-qual.vespa.version> <commons-beanutils.vespa.version>1.9.4</commons-beanutils.vespa.version> <commons-codec.vespa.version>1.16.0</commons-codec.vespa.version> diff --git a/lowercasing_test/src/tests/lowercasing/casingvariants_fastlib.cpp b/lowercasing_test/src/tests/lowercasing/casingvariants_fastlib.cpp index c723470f0fb..3aa2bbe5a86 100644 --- a/lowercasing_test/src/tests/lowercasing/casingvariants_fastlib.cpp +++ b/lowercasing_test/src/tests/lowercasing/casingvariants_fastlib.cpp @@ -27,7 +27,7 @@ main(int argc, char ** argv) ref.getline(refBuf, 128); ucs4_t inputChar = getUCS4Char(inputBuf); ucs4_t refChar = getUCS4Char(refBuf); - ucs4_t lowerChar = wordFolder.ToFold(inputChar); + ucs4_t lowerChar = wordFolder.lowercase_and_fold(inputChar); Fast_UnicodeUtil::utf8ncopy(lowerBuf, &lowerChar, 128, 1); if (refChar != lowerChar) { printf("input(%s,%u,0x%X), lower(%s,%u,0x%X), ref(%s,%u,0x%X) \n", diff --git a/parent/pom.xml b/parent/pom.xml index 42d8a15bca2..5fd89facdc5 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -317,7 +317,7 @@ --> <groupId>org.openrewrite.maven</groupId> <artifactId>rewrite-maven-plugin</artifactId> - <version>5.17.0</version> + <version>5.17.1</version> <configuration> <activeRecipes> <recipe>org.openrewrite.java.testing.junit5.JUnit5BestPractices</recipe> @@ -327,7 +327,7 @@ <dependency> <groupId>org.openrewrite.recipe</groupId> <artifactId>rewrite-testing-frameworks</artifactId> - <version>2.1.4</version> + <version>2.1.5</version> </dependency> </dependencies> </plugin> @@ -1157,7 +1157,7 @@ See pluginManagement of rewrite-maven-plugin for more details --> <groupId>org.openrewrite.recipe</groupId> <artifactId>rewrite-recipe-bom</artifactId> - <version>2.5.3</version> + <version>2.5.4</version> <type>pom</type> <scope>import</scope> </dependency> diff --git a/searchlib/src/vespa/searchlib/common/sortspec.cpp b/searchlib/src/vespa/searchlib/common/sortspec.cpp index 04bc87f1000..40e2616367f 100644 --- a/searchlib/src/vespa/searchlib/common/sortspec.cpp +++ b/searchlib/src/vespa/searchlib/common/sortspec.cpp @@ -30,7 +30,7 @@ LowercaseConverter::onConvert(const ConstBufferRef & src) const vespalib::Utf8Writer w(_buffer); while (r.hasMore()) { ucs4_t c = r.getChar(0xFFFD); - c = Fast_NormalizeWordFolder::ToFold(c); + c = Fast_NormalizeWordFolder::lowercase_and_fold(c); w.putChar(c); } return {_buffer.begin(), _buffer.size()}; diff --git a/searchsummary/src/vespa/juniper/tokenizer.cpp b/searchsummary/src/vespa/juniper/tokenizer.cpp index cd3c9c410ce..211ffe7054a 100644 --- a/searchsummary/src/vespa/juniper/tokenizer.cpp +++ b/searchsummary/src/vespa/juniper/tokenizer.cpp @@ -8,11 +8,10 @@ #include <vespa/log/log.h> LOG_SETUP(".juniper.tokenizer"); -JuniperTokenizer::JuniperTokenizer(const Fast_WordFolder* wordfolder, - const char* text, size_t len, ITokenProcessor* successor, - const juniper::SpecialTokenRegistry * registry) : +JuniperTokenizer::JuniperTokenizer(const Fast_WordFolder* wordfolder, const char* text, size_t len, + ITokenProcessor* successor, const juniper::SpecialTokenRegistry * registry) : _wordfolder(wordfolder), _text(text), _len(len), _successor(successor), _registry(registry), - _charpos(0), _wordpos(0) + _charpos(0), _wordpos(0), _buffer() { } @@ -32,19 +31,19 @@ void JuniperTokenizer::scan() const char* src = _text; const char* src_end = _text + _len; - const char* startpos = NULL; + const char* startpos = nullptr; ucs4_t* dst = _buffer; ucs4_t* dst_end = dst + TOKEN_DSTLEN; size_t result_len; while (src < src_end) { - if (_registry == NULL) { + if (_registry == nullptr) { // explicit prefetching seems to have negative effect with many threads src = _wordfolder->UCS4Tokenize(src, src_end, dst, dst_end, startpos, result_len); } else { const char * tmpSrc = _registry->tokenize(src, src_end, dst, dst_end, startpos, result_len); - if (tmpSrc == NULL) { + if (tmpSrc == nullptr) { src = _wordfolder->UCS4Tokenize(src, src_end, dst, dst_end, startpos, result_len); } else { src = tmpSrc; @@ -63,6 +62,6 @@ void JuniperTokenizer::scan() } token.bytepos = _len; token.bytelen = 0; - token.token = NULL; + token.token = nullptr; _successor->handle_end(token); } diff --git a/searchsummary/src/vespa/juniper/tokenizer.h b/searchsummary/src/vespa/juniper/tokenizer.h index 68ef8118f5d..910da3f67ef 100644 --- a/searchsummary/src/vespa/juniper/tokenizer.h +++ b/searchsummary/src/vespa/juniper/tokenizer.h @@ -12,8 +12,8 @@ class JuniperTokenizer { public: JuniperTokenizer(const Fast_WordFolder* wordfolder, - const char* text, size_t len, ITokenProcessor* = NULL, - const juniper::SpecialTokenRegistry * registry = NULL); + const char* text, size_t len, ITokenProcessor* = nullptr, + const juniper::SpecialTokenRegistry * registry = nullptr); inline void SetSuccessor(ITokenProcessor* successor) { _successor = successor; } void setRegistry(const juniper::SpecialTokenRegistry * registry) { _registry = registry; } @@ -23,13 +23,13 @@ public: void scan(); private: const Fast_WordFolder* _wordfolder; - const char* _text; // The current input text - size_t _len; // Length of the text input - ITokenProcessor* _successor; + const char* _text; // The current input text + size_t _len; // Length of the text input + ITokenProcessor* _successor; const juniper::SpecialTokenRegistry * _registry; - off_t _charpos; // Last utf8 character position - off_t _wordpos; // Offset in numbering of words compared to input (as result of splits) - ucs4_t _buffer[TOKEN_DSTLEN]; // Temp. buffer to store folding result + off_t _charpos; // Last utf8 character position + off_t _wordpos; // Offset in numbering of words compared to input (as result of splits) + ucs4_t _buffer[TOKEN_DSTLEN]; // Temp. buffer to store folding result private: JuniperTokenizer(const JuniperTokenizer&); JuniperTokenizer& operator=(const JuniperTokenizer&); diff --git a/streamingvisitors/src/vespa/vsm/searcher/utf8stringfieldsearcherbase.cpp b/streamingvisitors/src/vespa/vsm/searcher/utf8stringfieldsearcherbase.cpp index 4daea693e95..c31102ec0ab 100644 --- a/streamingvisitors/src/vespa/vsm/searcher/utf8stringfieldsearcherbase.cpp +++ b/streamingvisitors/src/vespa/vsm/searcher/utf8stringfieldsearcherbase.cpp @@ -25,8 +25,8 @@ UTF8StringFieldSearcherBase::tokenize(const byte * p, size_t maxSz, cmptype_t * if (c < 128) { if (!c) { break; } p++; - if (__builtin_expect(Fast_NormalizeWordFolder::_isWord[c], false)) { - *q++ = Fast_NormalizeWordFolder::_foldCase[c]; + if (__builtin_expect(Fast_NormalizeWordFolder::is_wordchar_ascii7bit(c), false)) { + *q++ = Fast_NormalizeWordFolder::lowercase_and_fold_ascii(c); c = 0; } else { c = *p; @@ -37,13 +37,13 @@ UTF8StringFieldSearcherBase::tokenize(const byte * p, size_t maxSz, cmptype_t * if (Fast_UnicodeUtil::IsWordChar(c)) { _utf8Count[p-oldP-1]++; const char *repl = Fast_NormalizeWordFolder::ReplacementString(c); - if (repl != NULL) { + if (repl != nullptr) { size_t repllen = strlen(repl); if (repllen > 0) { q = Fast_UnicodeUtil::ucs4copy(q,repl); } } else { - c = Fast_NormalizeWordFolder::ToFold(c); + c = Fast_NormalizeWordFolder::lowercase_and_fold(c); *q++ = c; } break; @@ -63,10 +63,10 @@ UTF8StringFieldSearcherBase::tokenize(const byte * p, size_t maxSz, cmptype_t * if (c < 128) { // Common case, ASCII if (!c) { break; } p++; - if (__builtin_expect(!Fast_NormalizeWordFolder::_isWord[c], false)) { + if (__builtin_expect(!Fast_NormalizeWordFolder::is_wordchar_ascii7bit(c), false)) { c = 0; } else { - *q++ = Fast_NormalizeWordFolder::_foldCase[c]; + *q++ = Fast_NormalizeWordFolder::lowercase_and_fold_ascii(c); c = *p; } } else { @@ -75,13 +75,13 @@ UTF8StringFieldSearcherBase::tokenize(const byte * p, size_t maxSz, cmptype_t * if (__builtin_expect(Fast_UnicodeUtil::IsWordChar(c), false)) { _utf8Count[p-oldP-1]++; const char *repl = Fast_NormalizeWordFolder::ReplacementString(c); - if (repl != NULL) { + if (repl != nullptr) { size_t repllen = strlen(repl); if (repllen > 0) { q = Fast_UnicodeUtil::ucs4copy(q,repl); } } else { - c = Fast_NormalizeWordFolder::ToFold(c); + c = Fast_NormalizeWordFolder::lowercase_and_fold(c); *q++ = c; } @@ -144,9 +144,9 @@ UTF8StringFieldSearcherBase::matchTermExact(const FieldRef & f, QueryTerm & qt) bool equal(true); for (; equal && (n < e) && (term < eterm); term++) { if (*term < 0x80) { - equal = (*term == Fast_NormalizeWordFolder::_foldCase[*n++]); + equal = (*term == Fast_NormalizeWordFolder::lowercase_ascii(*n++)); } else { - cmptype_t c = Fast_NormalizeWordFolder::ToFold(Fast_UnicodeUtil::GetUTF8CharNonAscii(n)); + cmptype_t c = Fast_NormalizeWordFolder::lowercase(Fast_UnicodeUtil::GetUTF8CharNonAscii(n)); equal = (*term == c); } } @@ -280,12 +280,12 @@ UTF8StringFieldSearcherBase::skipSeparators(const search::byte * p, size_t sz, T if (c < 128) { p++; if (!isSeparatorCharacter(c)) { - dstbuf.onCharacter(Fast_NormalizeWordFolder::_foldCase[c], (oldP - b)); + dstbuf.onCharacter(Fast_NormalizeWordFolder::lowercase_and_fold_ascii(c), (oldP - b)); } } else { c = Fast_UnicodeUtil::GetUTF8CharNonAscii(p); const char *repl = Fast_NormalizeWordFolder::ReplacementString(c); - if (repl != NULL) { + if (repl != nullptr) { size_t repllen = strlen(repl); if (repllen > 0) { ucs4_t * buf = dstbuf.getBuf(); @@ -300,7 +300,7 @@ UTF8StringFieldSearcherBase::skipSeparators(const search::byte * p, size_t sz, T } } } else { - c = Fast_NormalizeWordFolder::ToFold(c); + c = Fast_NormalizeWordFolder::lowercase_and_fold(c); dstbuf.onCharacter(c, (oldP - b)); } if (c == Fast_UnicodeUtil::_BadUTF8Char) { diff --git a/vespalib/src/vespa/fastlib/text/normwordfolder.cpp b/vespalib/src/vespa/fastlib/text/normwordfolder.cpp index ef6d17e20f1..f9dbf202fcb 100644 --- a/vespalib/src/vespa/fastlib/text/normwordfolder.cpp +++ b/vespalib/src/vespa/fastlib/text/normwordfolder.cpp @@ -13,7 +13,9 @@ bool Fast_NormalizeWordFolder::_doMulticharExpansion = false; bool Fast_NormalizeWordFolder::_isWord[128]; ucs4_t Fast_NormalizeWordFolder::_foldCase[767]; // Up to Latin Extended B (0x0250) +ucs4_t Fast_NormalizeWordFolder::_lowerCase[767]; ucs4_t Fast_NormalizeWordFolder::_foldCaseHighAscii[256]; // Latin Extended Additional (0x1E00 - 0x1F00) +ucs4_t Fast_NormalizeWordFolder::_lowerCaseHighAscii[256]; ucs4_t Fast_NormalizeWordFolder::_kanaMap[192]; ucs4_t Fast_NormalizeWordFolder::_halfwidth_fullwidthMap[240]; @@ -43,11 +45,10 @@ Fast_NormalizeWordFolder::Initialize() for (i = 0; i < 128; i++) _isWord[i] = Fast_UnicodeUtil::IsWordChar(i); for (i = 0; i < 767; i++) { - _foldCase[i] = Fast_UnicodeUtil::ToLower(i); + _foldCase[i] = _lowerCase[i] = Fast_UnicodeUtil::ToLower(i); } - for (i = 0x1E00; i < 0x1F00; i++) { - _foldCaseHighAscii[i - 0x1E00] = Fast_UnicodeUtil::ToLower(i); + _foldCaseHighAscii[i - 0x1E00] = _lowerCaseHighAscii[i - 0x1E00] = Fast_UnicodeUtil::ToLower(i); } if (_doAccentRemoval) { @@ -394,17 +395,11 @@ Fast_NormalizeWordFolder::Fast_NormalizeWordFolder() } -Fast_NormalizeWordFolder::~Fast_NormalizeWordFolder(void) -{ -} +Fast_NormalizeWordFolder::~Fast_NormalizeWordFolder() = default; const char* -Fast_NormalizeWordFolder::UCS4Tokenize(const char *buf, - const char *bufend, - ucs4_t *dstbuf, - ucs4_t *dstbufend, - const char*& origstart, - size_t& tokenlen) const +Fast_NormalizeWordFolder::UCS4Tokenize(const char *buf, const char *bufend, ucs4_t *dstbuf, + ucs4_t *dstbufend, const char*& origstart, size_t& tokenlen) const { ucs4_t c; @@ -451,7 +446,7 @@ Fast_NormalizeWordFolder::UCS4Tokenize(const char *buf, if (repllen > 0) q = Fast_UnicodeUtil::ucs4copy(q,repl); } else { - c = ToFold(c); + c = lowercase_and_fold(c); *q++ = c; } } @@ -563,7 +558,7 @@ Fast_NormalizeWordFolder::UCS4Tokenize(const char *buf, if (repllen > 0) q = Fast_UnicodeUtil::ucs4copy(q,repl); } else { - c = ToFold(c); + c = lowercase_and_fold(c); *q++ = c; } if (q >= eq) { // Junk rest of word diff --git a/vespalib/src/vespa/fastlib/text/normwordfolder.h b/vespalib/src/vespa/fastlib/text/normwordfolder.h index d7b07b698c9..c596b0fd2b4 100644 --- a/vespalib/src/vespa/fastlib/text/normwordfolder.h +++ b/vespalib/src/vespa/fastlib/text/normwordfolder.h @@ -11,21 +11,6 @@ */ class Fast_NormalizeWordFolder : public Fast_WordFolder { -private: - static bool _isInitialized; - - /** Features */ - static bool _doAccentRemoval; - static bool _doSharpSSubstitution; - static bool _doLigatureSubstitution; - static bool _doMulticharExpansion; - - /** - * Freeze the config, either from call to Setup, environment - * or defaults. - */ - static void Initialize(); - public: enum { DO_ACCENT_REMOVAL = 0x1 << 0, @@ -37,6 +22,10 @@ public: DO_LIGATURE_SUBSTITUTION = 0x1 << 6, DO_MULTICHAR_EXPANSION = 0x1 << 7 }; + Fast_NormalizeWordFolder(); + ~Fast_NormalizeWordFolder() override; + const char* UCS4Tokenize(const char *buf, const char *bufend, ucs4_t *dstbuf, + ucs4_t *dstbufend, const char*& origstart, size_t& tokenlen) const override; /** * Setup behaviour prior to constructing an object. * Not needed if default behaviour is wanted. The default is @@ -46,33 +35,38 @@ public: * added together. */ static void Setup(uint32_t flags); - -public: - /** character tables */ - static bool _isWord[128]; - static ucs4_t _foldCase[767]; // Up to Spacing Modifiers, inclusize (0x02FF) - static ucs4_t _foldCaseHighAscii[256]; // Latin Extended Additional (0x1E00 - 0x1F00) (incl. vietnamese) -private: - /** Map the values from range 0x3040 (0) - 0x30FF (191). */ - static ucs4_t _kanaMap[192]; - static ucs4_t _halfwidth_fullwidthMap[240]; -public: - static ucs4_t ToFold(ucs4_t testchar) { - if (testchar < 767) - return _foldCase[testchar]; - else if (testchar >= 0x1E00 && testchar < 0x1F00) - return _foldCaseHighAscii[testchar - 0x1E00]; + static ucs4_t lowercase_and_fold_ascii(ucs4_t c) noexcept { return _lowerCase[c]; } + static ucs4_t lowercase_ascii(ucs4_t c) noexcept { return _foldCase[c]; } + static bool is_wordchar_ascii7bit(ucs4_t c) noexcept { return _isWord[c]; } + static ucs4_t lowercase(ucs4_t c) { + if (c < 767) + return _lowerCase[c]; + else if (c >= 0x1E00 && c < 0x1F00) + return _lowerCaseHighAscii[c - 0x1E00]; + else + if (c >= 0x3040 && c < 0x3100) + return _kanaMap[c - 0x3040]; + else + if (c >= 0xFF00 && c < 0xFFF0) + return _halfwidth_fullwidthMap[c - 0xFF00]; + else + return Fast_UnicodeUtil::ToLower(c); + } + static ucs4_t lowercase_and_fold(ucs4_t c) { + if (c < 767) + return _foldCase[c]; + else if (c >= 0x1E00 && c < 0x1F00) + return _foldCaseHighAscii[c - 0x1E00]; else - if (testchar >= 0x3040 && testchar < 0x3100) - return _kanaMap[testchar - 0x3040]; + if (c >= 0x3040 && c < 0x3100) + return _kanaMap[c - 0x3040]; else - if (testchar >= 0xFF00 && testchar < 0xFFF0) - return _halfwidth_fullwidthMap[testchar - 0xFF00]; + if (c >= 0xFF00 && c < 0xFFF0) + return _halfwidth_fullwidthMap[c - 0xFF00]; else - return Fast_UnicodeUtil::ToLower(testchar); + return Fast_UnicodeUtil::ToLower(c); } -public: static const char *ReplacementString(ucs4_t testchar) { if (testchar < 0xc4 || testchar > 0x1f3) { return nullptr; @@ -150,18 +144,26 @@ public: } private: /** - * Check if the given char is a word character or used - * for interlinear annotation. - * @param c The character to check. - * @return true if c is a word character, or interlinear annotation syntax characters. + * Freeze the config, either from call to Setup, environment + * or defaults. */ + static void Initialize(); static bool IsWordCharOrIA(ucs4_t c) { - return Fast_UnicodeUtil::IsWordChar(c) - || c == 0xFFF9 || c == 0xFFFA || c == 0xFFFB; + return Fast_UnicodeUtil::IsWordChar(c) || c == 0xFFF9 || c == 0xFFFA || c == 0xFFFB; } -public: - Fast_NormalizeWordFolder(); - ~Fast_NormalizeWordFolder() override; - const char* UCS4Tokenize(const char *buf, const char *bufend, ucs4_t *dstbuf, - ucs4_t *dstbufend, const char*& origstart, size_t& tokenlen) const override; + + /** character tables */ + static bool _isWord[128]; + static ucs4_t _foldCase[767]; // Up to Spacing Modifiers, inclusize (0x02FF) + static ucs4_t _lowerCase[767]; + static ucs4_t _foldCaseHighAscii[256]; // Latin Extended Additional (0x1E00 - 0x1F00) (incl. vietnamese) + static ucs4_t _lowerCaseHighAscii[256]; + /** Map the values from range 0x3040 (0) - 0x30FF (191). */ + static ucs4_t _kanaMap[192]; + static ucs4_t _halfwidth_fullwidthMap[240]; + static bool _isInitialized; + static bool _doAccentRemoval; + static bool _doSharpSSubstitution; + static bool _doLigatureSubstitution; + static bool _doMulticharExpansion; }; diff --git a/vespalib/src/vespa/fastlib/text/wordfolder.h b/vespalib/src/vespa/fastlib/text/wordfolder.h index e5412859f3e..ac8c590be7c 100644 --- a/vespalib/src/vespa/fastlib/text/wordfolder.h +++ b/vespalib/src/vespa/fastlib/text/wordfolder.h @@ -7,10 +7,6 @@ class Fast_WordFolder { public: virtual ~Fast_WordFolder() = default; - virtual const char* UCS4Tokenize(const char *buf, - const char *bufend, - ucs4_t *dstbuf, - ucs4_t *dstbufend, - const char*& origstart, - size_t& tokenlen) const = 0; + virtual const char* UCS4Tokenize(const char *buf, const char *bufend, ucs4_t *dstbuf, + ucs4_t *dstbufend, const char*& origstart, size_t& tokenlen) const = 0; }; |