diff options
182 files changed, 2245 insertions, 1435 deletions
diff --git a/build_settings.cmake b/build_settings.cmake index d7dd26f5ee7..1a05ffe5073 100644 --- a/build_settings.cmake +++ b/build_settings.cmake @@ -66,8 +66,14 @@ else() set(VESPA_STDCXX_FS_LIB "stdc++fs") endif() +if(VESPA_OS_DISTRO_COMBINED STREQUAL "debian 10") + unset(VESPA_XXHASH_DEFINE) +else() + set(VESPA_XXHASH_DEFINE "-DXXH_INLINE_ALL") +endif() + # C and C++ compiler flags -set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O3 -fno-omit-frame-pointer ${C_WARN_OPTS} -fPIC ${VESPA_CXX_ABI_FLAGS} -DXXH_INLINE_ALL -DBOOST_DISABLE_ASSERTS ${VESPA_CPU_ARCH_FLAGS} ${EXTRA_C_FLAGS}") +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O3 -fno-omit-frame-pointer ${C_WARN_OPTS} -fPIC ${VESPA_CXX_ABI_FLAGS} ${VESPA_XXHASH_DEFINE} -DBOOST_DISABLE_ASSERTS ${VESPA_CPU_ARCH_FLAGS} ${EXTRA_C_FLAGS}") # AddressSanitizer/ThreadSanitizer work for both GCC and Clang if (VESPA_USE_SANITIZER) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=${VESPA_USE_SANITIZER}") diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java index 62b9cefab78..4c6dccebae4 100644 --- a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java +++ b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java @@ -197,7 +197,7 @@ public class InMemoryProvisioner implements HostProvisioner { int nextIndex = nextIndexInCluster.getOrDefault(new Pair<>(clusterGroup.type(), clusterGroup.id()), startIndex); while (allocation.size() < nodesInGroup) { - // Find the smallest host that can fit the requested requested + // Find the smallest host that can fit the requested resources Optional<NodeResources> hostResources = freeNodes.keySet().stream() .sorted(new MemoryDiskCpu()) .filter(resources -> requestedResources.isUnspecified() || resources.satisfies(requestedResources)) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java index f9338f9cb35..9ae9a158631 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java @@ -7,6 +7,7 @@ import com.yahoo.container.handler.ThreadpoolConfig; import com.yahoo.search.config.QrStartConfig; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.component.Handler; +import com.yahoo.vespa.model.container.component.SystemBindingPattern; /** * @author hmusum @@ -39,7 +40,7 @@ public class LogserverContainerCluster extends ContainerCluster<LogserverContain private void addLogHandler() { Handler<?> logHandler = Handler.fromClassName(ContainerCluster.LOG_HANDLER_CLASS); - logHandler.addServerBindings("http://*/logs"); + logHandler.addServerBindings(SystemBindingPattern.fromHttpPath("/logs")); addComponent(logHandler); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java index 08f4e2fa12f..5b3e4e1479e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java @@ -14,6 +14,7 @@ import com.yahoo.vespa.model.container.Container; import com.yahoo.vespa.model.container.component.AccessLogComponent; import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.container.component.Handler; +import com.yahoo.vespa.model.container.component.SystemBindingPattern; import com.yahoo.vespa.model.container.xml.PlatformBundles; import java.util.Set; @@ -36,10 +37,10 @@ public class ClusterControllerContainer extends Container implements super(parent, "" + index, index); addHandler("clustercontroller-status", "com.yahoo.vespa.clustercontroller.apps.clustercontroller.StatusHandler", - "clustercontroller-status/*"); + "/clustercontroller-status/*"); addHandler("clustercontroller-state-restapi-v2", "com.yahoo.vespa.clustercontroller.apps.clustercontroller.StateRestApiV2Handler", - "cluster/v2/*"); + "/cluster/v2/*"); if (runStandaloneZooKeeper) { addComponent("clustercontroller-zkrunner", "com.yahoo.vespa.zookeeper.VespaZooKeeperServerImpl", @@ -77,8 +78,8 @@ public class ClusterControllerContainer extends Container implements return ContainerServiceType.CLUSTERCONTROLLER_CONTAINER; } - private void addHandler(Handler h, String binding) { - h.addServerBindings("http://*/" + binding); + private void addHandler(Handler h, String path) { + h.addServerBindings(SystemBindingPattern.fromHttpPath(path)); super.addHandler(h); } @@ -96,9 +97,8 @@ public class ClusterControllerContainer extends Container implements addComponent(new Component<>(createComponentModel(id, className, bundle))); } - private void addHandler(String id, String className, String binding) { - addHandler(new Handler(createComponentModel(id, className, CLUSTERCONTROLLER_BUNDLE)), - binding); + private void addHandler(String id, String className, String path) { + addHandler(new Handler(createComponentModel(id, className, CLUSTERCONTROLLER_BUNDLE)), path); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java index 4dc9811a024..b5936887b50 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java @@ -7,12 +7,12 @@ import ai.vespa.metricsproxy.core.MetricsConsumers; import ai.vespa.metricsproxy.core.MetricsManager; import ai.vespa.metricsproxy.core.MonitoringConfig; import ai.vespa.metricsproxy.core.VespaMetrics; -import ai.vespa.metricsproxy.http.metrics.MetricsV1Handler; import ai.vespa.metricsproxy.http.application.ApplicationMetricsHandler; import ai.vespa.metricsproxy.http.application.ApplicationMetricsRetriever; import ai.vespa.metricsproxy.http.application.MetricsNodesConfig; -import ai.vespa.metricsproxy.http.yamas.YamasHandler; +import ai.vespa.metricsproxy.http.metrics.MetricsV1Handler; import ai.vespa.metricsproxy.http.prometheus.PrometheusHandler; +import ai.vespa.metricsproxy.http.yamas.YamasHandler; import ai.vespa.metricsproxy.metric.ExternalMetrics; import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensions; import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensionsConfig; @@ -38,6 +38,7 @@ import com.yahoo.vespa.model.admin.monitoring.MetricsConsumer; import com.yahoo.vespa.model.admin.monitoring.Monitoring; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.component.Handler; +import com.yahoo.vespa.model.container.component.SystemBindingPattern; import com.yahoo.vespa.model.container.xml.PlatformBundles; import java.nio.file.Path; @@ -129,8 +130,9 @@ public class MetricsProxyContainerCluster extends ContainerCluster<MetricsProxyC static Handler<AbstractConfigProducer<?>> createMetricsHandler(Class<? extends ThreadedHttpRequestHandler> clazz, String bindingPath) { Handler<AbstractConfigProducer<?>> metricsHandler = new Handler<>( new ComponentModel(clazz.getName(), null, METRICS_PROXY_BUNDLE_NAME, null)); - metricsHandler.addServerBindings("http://*" + bindingPath, - "http://*" + bindingPath + "/*"); + metricsHandler.addServerBindings( + SystemBindingPattern.fromHttpPath(bindingPath), + SystemBindingPattern.fromHttpPath(bindingPath + "/*")); return metricsHandler; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/UriBindingsValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/UriBindingsValidator.java new file mode 100644 index 00000000000..249827b11d9 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/UriBindingsValidator.java @@ -0,0 +1,80 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation; + +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.container.ApplicationContainerCluster; +import com.yahoo.vespa.model.container.component.BindingPattern; +import com.yahoo.vespa.model.container.component.Handler; +import com.yahoo.vespa.model.container.component.SystemBindingPattern; +import com.yahoo.vespa.model.container.http.FilterBinding; +import com.yahoo.vespa.model.container.http.Http; + +import java.util.logging.Level; + +import static com.yahoo.config.model.ConfigModelContext.ApplicationType.HOSTED_INFRASTRUCTURE; + +/** + * Validates URI bindings for filters and handlers + * + * @author bjorncs + */ +class UriBindingsValidator extends Validator { + + @Override + public void validate(VespaModel model, DeployState deployState) { + for (ApplicationContainerCluster cluster : model.getContainerClusters().values()) { + for (Handler<?> handler : cluster.getHandlers()) { + for (BindingPattern binding : handler.getServerBindings()) { + validateUserBinding(binding, model, deployState); + } + } + Http http = cluster.getHttp(); + if (http != null) { + for (FilterBinding binding : cluster.getHttp().getBindings()) { + validateUserBinding(binding.binding(), model, deployState); + } + } + } + } + + private static void validateUserBinding(BindingPattern binding, VespaModel model, DeployState deployState) { + validateScheme(binding, deployState); + if (isHostedApplication(model, deployState)) { + validateHostedApplicationUserBinding(binding); + } + } + + private static void validateScheme(BindingPattern binding, DeployState deployState) { + if (binding.scheme().equals("https")) { + String message = createErrorMessage( + binding, "'https' bindings are deprecated, use 'http' instead to bind to both http and https traffic."); + deployState.getDeployLogger().log(Level.WARNING, message); + } + } + + private static void validateHostedApplicationUserBinding(BindingPattern binding) { + // only perform these validation for used-generated bindings + // bindings produced by the hosted config model amender will violate some of the rules below + if (binding instanceof SystemBindingPattern) return; + + if (binding.port().isPresent()) { + throw new IllegalArgumentException(createErrorMessage(binding, "binding with port is not allowed")); + } + if (!binding.host().equals(BindingPattern.WILDCARD_PATTERN)) { + throw new IllegalArgumentException(createErrorMessage(binding, "only binding with wildcard ('*') for hostname is allowed")); + } + if (!binding.scheme().equals("http") && !binding.scheme().equals("https")) { + throw new IllegalArgumentException(createErrorMessage(binding, "only 'http' is allowed as scheme")); + } + } + + private static boolean isHostedApplication(VespaModel model, DeployState deployState) { + return deployState.isHosted() && model.getAdmin().getApplicationType() != HOSTED_INFRASTRUCTURE; + } + + private static String createErrorMessage(BindingPattern binding, String message) { + return String.format("For binding '%s': %s", binding.patternString(), message); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java index 22dd0289390..3a4dee300da 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java @@ -61,6 +61,7 @@ public class Validation { new AccessControlFilterValidator().validate(model, deployState); new CloudWatchValidator().validate(model, deployState); new AwsAccessControlValidator().validate(model, deployState); + new UriBindingsValidator().validate(model, deployState); List<ConfigChangeAction> result = Collections.emptyList(); if (deployState.getProperties().isFirstTimeDeployment()) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomClientProviderBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomClientProviderBuilder.java index 11fab0ada29..0fdd1af56f3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomClientProviderBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomClientProviderBuilder.java @@ -2,11 +2,12 @@ package com.yahoo.vespa.model.builder.xml.dom; import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.text.XML; import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.text.XML; import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.container.component.Handler; +import com.yahoo.vespa.model.container.component.UserBindingPattern; import org.w3c.dom.Element; /** @@ -24,10 +25,10 @@ public class DomClientProviderBuilder extends DomHandlerBuilder { Handler<? super Component<?, ?>> client = createHandler(clientElement); for (Element binding : XML.getChildren(clientElement, "binding")) - client.addClientBindings(XML.getValue(binding)); + client.addClientBindings(UserBindingPattern.fromPattern(XML.getValue(binding))); for (Element serverBinding : XML.getChildren(clientElement, "serverBinding")) - client.addServerBindings(XML.getValue(serverBinding)); + client.addServerBindings(UserBindingPattern.fromPattern(XML.getValue(serverBinding))); DomComponentBuilder.addChildren(deployState, parent, clientElement, client); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java index ac6d089cf24..145535fe06f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java @@ -8,8 +8,10 @@ import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.text.XML; import com.yahoo.vespa.model.container.ApplicationContainerCluster; +import com.yahoo.vespa.model.container.component.BindingPattern; import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.container.component.Handler; +import com.yahoo.vespa.model.container.component.UserBindingPattern; import com.yahoo.vespa.model.container.xml.BundleInstantiationSpecificationBuilder; import org.w3c.dom.Element; @@ -27,11 +29,14 @@ import static java.util.logging.Level.INFO; */ public class DomHandlerBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Handler> { - private static final Set<String> reservedBindings = Set.of(METRICS_V2_HANDLER_BINDING_1, - METRICS_V2_HANDLER_BINDING_2, - STATE_HANDLER_BINDING_1, - STATE_HANDLER_BINDING_2, - VIP_HANDLER_BINDING); + private static final Set<BindingPattern> reservedBindings = + Set.of( + METRICS_V2_HANDLER_BINDING_1, + METRICS_V2_HANDLER_BINDING_2, + STATE_HANDLER_BINDING_1, + STATE_HANDLER_BINDING_2, + VIP_HANDLER_BINDING); + private final ApplicationContainerCluster cluster; public DomHandlerBuilder(ApplicationContainerCluster cluster) { @@ -43,10 +48,10 @@ public class DomHandlerBuilder extends VespaDomBuilder.DomConfigProducerBuilder< Handler<? super Component<?, ?>> handler = createHandler(handlerElement); for (Element binding : XML.getChildren(handlerElement, "binding")) - addServerBinding(handler, XML.getValue(binding), deployState.getDeployLogger()); + addServerBinding(handler, UserBindingPattern.fromPattern(XML.getValue(binding)), deployState.getDeployLogger()); for (Element clientBinding : XML.getChildren(handlerElement, "clientBinding")) - handler.addClientBindings(XML.getValue(clientBinding)); + handler.addClientBindings(UserBindingPattern.fromPattern(XML.getValue(clientBinding))); DomComponentBuilder.addChildren(deployState, parent, handlerElement, handler); @@ -58,27 +63,30 @@ public class DomHandlerBuilder extends VespaDomBuilder.DomConfigProducerBuilder< return new Handler<>(new ComponentModel(bundleSpec)); } - private void addServerBinding(Handler<? super Component<?, ?>> handler, String binding, DeployLogger log) { + private void addServerBinding(Handler<? super Component<?, ?>> handler, BindingPattern binding, DeployLogger log) { throwIfBindingIsReserved(binding, handler); handler.addServerBindings(binding); removeExistingServerBinding(binding, handler, log); } - private void throwIfBindingIsReserved(String binding, Handler<?> newHandler) { + private void throwIfBindingIsReserved(BindingPattern binding, Handler<?> newHandler) { for (var reserved : reservedBindings) { - if (binding.equals(reserved)) { - throw new IllegalArgumentException("Binding '" + binding + "' is a reserved Vespa binding and " + + if (binding.hasSamePattern(reserved)) { + throw new IllegalArgumentException("Binding '" + binding.patternString() + "' is a reserved Vespa binding and " + "cannot be used by handler: " + newHandler.getComponentId()); } } } - private void removeExistingServerBinding(String binding, Handler<?> newHandler, DeployLogger log) { + private void removeExistingServerBinding(BindingPattern binding, Handler<?> newHandler, DeployLogger log) { for (var handler : cluster.getHandlers()) { - if (handler.getServerBindings().contains(binding)) { - handler.removeServerBinding(binding); - log.log(INFO, "Binding '" + binding + "' was already in use by handler '" + - handler.getComponentId() + "', but will now be taken over by handler: " + newHandler.getComponentId()); + for (BindingPattern serverBinding : handler.getServerBindings()) { + if (serverBinding.hasSamePattern(binding)) { + handler.removeServerBinding(serverBinding); + log.log(INFO, "Binding '" + binding.patternString() + "' was already in use by handler '" + + handler.getComponentId() + "', but will now be taken over by handler: " + newHandler.getComponentId()); + + } } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java b/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java index 58f03bffb30..159a87be27d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java @@ -6,6 +6,8 @@ import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.component.Handler; +import com.yahoo.vespa.model.container.component.SystemBindingPattern; +import com.yahoo.vespa.model.container.component.UserBindingPattern; import java.util.Collection; import java.util.Collections; @@ -24,7 +26,7 @@ public class ContainerDocumentApi { } private void setupHandlers(ContainerCluster cluster) { - cluster.addComponent(newVespaClientHandler("com.yahoo.document.restapi.resource.RestApi", "document/v1/*")); + cluster.addComponent(newVespaClientHandler("com.yahoo.document.restapi.resource.RestApi", "/document/v1/*")); cluster.addComponent(newVespaClientHandler("com.yahoo.vespa.http.server.FeedHandler", ContainerCluster.RESERVED_URI_PREFIX + "/feedapi")); } @@ -32,9 +34,18 @@ public class ContainerDocumentApi { Handler<AbstractConfigProducer<?>> handler = new Handler<>(new ComponentModel( BundleInstantiationSpecification.getFromStrings(componentId, null, vespaClientBundleSpecification), "")); - for (String rootBinding : options.bindings) { - handler.addServerBindings(rootBinding + bindingSuffix, - rootBinding + bindingSuffix + '/'); + if (options.bindings.isEmpty()) { + handler.addServerBindings( + SystemBindingPattern.fromHttpPath(bindingSuffix), + SystemBindingPattern.fromHttpPath(bindingSuffix + '/')); + } else { + for (String rootBinding : options.bindings) { + String pathWithoutLeadingSlash = bindingSuffix.substring(1); + handler.addServerBindings( + UserBindingPattern.fromPattern(rootBinding + pathWithoutLeadingSlash), + UserBindingPattern.fromPattern(rootBinding + pathWithoutLeadingSlash + '/')); + } + } return handler; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index b0ac02d0fe8..1427fa492dc 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -22,10 +22,12 @@ import com.yahoo.search.config.QrStartConfig; import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.vespa.config.search.core.RankingConstantsConfig; import com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainer; +import com.yahoo.vespa.model.container.component.BindingPattern; import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.container.component.ConfigProducerGroup; import com.yahoo.vespa.model.container.component.Handler; import com.yahoo.vespa.model.container.component.Servlet; +import com.yahoo.vespa.model.container.component.SystemBindingPattern; import com.yahoo.vespa.model.container.jersey.Jersey2Servlet; import com.yahoo.vespa.model.container.jersey.RestApi; import com.yahoo.vespa.model.container.xml.PlatformBundles; @@ -55,12 +57,12 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat MetricsProxyApiConfig.Producer { public static final String METRICS_V2_HANDLER_CLASS = MetricsV2Handler.class.getName(); - public static final String METRICS_V2_HANDLER_BINDING_1 = "http://*" + MetricsV2Handler.V2_PATH; - public static final String METRICS_V2_HANDLER_BINDING_2 = METRICS_V2_HANDLER_BINDING_1 + "/*"; + public static final BindingPattern METRICS_V2_HANDLER_BINDING_1 = SystemBindingPattern.fromHttpPath(MetricsV2Handler.V2_PATH); + public static final BindingPattern METRICS_V2_HANDLER_BINDING_2 = SystemBindingPattern.fromHttpPath(MetricsV2Handler.V2_PATH + "/*"); public static final String PROMETHEUS_V1_HANDLER_CLASS = PrometheusV1Handler.class.getName(); - private static final String PROMETHEUS_V1_HANDLER_BINDING_1 = "http://*" + PrometheusV1Handler.V1_PATH; - private static final String PROMETHEUS_V1_HANDLER_BINDING_2 = PROMETHEUS_V1_HANDLER_BINDING_1 + "/*"; + private static final BindingPattern PROMETHEUS_V1_HANDLER_BINDING_1 = SystemBindingPattern.fromHttpPath(PrometheusV1Handler.V1_PATH); + private static final BindingPattern PROMETHEUS_V1_HANDLER_BINDING_2 = SystemBindingPattern.fromHttpPath(PrometheusV1Handler.V1_PATH + "/*"); public static final int heapSizePercentageOfTotalNodeMemory = 60; public static final int heapSizePercentageOfTotalNodeMemoryWhenCombinedCluster = 17; @@ -125,7 +127,7 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat addMetricsHandler(PROMETHEUS_V1_HANDLER_CLASS, PROMETHEUS_V1_HANDLER_BINDING_1, PROMETHEUS_V1_HANDLER_BINDING_2); } - private void addMetricsHandler(String handlerClass, String rootBinding, String innerBinding) { + private void addMetricsHandler(String handlerClass, BindingPattern rootBinding, BindingPattern innerBinding) { Handler<AbstractConfigProducer<?>> handler = new Handler<>( new ComponentModel(handlerClass, null, null, null)); handler.addServerBindings(rootBinding, innerBinding); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index 240157fb7aa..8bb456ab7e7 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -39,6 +39,7 @@ import com.yahoo.vespa.model.Service; import com.yahoo.vespa.model.admin.monitoring.Monitoring; import com.yahoo.vespa.model.clients.ContainerDocumentApi; import com.yahoo.vespa.model.container.component.AccessLogComponent; +import com.yahoo.vespa.model.container.component.BindingPattern; import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.container.component.ComponentGroup; import com.yahoo.vespa.model.container.component.ComponentsConfigGenerator; @@ -47,6 +48,7 @@ import com.yahoo.vespa.model.container.component.FileStatusHandlerComponent; import com.yahoo.vespa.model.container.component.Handler; import com.yahoo.vespa.model.container.component.SimpleComponent; import com.yahoo.vespa.model.container.component.StatisticsComponent; +import com.yahoo.vespa.model.container.component.SystemBindingPattern; import com.yahoo.vespa.model.container.component.chain.ProcessingHandler; import com.yahoo.vespa.model.container.docproc.ContainerDocproc; import com.yahoo.vespa.model.container.docproc.DocprocChains; @@ -107,7 +109,7 @@ public abstract class ContainerCluster<CONTAINER extends Container> * normal compatibility concerns only applies to libraries using the URIs in * question, not contents served from the URIs themselves. */ - public static final String RESERVED_URI_PREFIX = "reserved-for-internal-use"; + public static final String RESERVED_URI_PREFIX = "/reserved-for-internal-use"; public static final String APPLICATION_STATUS_HANDLER_CLASS = "com.yahoo.container.handler.observability.ApplicationStatusHandler"; public static final String BINDINGS_OVERVIEW_HANDLER_CLASS = BindingsOverviewHandler.class.getName(); @@ -117,13 +119,13 @@ public abstract class ContainerCluster<CONTAINER extends Container> public static final String G1GC = "-XX:+UseG1GC -XX:MaxTenuringThreshold=15"; public static final String STATE_HANDLER_CLASS = "com.yahoo.container.jdisc.state.StateHandler"; - public static final String STATE_HANDLER_BINDING_1 = "http://*" + StateHandler.STATE_API_ROOT; - public static final String STATE_HANDLER_BINDING_2 = STATE_HANDLER_BINDING_1 + "/*"; + public static final BindingPattern STATE_HANDLER_BINDING_1 = SystemBindingPattern.fromHttpPath(StateHandler.STATE_API_ROOT); + public static final BindingPattern STATE_HANDLER_BINDING_2 = SystemBindingPattern.fromHttpPath(StateHandler.STATE_API_ROOT + "/*"); public static final String ROOT_HANDLER_PATH = "/"; - public static final String ROOT_HANDLER_BINDING = "http://*" + ROOT_HANDLER_PATH; + public static final BindingPattern ROOT_HANDLER_BINDING = SystemBindingPattern.fromHttpPath(ROOT_HANDLER_PATH); - public static final String VIP_HANDLER_BINDING = "http://*/status.html"; + public static final BindingPattern VIP_HANDLER_BINDING = SystemBindingPattern.fromHttpPath("/status.html"); private final String name; @@ -234,7 +236,7 @@ public abstract class ContainerCluster<CONTAINER extends Container> Handler<AbstractConfigProducer<?>> statusHandler = new Handler<>( new ComponentModel(BundleInstantiationSpecification.getInternalHandlerSpecificationFromStrings( APPLICATION_STATUS_HANDLER_CLASS, null), null)); - statusHandler.addServerBindings("http://*/ApplicationStatus"); + statusHandler.addServerBindings(SystemBindingPattern.fromHttpPath("/ApplicationStatus")); addComponent(statusHandler); } @@ -309,7 +311,7 @@ public abstract class ContainerCluster<CONTAINER extends Container> containers.forEach(this::addContainer); } - public void setProcessingChains(ProcessingChains processingChains, String... serverBindings) { + public void setProcessingChains(ProcessingChains processingChains, BindingPattern... serverBindings) { if (this.processingChains != null) throw new IllegalStateException("ProcessingChains should only be set once."); @@ -320,7 +322,7 @@ public abstract class ContainerCluster<CONTAINER extends Container> processingChains, "com.yahoo.processing.handler.ProcessingHandler"); - for (String binding: serverBindings) + for (BindingPattern binding: serverBindings) processingHandler.addServerBindings(binding); addComponent(processingHandler); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java index 6b4f8d486ec..72f1921e6a2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java @@ -7,6 +7,7 @@ import com.yahoo.searchdefinition.derived.RankProfileList; import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.vespa.config.search.core.RankingConstantsConfig; import com.yahoo.vespa.model.container.component.Handler; +import com.yahoo.vespa.model.container.component.SystemBindingPattern; import java.util.List; import java.util.Objects; @@ -21,7 +22,7 @@ public class ContainerModelEvaluation implements RankProfilesConfig.Producer, Ra private final static String BUNDLE_NAME = "model-evaluation"; private final static String EVALUATOR_NAME = ModelsEvaluator.class.getName(); private final static String REST_HANDLER_NAME = "ai.vespa.models.handler.ModelsEvaluationHandler"; - private final static String REST_BINDING = "model-evaluation/v1"; + private final static String REST_BINDING_PATH = "/model-evaluation/v1"; /** Global rank profiles, aka models */ private final RankProfileList rankProfileList; @@ -48,8 +49,9 @@ public class ContainerModelEvaluation implements RankProfilesConfig.Producer, Ra public static Handler<?> getHandler() { Handler<?> handler = new Handler<>(new ComponentModel(REST_HANDLER_NAME, null, BUNDLE_NAME)); - handler.addServerBindings("http://*/" + REST_BINDING, - "http://*/" + REST_BINDING + "/*"); + handler.addServerBindings( + SystemBindingPattern.fromHttpPath(REST_BINDING_PATH), + SystemBindingPattern.fromHttpPath(REST_BINDING_PATH + "/*")); return handler; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/BindingPattern.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/BindingPattern.java new file mode 100644 index 00000000000..1d5736ba7e2 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/BindingPattern.java @@ -0,0 +1,90 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.component; + +import java.util.Comparator; +import java.util.Objects; +import java.util.Optional; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * URI binding pattern used by filter and handler bindings. + * + * @author bjorncs + */ +public abstract class BindingPattern implements Comparable<BindingPattern> { + + private static final Pattern BINDING_PATTERN = + Pattern.compile("([^:]+)://([^:/]+)(:((\\*)|([0-9]+)))?(/.*)", Pattern.UNICODE_CASE | Pattern.CANON_EQ); + + public static final String WILDCARD_PATTERN = "*"; + + private final String scheme; + private final String host; + private final String port; + private final String path; + + protected BindingPattern( + String scheme, + String host, + String port, + String path) { + this.scheme = Objects.requireNonNull(scheme, "Scheme in binding must be specified"); + this.host = Objects.requireNonNull(host, "Host must be specified"); + this.port = port; + this.path = validatePath(path); + } + + protected BindingPattern(String binding) { + Matcher matcher = BINDING_PATTERN.matcher(binding); + if (!matcher.matches()) throw new IllegalArgumentException("Invalid binding: " + binding); + this.scheme = matcher.group(1); + this.host = matcher.group(2); + this.port = matcher.group(4); + this.path = matcher.group(7); + } + + private static String validatePath(String path) { + Objects.requireNonNull(path, "Path must be specified"); + if (!path.startsWith("/")) throw new IllegalArgumentException("Path must have '/' as prefix: " + path); + return path; + } + + public String scheme() { return scheme; } + public String host() { return host; } + public Optional<String> port() { return Optional.ofNullable(port); } + public String path() { return path; } + + public String patternString() { + StringBuilder builder = new StringBuilder(scheme).append("://").append(host); + if (port != null) { + builder.append(':').append(port); + } + return builder.append(path).toString(); + } + + /** Compares the underlying pattern string for equality */ + public boolean hasSamePattern(BindingPattern other) { return this.patternString().equals(other.patternString()); } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + BindingPattern that = (BindingPattern) o; + return Objects.equals(scheme, that.scheme) && + Objects.equals(host, that.host) && + Objects.equals(port, that.port) && + Objects.equals(path, that.path); + } + + @Override public int hashCode() { return Objects.hash(scheme, host, port, path); } + + @Override + public int compareTo(BindingPattern o) { + return Comparator.comparing(BindingPattern::scheme) + .thenComparing(BindingPattern::host) + .thenComparing(pattern -> pattern.port().orElse(null)) + .thenComparing(BindingPattern::path) + .compare(this, o); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/DiscBindingsConfigGenerator.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/DiscBindingsConfigGenerator.java index d7e393ee474..02face328d9 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/DiscBindingsConfigGenerator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/DiscBindingsConfigGenerator.java @@ -1,13 +1,16 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container.component; -import java.util.*; +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; import static com.yahoo.container.jdisc.JdiscBindingsConfig.Handlers; +import static java.util.stream.Collectors.toList; /** * @author gjoranv - * @since 5.1.8 */ public class DiscBindingsConfigGenerator { @@ -26,7 +29,11 @@ public class DiscBindingsConfigGenerator { return Collections.singletonMap(handler.model.getComponentId().stringValue(), new Handlers.Builder() - .serverBindings(handler.getServerBindings()) - .clientBindings(handler.getClientBindings())); + .serverBindings(toStrings(handler.getServerBindings())) + .clientBindings(toStrings(handler.getClientBindings()))); + } + + private static Collection<String> toStrings(Collection<BindingPattern> bindings) { + return bindings.stream().map(BindingPattern::patternString).collect(toList()); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/FileStatusHandlerComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/FileStatusHandlerComponent.java index 3d9a1b2e665..839594502c6 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/FileStatusHandlerComponent.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/FileStatusHandlerComponent.java @@ -15,7 +15,7 @@ public class FileStatusHandlerComponent extends Handler implements VipStatusConf private final String fileName; - public FileStatusHandlerComponent(String id, String fileName, String... bindings) { + public FileStatusHandlerComponent(String id, String fileName, BindingPattern... bindings) { super(new ComponentModel(id, CLASS, null, null)); this.fileName = fileName; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java index 82484e07773..efee5c6a9a0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java @@ -1,9 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container.component; -import com.yahoo.container.bundle.BundleInstantiationSpecification; -import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.osgi.provider.model.ComponentModel; import java.util.ArrayList; import java.util.Arrays; @@ -23,8 +22,8 @@ import java.util.Set; */ public class Handler<CHILD extends AbstractConfigProducer<?>> extends Component<CHILD, ComponentModel> { - private Set<String> serverBindings = new LinkedHashSet<>(); - private List<String> clientBindings = new ArrayList<>(); + private final Set<BindingPattern> serverBindings = new LinkedHashSet<>(); + private final List<BindingPattern> clientBindings = new ArrayList<>(); public Handler(ComponentModel model) { super(model); @@ -34,27 +33,23 @@ public class Handler<CHILD extends AbstractConfigProducer<?>> extends Component< return new Handler<>(new ComponentModel(className, null, null, null)); } - public static Handler<AbstractConfigProducer<?>> getVespaHandlerFromClassName(String className) { - return new Handler<>(new ComponentModel(BundleInstantiationSpecification.getInternalHandlerSpecificationFromStrings(className, null), null)); - } - - public void addServerBindings(String... bindings) { + public void addServerBindings(BindingPattern... bindings) { serverBindings.addAll(Arrays.asList(bindings)); } - public void removeServerBinding(String binding) { + public void removeServerBinding(BindingPattern binding) { serverBindings.remove(binding); } - public void addClientBindings(String... bindings) { + public void addClientBindings(BindingPattern... bindings) { clientBindings.addAll(Arrays.asList(bindings)); } - public final Set<String> getServerBindings() { + public final Set<BindingPattern> getServerBindings() { return Collections.unmodifiableSet(serverBindings); } - public final List<String> getClientBindings() { + public final List<BindingPattern> getClientBindings() { return Collections.unmodifiableList(clientBindings); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/SystemBindingPattern.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/SystemBindingPattern.java new file mode 100644 index 00000000000..3ae531539ef --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/SystemBindingPattern.java @@ -0,0 +1,27 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.component; + +/** + * A {@link BindingPattern} which is implicitly constructed by the model, e.g for built-in handlers and filter chains. + * + * @author bjorncs + */ +public class SystemBindingPattern extends BindingPattern { + + private SystemBindingPattern(String scheme, String host, String port, String path) { super(scheme, host, port, path); } + private SystemBindingPattern(String binding) { super(binding); } + + public static SystemBindingPattern fromHttpPath(String path) { return new SystemBindingPattern("http", "*", null, path);} + public static SystemBindingPattern fromPattern(String binding) { return new SystemBindingPattern(binding);} + public static SystemBindingPattern fromHttpPortAndPath(String port, String path) { return new SystemBindingPattern("http", "*", port, path); } + + @Override + public String toString() { + return "SystemBindingPattern{" + + "scheme='" + scheme() + '\'' + + ", host='" + host() + '\'' + + ", port='" + port().orElse(null) + '\'' + + ", path='" + path() + '\'' + + '}'; + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/UserBindingPattern.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/UserBindingPattern.java new file mode 100644 index 00000000000..43f57fa0343 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/UserBindingPattern.java @@ -0,0 +1,26 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.component; + +/** + * A {@link BindingPattern} which is constructed directly from a user provided 'binding' element from services.xml. + * + * @author bjorncs + */ +public class UserBindingPattern extends BindingPattern { + + private UserBindingPattern(String scheme, String host, String port, String path) { super(scheme, host, port, path); } + private UserBindingPattern(String binding) { super(binding); } + + public static UserBindingPattern fromHttpPath(String path) { return new UserBindingPattern("http", "*", null, path); } + public static UserBindingPattern fromPattern(String binding) { return new UserBindingPattern(binding); } + + @Override + public String toString() { + return "UserBindingPattern{" + + "scheme='" + scheme() + '\'' + + ", host='" + host() + '\'' + + ", port='" + port().orElse(null) + '\'' + + ", path='" + path() + '\'' + + '}'; + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/ContainerDocproc.java b/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/ContainerDocproc.java index d4b4dcea78e..82061a0425f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/ContainerDocproc.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/ContainerDocproc.java @@ -9,6 +9,7 @@ import com.yahoo.container.jdisc.config.SessionConfig; import com.yahoo.docproc.jdisc.messagebus.MbusRequestContext; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.component.ContainerSubsystem; +import com.yahoo.vespa.model.container.component.SystemBindingPattern; import java.util.HashMap; import java.util.Map; @@ -44,7 +45,7 @@ public class ContainerDocproc extends ContainerSubsystem<DocprocChains> private void addSource( final ContainerCluster cluster, final String name, final SessionConfig.Type.Enum type) { final MbusClient mbusClient = new MbusClient(name, type); - mbusClient.addClientBindings("mbus://*/" + mbusClient.getSessionName()); + mbusClient.addClientBindings(SystemBindingPattern.fromPattern("mbus://*/" + mbusClient.getSessionName())); cluster.addComponent(mbusClient); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/DocprocChains.java b/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/DocprocChains.java index 5d08a0a6998..68dc2518c23 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/DocprocChains.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/DocprocChains.java @@ -7,6 +7,7 @@ import com.yahoo.container.jdisc.config.SessionConfig; import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.component.Component; +import com.yahoo.vespa.model.container.component.SystemBindingPattern; import com.yahoo.vespa.model.container.component.chain.Chains; import com.yahoo.vespa.model.container.component.chain.ProcessingHandler; @@ -38,12 +39,12 @@ public class DocprocChains extends Chains<DocprocChain> { } private void addServerAndClientForChain(ApplicationContainerCluster cluster, DocprocChain docprocChain) { - docprocHandler.addServerBindings("mbus://*/" + docprocChain.getSessionName()); + docprocHandler.addServerBindings(SystemBindingPattern.fromPattern("mbus://*/" + docprocChain.getSessionName())); cluster.addMbusServer(ComponentId.fromString(docprocChain.getSessionName())); MbusClient client = new MbusClient(docprocChain.getSessionName(), SessionConfig.Type.INTERMEDIATE); - client.addClientBindings("mbus://*/" + client.getSessionName()); + client.addClientBindings(SystemBindingPattern.fromPattern("mbus://*/" + client.getSessionName())); addComponent(client); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java index 9676b8b1e4a..e96951dc83a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java @@ -3,21 +3,20 @@ package com.yahoo.vespa.model.container.http; import com.yahoo.component.ComponentId; import com.yahoo.component.ComponentSpecification; -import com.yahoo.config.application.api.DeployLogger; import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.ContainerCluster; +import com.yahoo.vespa.model.container.component.BindingPattern; import com.yahoo.vespa.model.container.component.FileStatusHandlerComponent; import com.yahoo.vespa.model.container.component.Handler; -import com.yahoo.vespa.model.container.component.Servlet; +import com.yahoo.vespa.model.container.component.SystemBindingPattern; +import com.yahoo.vespa.model.container.component.chain.Chain; -import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; import java.util.Set; -import java.util.stream.Collectors; -import java.util.stream.Stream; /** * Helper class for http access control. @@ -25,11 +24,15 @@ import java.util.stream.Stream; * @author gjoranv * @author bjorncs */ -public final class AccessControl { +public class AccessControl { public static final ComponentId ACCESS_CONTROL_CHAIN_ID = ComponentId.fromString("access-control-chain"); + public static final ComponentId ACCESS_CONTROL_EXCLUDED_CHAIN_ID = ComponentId.fromString("access-control-excluded-chain"); - public static final List<String> UNPROTECTED_HANDLERS = List.of( + private static final int HOSTED_CONTAINER_PORT = 4443; + + // Handlers that are excluded from access control + private static final List<String> EXCLUDED_HANDLERS = List.of( FileStatusHandlerComponent.CLASS, ContainerCluster.APPLICATION_STATUS_HANDLER_CLASS, ContainerCluster.BINDINGS_OVERVIEW_HANDLER_CLASS, @@ -39,18 +42,15 @@ public final class AccessControl { ApplicationContainerCluster.PROMETHEUS_V1_HANDLER_CLASS ); - public static final class Builder { - private String domain; + public static class Builder { + private final String domain; private boolean readEnabled = false; private boolean writeEnabled = true; - private final Set<String> excludeBindings = new LinkedHashSet<>(); + private final Set<BindingPattern> excludeBindings = new LinkedHashSet<>(); private Collection<Handler<?>> handlers = Collections.emptyList(); - private Collection<Servlet> servlets = Collections.emptyList(); - private final DeployLogger logger; - public Builder(String domain, DeployLogger logger) { + public Builder(String domain) { this.domain = domain; - this.logger = logger; } public Builder readEnabled(boolean readEnabled) { @@ -58,102 +58,111 @@ public final class AccessControl { return this; } - public Builder writeEnabled(boolean writeEnalbed) { - this.writeEnabled = writeEnalbed; + public Builder writeEnabled(boolean writeEnabled) { + this.writeEnabled = writeEnabled; return this; } - public Builder excludeBinding(String binding) { + public Builder excludeBinding(BindingPattern binding) { this.excludeBindings.add(binding); return this; } public Builder setHandlers(ApplicationContainerCluster cluster) { this.handlers = cluster.getHandlers(); - this.servlets = cluster.getAllServlets(); return this; } public AccessControl build() { - return new AccessControl(domain, writeEnabled, readEnabled, - excludeBindings, servlets, handlers, logger); + return new AccessControl(domain, writeEnabled, readEnabled, excludeBindings, handlers); } } public final String domain; public final boolean readEnabled; public final boolean writeEnabled; - private final Set<String> excludedBindings; + private final Set<BindingPattern> excludedBindings; private final Collection<Handler<?>> handlers; - private final Collection<Servlet> servlets; - private final DeployLogger logger; private AccessControl(String domain, boolean writeEnabled, boolean readEnabled, - Set<String> excludedBindings, - Collection<Servlet> servlets, - Collection<Handler<?>> handlers, - DeployLogger logger) { + Set<BindingPattern> excludedBindings, + Collection<Handler<?>> handlers) { this.domain = domain; this.readEnabled = readEnabled; this.writeEnabled = writeEnabled; this.excludedBindings = Collections.unmodifiableSet(excludedBindings); this.handlers = handlers; - this.servlets = servlets; - this.logger = logger; } - public List<Binding> getBindings() { - return Stream.concat(getHandlerBindings(), getServletBindings()) - .collect(Collectors.toCollection(ArrayList::new)); + public void configureHttpFilterChains(Http http) { + http.setAccessControl(this); + addAccessControlFilterChain(http); + addAccessControlExcludedChain(http); + removeDuplicateBindingsFromAccessControlChain(http); } public static boolean hasHandlerThatNeedsProtection(ApplicationContainerCluster cluster) { - return cluster.getHandlers().stream().anyMatch(AccessControl::handlerNeedsProtection); - } - - private Stream<Binding> getHandlerBindings() { - return handlers.stream() - .filter(this::shouldHandlerBeProtected) - .flatMap(handler -> handler.getServerBindings().stream()) - .map(binding -> accessControlBinding(binding, logger)); + return cluster.getHandlers().stream() + .anyMatch(handler -> ! isExcludedHandler(handler) && hasNonMbusBinding(handler)); } - private Stream<Binding> getServletBindings() { - return servlets.stream() - .filter(this::shouldServletBeProtected) - .flatMap(AccessControl::servletBindings) - .map(binding -> accessControlBinding(binding, logger)); + private void addAccessControlFilterChain(Http http) { + http.getFilterChains().add(createChain(ACCESS_CONTROL_CHAIN_ID)); + http.getBindings().addAll(List.of(createAccessControlBinding("/"), createAccessControlBinding("/*"))); } - private boolean shouldHandlerBeProtected(Handler<?> handler) { - return ! isBuiltinGetOnly(handler) - && handler.getServerBindings().stream().noneMatch(excludedBindings::contains); + private void addAccessControlExcludedChain(Http http) { + http.getFilterChains().add(createChain(ACCESS_CONTROL_EXCLUDED_CHAIN_ID)); + for (BindingPattern excludedBinding : excludedBindings) { + http.getBindings().add(createAccessControlExcludedBinding(excludedBinding)); + } + for (Handler<?> handler : handlers) { + if (isExcludedHandler(handler)) { + for (BindingPattern binding : handler.getServerBindings()) { + http.getBindings().add(createAccessControlExcludedBinding(binding)); + } + } + } } - private static boolean isBuiltinGetOnly(Handler<?> handler) { - return UNPROTECTED_HANDLERS.contains(handler.getClassId().getName()); + // Remove bindings from access control chain that have binding pattern as a different filter chain + private void removeDuplicateBindingsFromAccessControlChain(Http http) { + Set<FilterBinding> duplicateBindings = new HashSet<>(); + for (FilterBinding binding : http.getBindings()) { + if (binding.chainId().toId().equals(ACCESS_CONTROL_CHAIN_ID)) { + for (FilterBinding otherBinding : http.getBindings()) { + if (!binding.chainId().equals(otherBinding.chainId()) + && binding.binding().equals(otherBinding.binding())) { + duplicateBindings.add(binding); + } + } + } + } + duplicateBindings.forEach(http.getBindings()::remove); } - private boolean shouldServletBeProtected(Servlet servlet) { - return servletBindings(servlet).noneMatch(excludedBindings::contains); + private static FilterBinding createAccessControlBinding(String path) { + return FilterBinding.create( + new ComponentSpecification(ACCESS_CONTROL_CHAIN_ID.stringValue()), + SystemBindingPattern.fromHttpPortAndPath(Integer.toString(HOSTED_CONTAINER_PORT), path)); } - private static Binding accessControlBinding(String binding, DeployLogger logger) { - return Binding.create(new ComponentSpecification(ACCESS_CONTROL_CHAIN_ID.stringValue()), binding, logger); + private static FilterBinding createAccessControlExcludedBinding(BindingPattern excludedBinding) { + BindingPattern rewrittenBinding = SystemBindingPattern.fromHttpPortAndPath( + Integer.toString(HOSTED_CONTAINER_PORT), excludedBinding.path()); // only keep path from excluded binding + return FilterBinding.create( + new ComponentSpecification(ACCESS_CONTROL_EXCLUDED_CHAIN_ID.stringValue()), + rewrittenBinding); } - private static Stream<String> servletBindings(Servlet servlet) { - return Stream.of("http://*/").map(protocol -> protocol + servlet.bindingPath); - } + private static Chain<Filter> createChain(ComponentId id) { return new Chain<>(FilterChains.emptyChainSpec(id)); } - private static boolean handlerNeedsProtection(Handler<?> handler) { - return ! isBuiltinGetOnly(handler) && hasNonMbusBinding(handler); - } + private static boolean isExcludedHandler(Handler<?> handler) { return EXCLUDED_HANDLERS.contains(handler.getClassId().getName()); } private static boolean hasNonMbusBinding(Handler<?> handler) { - return handler.getServerBindings().stream().anyMatch(binding -> ! binding.startsWith("mbus")); + return handler.getServerBindings().stream().anyMatch(binding -> ! binding.scheme().equals("mbus")); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/Binding.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/Binding.java deleted file mode 100644 index 28f4949f210..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/Binding.java +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.container.http; - -import com.yahoo.component.ComponentSpecification; -import com.yahoo.config.application.api.DeployLogger; - -import java.util.logging.Level; - -/** - * @author bjorncs - */ -public class Binding { - - private final ComponentSpecification filterId; - private final String binding; - - private Binding(ComponentSpecification filterId, String binding) { - this.filterId = filterId; - this.binding = binding; - } - - public static Binding create(ComponentSpecification filterId, String binding, DeployLogger logger) { - if (binding.startsWith("https://")) { - logger.log(Level.WARNING, String.format("For binding '%s' on '%s': 'https' bindings are deprecated, " + - "use 'http' instead to bind to both http and https traffic.", - binding, filterId)); - } - return new Binding(filterId, binding); - } - - public ComponentSpecification filterId() { - return filterId; - } - - public String binding() { - return binding; - } - -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/FilterBinding.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/FilterBinding.java new file mode 100644 index 00000000000..1ca54769683 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/FilterBinding.java @@ -0,0 +1,47 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.http; + +import com.yahoo.component.ComponentSpecification; +import com.yahoo.vespa.model.container.component.BindingPattern; + +import java.util.Objects; + +/** + * @author bjorncs + */ +public class FilterBinding { + + private final ComponentSpecification chainId; + private final BindingPattern binding; + + private FilterBinding(ComponentSpecification chainId, BindingPattern binding) { + this.chainId = chainId; + this.binding = binding; + } + + public static FilterBinding create(ComponentSpecification chainId, BindingPattern binding) { + return new FilterBinding(chainId, binding); + } + + public ComponentSpecification chainId() { + return chainId; + } + + public BindingPattern binding() { + return binding; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + FilterBinding that = (FilterBinding) o; + return Objects.equals(chainId, that.chainId) && + Objects.equals(binding, that.binding); + } + + @Override + public int hashCode() { + return Objects.hash(chainId, binding); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/Http.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/Http.java index 0fcf7b2d06c..f58f5faa382 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/Http.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/Http.java @@ -21,7 +21,7 @@ import java.util.concurrent.CopyOnWriteArrayList; public class Http extends AbstractConfigProducer<AbstractConfigProducer<?>> implements ServerConfig.Producer { private final FilterChains filterChains; - private final List<Binding> bindings = new CopyOnWriteArrayList<>(); + private final List<FilterBinding> bindings = new CopyOnWriteArrayList<>(); private volatile JettyHttpServer httpServer; private volatile AccessControl accessControl; @@ -64,7 +64,7 @@ public class Http extends AbstractConfigProducer<AbstractConfigProducer<?>> impl setHttpServer(null); } - public List<Binding> getBindings() { + public List<FilterBinding> getBindings() { return bindings; } @@ -74,16 +74,16 @@ public class Http extends AbstractConfigProducer<AbstractConfigProducer<?>> impl @Override public void getConfig(ServerConfig.Builder builder) { - for (Binding binding : bindings) { + for (FilterBinding binding : bindings) { builder.filter(new ServerConfig.Filter.Builder() - .id(binding.filterId().stringValue()) - .binding(binding.binding())); + .id(binding.chainId().stringValue()) + .binding(binding.binding().patternString())); } } @Override public void validate() { - if (((Collection<Binding>) bindings).isEmpty()) return; + if (((Collection<FilterBinding>) bindings).isEmpty()) return; if (filterChains == null) throw new IllegalArgumentException("Null FilterChains are not allowed when there are filter bindings"); @@ -91,9 +91,9 @@ public class Http extends AbstractConfigProducer<AbstractConfigProducer<?>> impl ComponentRegistry<ChainedComponent<?>> filters = filterChains.componentsRegistry(); ComponentRegistry<Chain<Filter>> chains = filterChains.allChains(); - for (Binding binding: bindings) { - if (filters.getComponent(binding.filterId()) == null && chains.getComponent(binding.filterId()) == null) - throw new RuntimeException("Can't find filter " + binding.filterId() + " for binding " + binding.binding()); + for (FilterBinding binding: bindings) { + if (filters.getComponent(binding.chainId()) == null && chains.getComponent(binding.chainId()) == null) + throw new RuntimeException("Can't find filter " + binding.chainId() + " for binding " + binding.binding()); } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java index bfde9b9add1..c86d8b206d5 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java @@ -13,9 +13,9 @@ import com.yahoo.vespa.model.builder.xml.dom.ModelElement; import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.Container; -import com.yahoo.vespa.model.container.component.chain.Chain; +import com.yahoo.vespa.model.container.component.UserBindingPattern; import com.yahoo.vespa.model.container.http.AccessControl; -import com.yahoo.vespa.model.container.http.Binding; +import com.yahoo.vespa.model.container.http.FilterBinding; import com.yahoo.vespa.model.container.http.FilterChains; import com.yahoo.vespa.model.container.http.Http; import org.w3c.dom.Element; @@ -25,8 +25,6 @@ import java.util.List; import java.util.Optional; import java.util.logging.Level; -import static com.yahoo.vespa.model.container.http.AccessControl.ACCESS_CONTROL_CHAIN_ID; - /** * @author Tony Vaagenes * @author gjoranv @@ -36,19 +34,17 @@ public class HttpBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Http> @Override protected Http doBuild(DeployState deployState, AbstractConfigProducer ancestor, Element spec) { FilterChains filterChains; - List<Binding> bindings = new ArrayList<>(); + List<FilterBinding> bindings = new ArrayList<>(); AccessControl accessControl = null; Element filteringElem = XML.getChild(spec, "filtering"); if (filteringElem != null) { filterChains = new FilterChainsBuilder().build(deployState, ancestor, filteringElem); - bindings = readFilterBindings(filteringElem, deployState.getDeployLogger()); + bindings = readFilterBindings(filteringElem); Element accessControlElem = XML.getChild(filteringElem, "access-control"); if (accessControlElem != null) { accessControl = buildAccessControl(deployState, ancestor, accessControlElem); - bindings.addAll(accessControl.getBindings()); - filterChains.add(new Chain<>(FilterChains.emptyChainSpec(ACCESS_CONTROL_CHAIN_ID))); } } else { filterChains = new FilterChainsBuilder().newChainsInstance(ancestor); @@ -56,14 +52,16 @@ public class HttpBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Http> Http http = new Http(filterChains); http.getBindings().addAll(bindings); - http.setAccessControl(accessControl); http.setHttpServer(new JettyHttpServerBuilder().build(deployState, ancestor, spec)); + if (accessControl != null) { + accessControl.configureHttpFilterChains(http); + } return http; } private AccessControl buildAccessControl(DeployState deployState, AbstractConfigProducer ancestor, Element accessControlElem) { AthenzDomain domain = getAccessControlDomain(deployState, accessControlElem); - AccessControl.Builder builder = new AccessControl.Builder(domain.value(), deployState.getDeployLogger()); + AccessControl.Builder builder = new AccessControl.Builder(domain.value()); getContainerCluster(ancestor).ifPresent(builder::setHandlers); @@ -75,7 +73,7 @@ public class HttpBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Http> Element excludeElem = XML.getChild(accessControlElem, "exclude"); if (excludeElem != null) { XML.getChildren(excludeElem, "binding").stream() - .map(XML::getValue) + .map(xml -> UserBindingPattern.fromPattern(XML.getValue(xml))) .forEach(builder::excludeBinding); } return builder.build(); @@ -113,8 +111,8 @@ public class HttpBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Http> return Optional.of((ApplicationContainerCluster) currentProducer); } - private List<Binding> readFilterBindings(Element filteringSpec, DeployLogger logger) { - List<Binding> result = new ArrayList<>(); + private List<FilterBinding> readFilterBindings(Element filteringSpec) { + List<FilterBinding> result = new ArrayList<>(); for (Element child: XML.getChildren(filteringSpec)) { String tagName = child.getTagName(); @@ -123,7 +121,7 @@ public class HttpBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Http> for (Element bindingSpec: XML.getChildren(child, "binding")) { String binding = XML.getValue(bindingSpec); - result.add(Binding.create(chainId, binding, logger)); + result.add(FilterBinding.create(chainId, UserBindingPattern.fromPattern(binding))); } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/processing/ProcessingChains.java b/config-model/src/main/java/com/yahoo/vespa/model/container/processing/ProcessingChains.java index 4fd79a4f335..f6b24bf9635 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/processing/ProcessingChains.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/processing/ProcessingChains.java @@ -2,6 +2,8 @@ package com.yahoo.vespa.model.container.processing; import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.model.container.component.BindingPattern; +import com.yahoo.vespa.model.container.component.SystemBindingPattern; import com.yahoo.vespa.model.container.component.chain.Chains; /** @@ -11,7 +13,7 @@ import com.yahoo.vespa.model.container.component.chain.Chains; */ public class ProcessingChains extends Chains<ProcessingChain> { - public static final String[] defaultBindings = new String[] {"http://*/processing/*"}; + public static final BindingPattern[] defaultBindings = new BindingPattern[]{SystemBindingPattern.fromHttpPath("/processing/*")}; public ProcessingChains(AbstractConfigProducer parent, String subId) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/GUIHandler.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/GUIHandler.java index 1e717f89819..f01bbcd3951 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/GUIHandler.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/GUIHandler.java @@ -14,7 +14,7 @@ public class GUIHandler extends Handler<AbstractConfigProducer<?>> { public static final String BUNDLE = "container-search-gui"; public static final String CLASS = "com.yahoo.search.query.gui.GUIHandler"; - public static final String BINDING = "*/querybuilder/*"; + public static final String BINDING_PATH = "/querybuilder/*"; public GUIHandler() { super(new ComponentModel(bundleSpec(CLASS, BUNDLE))); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 41e092c7ea5..51583588201 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -29,7 +29,6 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; import com.yahoo.search.rendering.RendererRegistry; import com.yahoo.searchdefinition.derived.RankProfileList; @@ -57,10 +56,11 @@ import com.yahoo.vespa.model.container.ContainerModel; import com.yahoo.vespa.model.container.ContainerModelEvaluation; import com.yahoo.vespa.model.container.IdentityProvider; import com.yahoo.vespa.model.container.SecretStore; -import com.yahoo.vespa.model.container.component.Component; +import com.yahoo.vespa.model.container.component.BindingPattern; import com.yahoo.vespa.model.container.component.FileStatusHandlerComponent; import com.yahoo.vespa.model.container.component.Handler; -import com.yahoo.vespa.model.container.component.chain.Chain; +import com.yahoo.vespa.model.container.component.SystemBindingPattern; +import com.yahoo.vespa.model.container.component.UserBindingPattern; import com.yahoo.vespa.model.container.component.chain.ProcessingHandler; import com.yahoo.vespa.model.container.docproc.ContainerDocproc; import com.yahoo.vespa.model.container.docproc.DocprocChains; @@ -93,7 +93,6 @@ import java.util.function.Consumer; import java.util.regex.Pattern; import java.util.stream.Collectors; -import static com.yahoo.vespa.model.container.http.AccessControl.ACCESS_CONTROL_CHAIN_ID; import static java.util.logging.Level.WARNING; /** @@ -113,7 +112,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { private static final String ENVIRONMENT_VARIABLES_ELEMENT = "environment-variables"; static final String SEARCH_HANDLER_CLASS = com.yahoo.search.handler.SearchHandler.class.getName(); - static final String SEARCH_HANDLER_BINDING = "http://*/search/*"; + static final BindingPattern SEARCH_HANDLER_BINDING = SystemBindingPattern.fromHttpPath("/search/*"); public enum Networking { disable, enable } @@ -278,8 +277,10 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { String name = "status.html"; Optional<String> statusFile = Optional.ofNullable(System.getenv(HOSTED_VESPA_STATUS_FILE_SETTING)); cluster.addComponent( - new FileStatusHandlerComponent(name + "-status-handler", statusFile.orElse(HOSTED_VESPA_STATUS_FILE), - "http://*/" + name)); + new FileStatusHandlerComponent( + name + "-status-handler", + statusFile.orElse(HOSTED_VESPA_STATUS_FILE), + SystemBindingPattern.fromHttpPath("/" + name))); } else { cluster.addVipHandler(); } @@ -368,15 +369,12 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { if (http.getAccessControl().isPresent()) return; // access control added explicitly AthenzDomain tenantDomain = deployState.getProperties().athenzDomain().orElse(null); if (tenantDomain == null) return; // tenant domain not present, cannot add access control. this should eventually be a failure. - AccessControl accessControl = - new AccessControl.Builder(tenantDomain.value(), deployState.getDeployLogger()) - .setHandlers(cluster) - .readEnabled(false) - .writeEnabled(false) - .build(); - http.getFilterChains().add(new Chain<>(FilterChains.emptyChainSpec(ACCESS_CONTROL_CHAIN_ID))); - http.setAccessControl(accessControl); - http.getBindings().addAll(accessControl.getBindings()); + new AccessControl.Builder(tenantDomain.value()) + .setHandlers(cluster) + .readEnabled(false) + .writeEnabled(false) + .build() + .configureHttpFilterChains(http); } private Http buildHttp(DeployState deployState, ApplicationContainerCluster cluster, Element httpElement) { @@ -795,8 +793,8 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { ProcessingHandler<SearchChains> searchHandler = new ProcessingHandler<>(cluster.getSearch().getChains(), "com.yahoo.search.handler.SearchHandler"); - String[] defaultBindings = {SEARCH_HANDLER_BINDING}; - for (String binding: serverBindings(searchElement, defaultBindings)) { + BindingPattern[] defaultBindings = {SEARCH_HANDLER_BINDING}; + for (BindingPattern binding: serverBindings(searchElement, defaultBindings)) { searchHandler.addServerBindings(binding); } @@ -805,12 +803,12 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { private void addGUIHandler(ApplicationContainerCluster cluster) { Handler<?> guiHandler = new GUIHandler(); - guiHandler.addServerBindings("http://"+GUIHandler.BINDING); + guiHandler.addServerBindings(SystemBindingPattern.fromHttpPath(GUIHandler.BINDING_PATH)); cluster.addComponent(guiHandler); } - private String[] serverBindings(Element searchElement, String... defaultBindings) { + private BindingPattern[] serverBindings(Element searchElement, BindingPattern... defaultBindings) { List<Element> bindings = XML.getChildren(searchElement, "binding"); if (bindings.isEmpty()) return defaultBindings; @@ -818,16 +816,16 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { return toBindingList(bindings); } - private String[] toBindingList(List<Element> bindingElements) { - List<String> result = new ArrayList<>(); + private BindingPattern[] toBindingList(List<Element> bindingElements) { + List<BindingPattern> result = new ArrayList<>(); for (Element element: bindingElements) { String text = element.getTextContent().trim(); if (!text.isEmpty()) - result.add(text); + result.add(UserBindingPattern.fromPattern(text)); } - return result.toArray(new String[result.size()]); + return result.toArray(BindingPattern[]::new); } private ContainerDocumentApi buildDocumentApi(ApplicationContainerCluster cluster, Element spec) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocumentApiOptionsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocumentApiOptionsBuilder.java index ae74dbdb4a7..61464799812 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocumentApiOptionsBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocumentApiOptionsBuilder.java @@ -6,19 +6,17 @@ import com.yahoo.vespa.model.clients.ContainerDocumentApi; import org.w3c.dom.Element; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.List; import java.util.logging.Logger; /** * @author Einar M R Rosenvinge - * @since 5.1.11 */ public class DocumentApiOptionsBuilder { private static final Logger log = Logger.getLogger(DocumentApiOptionsBuilder.class.getName()); - private static final String[] DEFAULT_BINDINGS = {"http://*/"}; + public static ContainerDocumentApi.Options build(Element spec) { return new ContainerDocumentApi.Options(getBindings(spec)); @@ -27,8 +25,7 @@ public class DocumentApiOptionsBuilder { private static List<String> getBindings(Element spec) { Collection<Element> bindingElems = XML.getChildren(spec, "binding"); if (bindingElems.isEmpty()) - return Arrays.asList(DEFAULT_BINDINGS); - + return List.of(); List<String> bindings = new ArrayList<>(); for (Element e :bindingElems) { String binding = getBinding(e); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/NearestNeighborTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/NearestNeighborTestCase.java index 9f57b22fd58..f6374677261 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/NearestNeighborTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/NearestNeighborTestCase.java @@ -2,7 +2,6 @@ package com.yahoo.searchdefinition.derived; import com.yahoo.component.ComponentId; -import com.yahoo.prelude.query.QueryException; import com.yahoo.search.Query; import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; import com.yahoo.search.query.profile.config.QueryProfileConfigurer; @@ -28,9 +27,9 @@ public class NearestNeighborTestCase extends AbstractExportingTestCase { Query q = new Query("?ranking.features.query(q_vec)=[1,2,3,4,5,6]", // length is 6, not 5 queryProfiles.getComponent("default")); fail("This should fail when q_vec is parsed as a tensor"); - } catch (QueryException e) { + } catch (IllegalArgumentException e) { // success - assertEquals("Invalid request parameter", e.getMessage()); + assertEquals("Could not set 'ranking.features.query(q_vec)' to '[1,2,3,4,5,6]'", e.getMessage()); } catch (RuntimeException e) { e.printStackTrace(); throw e; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/UriBindingsValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/UriBindingsValidatorTest.java new file mode 100644 index 00000000000..cce88bc02f9 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/UriBindingsValidatorTest.java @@ -0,0 +1,104 @@ +package com.yahoo.vespa.model.application.validation;// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.NullConfigModelRegistry; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.vespa.model.VespaModel; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.xml.sax.SAXException; + +import java.io.IOException; + +/** + * @author bjorncs + */ +public class UriBindingsValidatorTest { + + @Rule + public ExpectedException exceptionRule = ExpectedException.none(); + + @Test + public void fails_on_user_handler_binding_with_port() throws IOException, SAXException { + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule.expectMessage("For binding 'http://*:4443/my-handler': binding with port is not allowed"); + runUriBindingValidator(true, createServicesXmlWithHandler("http://*:4443/my-handler")); + } + + @Test + public void fails_on_user_handler_binding_with_hostname() throws IOException, SAXException { + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule.expectMessage("For binding 'http://myhostname/my-handler': only binding with wildcard ('*') for hostname is allowed"); + runUriBindingValidator(true, createServicesXmlWithHandler("http://myhostname/my-handler")); + } + + @Test + public void fails_on_user_handler_binding_with_non_http_scheme() throws IOException, SAXException { + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule.expectMessage("For binding 'ftp://*/my-handler': only 'http' is allowed as scheme"); + runUriBindingValidator(true, createServicesXmlWithHandler("ftp://*/my-handler")); + } + + @Test + public void fails_on_invalid_filter_binding() throws IOException, SAXException { + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule.expectMessage("For binding 'https://*:4443/my-request-filer-chain': binding with port is not allowed"); + runUriBindingValidator(true, createServicesXmlWithRequestFilterChain("https://*:4443/my-request-filer-chain")); + } + + @Test + public void allows_valid_user_binding() throws IOException, SAXException { + runUriBindingValidator(true, createServicesXmlWithHandler("http://*/my-handler")); + } + + @Test + public void only_restricts_user_bindings_on_hosted() throws IOException, SAXException { + runUriBindingValidator(false, createServicesXmlWithRequestFilterChain("https://*:4443/my-request-filer-chain")); + } + + private void runUriBindingValidator(boolean isHosted, String servicesXml) throws IOException, SAXException { + ApplicationPackage app = new MockApplicationPackage.Builder() + .withServices(servicesXml) + .build(); + DeployState deployState = new DeployState.Builder() + .applicationPackage(app) + .properties(new TestProperties().setHostedVespa(isHosted)) + .build(); + VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); + new UriBindingsValidator().validate(model, deployState); + } + + private static String createServicesXmlWithHandler(String handlerBinding) { + return String.join( + "\n", + "<services version='1.0'>", + " <container id='default' version='1.0'>", + " <handler id='custom.Handler'>", + " <binding>" + handlerBinding + "</binding>", + " </handler>", + " </container>", + "</services>"); + } + + private static String createServicesXmlWithRequestFilterChain(String filterBinding) { + return String.join( + "\n", + "<services version='1.0'>", + " <container version='1.0'>", + " <http>", + " <server port='8080' id='main' />", + " <filtering>", + " <request-chain id='myChain'>", + " <filter id='myFilter'/>", + " <binding>" + filterBinding + "</binding>", + " </request-chain>", + " </filtering>", + " </http>", + " </container>", + "</services>"); + } + +}
\ No newline at end of file diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/component/BindingPatternTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/component/BindingPatternTest.java new file mode 100644 index 00000000000..91a2b65c0e0 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/component/BindingPatternTest.java @@ -0,0 +1,53 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.component; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +/** + * @author bjorncs + */ +public class BindingPatternTest { + + @Test + public void parses_valid_bindings_correctly() { + assertBindingParses("http://host:1234/path"); + assertBindingParses("http://host/path"); + assertBindingParses("http://host/"); + assertBindingParses("*://*:*/*"); + assertBindingParses("http://*/*"); + assertBindingParses("https://*/my/path"); + assertBindingParses("https://*/path/*"); + assertBindingParses("https://host:*/path/*"); + assertBindingParses("https://host:1234/*"); + } + + @Test + public void getters_returns_correct_components() { + { + BindingPattern pattern = SystemBindingPattern.fromPattern("http://host:1234/path/*"); + assertEquals("http", pattern.scheme()); + assertEquals("host", pattern.host()); + assertEquals("1234", pattern.port().get()); + assertEquals("/path/*", pattern.path()); + } + { + BindingPattern pattern = SystemBindingPattern.fromPattern("https://*/path/v1/"); + assertEquals("https", pattern.scheme()); + assertEquals("*", pattern.host()); + assertFalse(pattern.port().isPresent()); + assertEquals("/path/v1/", pattern.path()); + } + } + + private static void assertBindingParses(String binding) { + BindingPattern pattern = SystemBindingPattern.fromPattern(binding); + String stringRepresentation = pattern.patternString(); + assertEquals( + "Expected string representation of parsed binding to match original binding string", + binding, stringRepresentation); + } + +}
\ No newline at end of file diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/http/FilterBindingsTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/http/FilterBindingsTest.java index 0f9de516a4b..5b0c13a4038 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/http/FilterBindingsTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/http/FilterBindingsTest.java @@ -1,10 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container.http; -import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.builder.xml.test.DomBuilderTest; +import com.yahoo.config.model.deploy.DeployState; import com.yahoo.jdisc.http.ServerConfig; import com.yahoo.vespa.model.container.ContainerModel; +import com.yahoo.vespa.model.container.component.BindingPattern; +import com.yahoo.vespa.model.container.component.UserBindingPattern; import com.yahoo.vespa.model.container.component.chain.Chain; import com.yahoo.vespa.model.container.http.xml.HttpBuilder; import com.yahoo.vespa.model.container.xml.ContainerModelBuilder; @@ -21,7 +23,7 @@ import static org.junit.Assert.assertNotNull; */ public class FilterBindingsTest extends DomBuilderTest { - private static final String MY_CHAIN_BINDING = "http://*/my-chain-binding"; + private static final BindingPattern MY_CHAIN_BINDING = UserBindingPattern.fromHttpPath("/my-chain-binding"); private Http buildHttp(Element xml) { Http http = new HttpBuilder().build(root.getDeployState(), root, xml); @@ -42,14 +44,14 @@ public class FilterBindingsTest extends DomBuilderTest { "<http>", " <filtering>", " <request-chain id='my-request-chain'>", - " <binding>" + MY_CHAIN_BINDING + "</binding>", + " <binding>" + MY_CHAIN_BINDING.patternString() + "</binding>", " </request-chain>", " </filtering>", "</http>"); Http http = buildHttp(xml); - Binding binding = first(http.getBindings()); - assertEquals("my-request-chain", binding.filterId().getName()); + FilterBinding binding = first(http.getBindings()); + assertEquals("my-request-chain", binding.chainId().getName()); assertEquals(MY_CHAIN_BINDING, binding.binding()); Chain<Filter> myChain = http.getFilterChains().allChains().getComponent("my-request-chain"); @@ -62,14 +64,14 @@ public class FilterBindingsTest extends DomBuilderTest { "<http>", " <filtering>", " <response-chain id='my-response-chain'>", - " <binding>" + MY_CHAIN_BINDING + "</binding>", + " <binding>" + MY_CHAIN_BINDING.patternString() + "</binding>", " </response-chain>", " </filtering>", "</http>"); Http http = buildHttp(xml); - Binding binding = first(http.getBindings()); - assertEquals("my-response-chain", binding.filterId().getName()); + FilterBinding binding = first(http.getBindings()); + assertEquals("my-response-chain", binding.chainId().getName()); assertEquals(MY_CHAIN_BINDING, binding.binding()); Chain<Filter> myChain = http.getFilterChains().allChains().getComponent("my-response-chain"); @@ -83,7 +85,7 @@ public class FilterBindingsTest extends DomBuilderTest { " <http>", " <filtering>", " <request-chain id='my-request-chain'>", - " <binding>" + MY_CHAIN_BINDING + "</binding>", + " <binding>" + MY_CHAIN_BINDING.patternString() + "</binding>", " </request-chain>", " </filtering>", " <server id='server1' port='8000' />", @@ -96,13 +98,13 @@ public class FilterBindingsTest extends DomBuilderTest { final ServerConfig config = root.getConfig(ServerConfig.class, "container/http/jdisc-jetty/server1"); assertEquals(1, config.filter().size()); assertEquals("my-request-chain", config.filter(0).id()); - assertEquals(MY_CHAIN_BINDING, config.filter(0).binding()); + assertEquals(MY_CHAIN_BINDING.patternString(), config.filter(0).binding()); } { final ServerConfig config = root.getConfig(ServerConfig.class, "container/http/jdisc-jetty/server2"); assertEquals(1, config.filter().size()); assertEquals("my-request-chain", config.filter(0).id()); - assertEquals(MY_CHAIN_BINDING, config.filter(0).binding()); + assertEquals(MY_CHAIN_BINDING.patternString(), config.filter(0).binding()); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessControlTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessControlTest.java index 28e23ce3222..4c3a1084005 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessControlTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessControlTest.java @@ -1,271 +1,182 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container.xml; -import com.google.common.collect.ImmutableSet; -import com.yahoo.collections.CollectionUtil; import com.yahoo.component.ComponentId; import com.yahoo.config.model.builder.xml.test.DomBuilderTest; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.config.provision.AthenzDomain; -import com.yahoo.container.jdisc.state.StateHandler; import com.yahoo.vespa.model.container.ApplicationContainer; -import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.http.AccessControl; +import com.yahoo.vespa.model.container.http.FilterChains; import com.yahoo.vespa.model.container.http.Http; -import com.yahoo.vespa.model.container.http.Binding; -import com.yahoo.vespa.model.container.http.xml.HttpBuilder; -import com.yahoo.vespa.model.container.jersey.Jersey2Servlet; import org.junit.Test; -import org.w3c.dom.Element; -import java.util.Collection; -import java.util.HashSet; +import java.util.ArrayList; +import java.util.List; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; -import static com.yahoo.config.model.test.TestUtil.joinLines; import static com.yahoo.vespa.defaults.Defaults.getDefaults; +import static org.hamcrest.CoreMatchers.hasItem; import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasItems; +import static org.hamcrest.Matchers.not; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotEquals; -import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; /** * @author gjoranv + * @author bjorncs */ public class AccessControlTest extends ContainerModelBuilderTestBase { - private static final Set<String> REQUIRED_HANDLER_BINDINGS = ImmutableSet.of( - "/custom-handler/", - "/search/", - "/document/", - ContainerCluster.RESERVED_URI_PREFIX); - - private static final Set<String> FORBIDDEN_HANDLER_BINDINGS = ImmutableSet.of( - "/ApplicationStatus", - "/status.html", - "/statistics/", - StateHandler.STATE_API_ROOT, - ContainerCluster.ROOT_HANDLER_PATH); - @Test - public void access_control_filter_chain_is_set_up() { - Element clusterElem = DomBuilderTest.parse( + public void access_control_filter_chains_are_set_up() { + Http http = createModelAndGetHttp( " <http>", " <filtering>", - " <access-control domain='foo' />", + " <access-control domain='my-tenant-domain' />", " </filtering>", " </http>"); - Http http = new HttpBuilder().build(root.getDeployState(), root, clusterElem); - root.freezeModelTopology(); - - assertTrue(http.getFilterChains().hasChain(AccessControl.ACCESS_CONTROL_CHAIN_ID)); + FilterChains filterChains = http.getFilterChains(); + assertTrue(filterChains.hasChain(AccessControl.ACCESS_CONTROL_CHAIN_ID)); + assertTrue(filterChains.hasChain(AccessControl.ACCESS_CONTROL_EXCLUDED_CHAIN_ID)); } @Test public void properties_are_set_from_xml() { - Element clusterElem = DomBuilderTest.parse( + Http http = createModelAndGetHttp( " <http>", " <filtering>", - " <access-control domain='my-domain'/>", + " <access-control domain='my-tenant-domain'/>", " </filtering>", " </http>"); - Http http = new HttpBuilder().build(root.getDeployState(), root, clusterElem); - root.freezeModelTopology(); AccessControl accessControl = http.getAccessControl().get(); - assertEquals("Wrong domain.", "my-domain", accessControl.domain); + assertEquals("Wrong domain.", "my-tenant-domain", accessControl.domain); } @Test public void read_is_disabled_and_write_is_enabled_by_default() { - Element clusterElem = DomBuilderTest.parse( + Http http = createModelAndGetHttp( " <http>", " <filtering>", - " <access-control domain='foo' />", + " <access-control domain='my-tenant-domain'/>", " </filtering>", " </http>"); - Http http = new HttpBuilder().build(root.getDeployState(), root, clusterElem); - root.freezeModelTopology(); - assertFalse("Wrong default value for read.", http.getAccessControl().get().readEnabled); assertTrue("Wrong default value for write.", http.getAccessControl().get().writeEnabled); } @Test public void read_and_write_can_be_overridden() { - Element clusterElem = DomBuilderTest.parse( + Http http = createModelAndGetHttp( " <http>", " <filtering>", - " <access-control domain='foo' read='true' write='false'/>", + " <access-control domain='my-tenant-domain' read='true' write='false'/>", " </filtering>", " </http>"); - Http http = new HttpBuilder().build(root.getDeployState(), root, clusterElem); - root.freezeModelTopology(); - assertTrue("Given read value not honoured.", http.getAccessControl().get().readEnabled); assertFalse("Given write value not honoured.", http.getAccessControl().get().writeEnabled); } @Test - public void access_control_filter_chain_has_correct_handler_bindings() { - Element clusterElem = DomBuilderTest.parse( - "<container version='1.0'>", - " <search/>", - " <document-api/>", - " <handler id='custom.Handler'>", - " <binding>http://*/custom-handler/*</binding>", - " </handler>", + public void access_control_excluded_filter_chain_has_all_bindings_from_excluded_handlers() { + Http http = createModelAndGetHttp( " <http>", " <filtering>", - " <access-control domain='foo' />", + " <access-control/>", " </filtering>", - " </http>", - "</container>"); - - Http http = getHttp(clusterElem); - - Set<String> foundRequiredBindings = REQUIRED_HANDLER_BINDINGS.stream() - .filter(requiredBinding -> containsBinding(http.getBindings(), requiredBinding)) - .collect(Collectors.toSet()); - Set<String> missingRequiredBindings = new HashSet<>(REQUIRED_HANDLER_BINDINGS); - missingRequiredBindings.removeAll(foundRequiredBindings); - assertTrue("Access control chain was not bound to: " + CollectionUtil.mkString(missingRequiredBindings, ", "), - missingRequiredBindings.isEmpty()); - - FORBIDDEN_HANDLER_BINDINGS.forEach(forbiddenPath -> { - String forbiddenBinding = String.format("http://*%s", forbiddenPath); - http.getBindings().forEach( - binding -> assertNotEquals("Access control chain was bound to: " + binding.binding(), binding.binding(), forbiddenBinding)); - }); - } - - @Test - public void handler_can_be_excluded_by_excluding_one_of_its_bindings() { - final String notExcludedBinding = "http://*/custom-handler/*"; - final String excludedBinding = "http://*/excluded/*"; - Element clusterElem = DomBuilderTest.parse( - "<container version='1.0'>", - httpWithExcludedBinding(excludedBinding), - " <handler id='custom.Handler'>", - " <binding>" + notExcludedBinding + "</binding>", - " <binding>" + excludedBinding + "</binding>", - " </handler>", - "</container>"); - - Http http = getHttp(clusterElem); - assertFalse("Excluded binding was not removed.", - containsBinding(http.getBindings(), excludedBinding)); - assertFalse("Not all bindings of an excluded handler were removed.", - containsBinding(http.getBindings(), notExcludedBinding)); + " </http>"); + Set<String> actualBindings = getFilterBindings(http, AccessControl.ACCESS_CONTROL_EXCLUDED_CHAIN_ID); + assertThat(actualBindings, containsInAnyOrder( + "http://*:4443/ApplicationStatus", + "http://*:4443/status.html", + "http://*:4443/state/v1", + "http://*:4443/state/v1/*", + "http://*:4443/prometheus/v1", + "http://*:4443/prometheus/v1/*", + "http://*:4443/metrics/v2", + "http://*:4443/metrics/v2/*", + "http://*:4443/")); } @Test - public void access_control_filter_chain_has_all_servlet_bindings() { - final String servletPath = "servlet/path"; - final String restApiPath = "api/v0"; - final Set<String> requiredBindings = ImmutableSet.of(servletPath, restApiPath); - Element clusterElem = DomBuilderTest.parse( - "<container version='1.0'>", - " <servlet id='foo' class='bar' bundle='baz'>", - " <path>" + servletPath + "</path>", - " </servlet>", - " <rest-api jersey2='true' path='" + restApiPath + "' />", + public void access_control_excluded_chain_does_not_contain_any_bindings_from_access_control_chain() { + Http http = createModelAndGetHttp( " <http>", " <filtering>", - " <access-control domain='foo' />", + " <access-control/>", " </filtering>", - " </http>", - "</container>"); - - Http http = getHttp(clusterElem); + " </http>"); - Set<String> missingRequiredBindings = requiredBindings.stream() - .filter(requiredBinding -> ! containsBinding(http.getBindings(), requiredBinding)) - .collect(Collectors.toSet()); + Set<String> bindings = getFilterBindings(http, AccessControl.ACCESS_CONTROL_CHAIN_ID); + Set<String> excludedBindings = getFilterBindings(http, AccessControl.ACCESS_CONTROL_EXCLUDED_CHAIN_ID); - assertTrue("Access control chain was not bound to: " + CollectionUtil.mkString(missingRequiredBindings, ", "), - missingRequiredBindings.isEmpty()); + for (String binding : bindings) { + assertThat(excludedBindings, not(hasItem(binding))); + } } - @Test - public void servlet_can_be_excluded_by_excluding_one_of_its_bindings() { - final String servletPath = "servlet/path"; - final String notExcludedBinding = "http://*:8081/" + servletPath; - final String excludedBinding = "http://*:8080/" + servletPath; - Element clusterElem = DomBuilderTest.parse( - "<container version='1.0'>", - httpWithExcludedBinding(excludedBinding), - " <servlet id='foo' class='bar' bundle='baz'>", - " <path>" + servletPath + "</path>", - " </servlet>", - "</container>"); - Http http = getHttp(clusterElem); - assertFalse("Excluded binding was not removed.", - containsBinding(http.getBindings(), excludedBinding)); - assertFalse("Not all bindings of an excluded servlet were removed.", - containsBinding(http.getBindings(), notExcludedBinding)); + @Test + public void access_control_excluded_filter_chain_has_user_provided_excluded_bindings() { + Http http = createModelAndGetHttp( + " <http>", + " <handler id='custom.Handler'>", + " <binding>http://*/custom-handler/*</binding>", + " </handler>", + " <filtering>", + " <access-control>", + " <exclude>", + " <binding>http://*/custom-handler/*</binding>", + " <binding>http://*/search/*</binding>", + " </exclude>", + " </access-control>", + " </filtering>", + " </http>"); + Set<String> actualBindings = getFilterBindings(http, AccessControl.ACCESS_CONTROL_EXCLUDED_CHAIN_ID); + assertThat(actualBindings, hasItems("http://*:4443/custom-handler/*", "http://*:4443/search/*", "http://*:4443/status.html")); } @Test - public void rest_api_can_be_excluded_by_excluding_one_of_its_bindings() { - final String restApiPath = "api/v0"; - final String notExcludedBinding = "http://*:8081/" + restApiPath + Jersey2Servlet.BINDING_SUFFIX;; - final String excludedBinding = "http://*:8080/" + restApiPath + Jersey2Servlet.BINDING_SUFFIX;; - Element clusterElem = DomBuilderTest.parse( - "<container version='1.0'>", - httpWithExcludedBinding(excludedBinding), - " <rest-api jersey2='true' path='" + restApiPath + "' />", - "</container>"); - - Http http = getHttp(clusterElem); - assertFalse("Excluded binding was not removed.", - containsBinding(http.getBindings(), excludedBinding)); - assertFalse("Not all bindings of an excluded rest-api were removed.", - containsBinding(http.getBindings(), notExcludedBinding)); - + public void access_control_filter_chain_contains_catchall_bindings() { + Http http = createModelAndGetHttp( + " <http>", + " <filtering>", + " <access-control/>", + " </filtering>", + " </http>"); + Set<String> actualBindings = getFilterBindings(http, AccessControl.ACCESS_CONTROL_CHAIN_ID); + assertThat(actualBindings, containsInAnyOrder("http://*:4443/*")); } - @Test public void access_control_is_implicitly_added_for_hosted_apps() { - Element clusterElem = DomBuilderTest.parse( - "<container version='1.0'>", - nodesXml, - "</container>" ); - AthenzDomain tenantDomain = AthenzDomain.from("my-tenant-domain"); - DeployState state = new DeployState.Builder().properties( - new TestProperties() - .setAthenzDomain(tenantDomain) - .setHostedVespa(true)) - .build(); - createModel(root, state, null, clusterElem); - Optional<AccessControl> maybeAccessControl = - ((ApplicationContainer) root.getProducer("container/container.0")).getHttp().getAccessControl(); + Http http = createModelAndGetHttp("<container version='1.0'/>"); + Optional<AccessControl> maybeAccessControl = http.getAccessControl(); assertThat(maybeAccessControl.isPresent(), is(true)); AccessControl accessControl = maybeAccessControl.get(); assertThat(accessControl.writeEnabled, is(false)); assertThat(accessControl.readEnabled, is(false)); - assertThat(accessControl.domain, equalTo(tenantDomain.value())); + assertThat(accessControl.domain, equalTo("my-tenant-domain")); } @Test public void access_control_is_implicitly_added_for_hosted_apps_with_existing_http_element() { - Element clusterElem = DomBuilderTest.parse( - "<container version='1.0'>", + Http http = createModelAndGetHttp( " <http>", " <server port='" + getDefaults().vespaWebServicePort() + "' id='main' />", " <filtering>", @@ -274,49 +185,33 @@ public class AccessControlTest extends ContainerModelBuilderTestBase { " <filter id='inner' />", " </request-chain>", " </filtering>", - " </http>", - nodesXml, - "</container>" ); - AthenzDomain tenantDomain = AthenzDomain.from("my-tenant-domain"); - DeployState state = new DeployState.Builder().properties( - new TestProperties() - .setAthenzDomain(tenantDomain) - .setHostedVespa(true)) - .build(); - createModel(root, state, null, clusterElem); - Http http = ((ApplicationContainer) root.getProducer("container/container.0")).getHttp(); + " </http>"); assertThat(http.getAccessControl().isPresent(), is(true)); assertThat(http.getFilterChains().hasChain(AccessControl.ACCESS_CONTROL_CHAIN_ID), is(true)); assertThat(http.getFilterChains().hasChain(ComponentId.fromString("myChain")), is(true)); } + private Http createModelAndGetHttp(String... httpElement) { + List<String> servicesXml = new ArrayList<>(); + servicesXml.add("<container version='1.0'>"); + servicesXml.addAll(List.of(httpElement)); + servicesXml.add("</container>"); - private String httpWithExcludedBinding(String excludedBinding) { - return joinLines( - " <http>", - " <filtering>", - " <access-control domain='foo'>", - " <exclude>", - " <binding>" + excludedBinding + "</binding>", - " </exclude>", - " </access-control>", - " </filtering>", - " </http>"); + AthenzDomain tenantDomain = AthenzDomain.from("my-tenant-domain"); + DeployState state = new DeployState.Builder().properties( + new TestProperties() + .setAthenzDomain(tenantDomain) + .setHostedVespa(true)) + .build(); + createModel(root, state, null, DomBuilderTest.parse(servicesXml.toArray(String[]::new))); + return ((ApplicationContainer) root.getProducer("container/container.0")).getHttp(); } - private Http getHttp(Element clusterElem) { - createModel(root, clusterElem); - ContainerCluster cluster = (ContainerCluster) root.getChildren().get("container"); - Http http = cluster.getHttp(); - assertNotNull(http); - return http; + private static Set<String> getFilterBindings(Http http, ComponentId filerChain) { + return http.getBindings().stream() + .filter(binding -> binding.chainId().toId().equals(filerChain)) + .map(binding -> binding.binding().patternString()) + .collect(Collectors.toSet()); } - private boolean containsBinding(Collection<Binding> bindings, String binding) { - for (Binding b : bindings) { - if (b.binding().contains(binding)) - return true; - } - return false; - } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerDocumentApiBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerDocumentApiBuilderTest.java index ac2e1b88c0b..73a68429b6d 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerDocumentApiBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerDocumentApiBuilderTest.java @@ -4,6 +4,8 @@ package com.yahoo.vespa.model.container.xml; import com.yahoo.config.model.builder.xml.test.DomBuilderTest; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.component.Handler; +import com.yahoo.vespa.model.container.component.SystemBindingPattern; +import com.yahoo.vespa.model.container.component.UserBindingPattern; import org.junit.Test; import org.w3c.dom.Element; @@ -40,24 +42,21 @@ public class ContainerDocumentApiBuilderTest extends ContainerModelBuilderTestBa "<container id='cluster1' version='1.0'>", " <document-api>", " <binding>http://*/document-api/</binding>", - " <binding>missing-trailing-slash</binding>", " </document-api>", nodesXml, "</container>"); createModel(root, elem); - verifyCustomBindings("com.yahoo.vespa.http.server.FeedHandler", ContainerCluster.RESERVED_URI_PREFIX + "/feedapi"); + verifyCustomBindings("com.yahoo.vespa.http.server.FeedHandler"); } - private void verifyCustomBindings(String id, String bindingSuffix) { + private void verifyCustomBindings(String id) { Handler<?> handler = getHandlers("cluster1").get(id); - assertThat(handler.getServerBindings(), hasItem("http://*/document-api/" + bindingSuffix)); - assertThat(handler.getServerBindings(), hasItem("http://*/document-api/" + bindingSuffix + "/")); - assertThat(handler.getServerBindings(), hasItem("missing-trailing-slash/" + bindingSuffix)); - assertThat(handler.getServerBindings(), hasItem("missing-trailing-slash/" + bindingSuffix + "/")); + assertThat(handler.getServerBindings(), hasItem(UserBindingPattern.fromHttpPath("/document-api/reserved-for-internal-use/feedapi"))); + assertThat(handler.getServerBindings(), hasItem(UserBindingPattern.fromHttpPath("/document-api/reserved-for-internal-use/feedapi/"))); - assertThat(handler.getServerBindings().size(), is(4)); + assertThat(handler.getServerBindings().size(), is(2)); } @Test @@ -76,8 +75,12 @@ public class ContainerDocumentApiBuilderTest extends ContainerModelBuilderTestBa assertThat(handlerMap.get("com.yahoo.container.jdisc.state.StateHandler"), not(nullValue())); assertThat(handlerMap.get("com.yahoo.vespa.http.server.FeedHandler"), not(nullValue())); - assertThat(handlerMap.get("com.yahoo.vespa.http.server.FeedHandler").getServerBindings().contains("http://*/" + ContainerCluster.RESERVED_URI_PREFIX + "/feedapi"), is(true)); - assertThat(handlerMap.get("com.yahoo.vespa.http.server.FeedHandler").getServerBindings().contains("http://*/" + ContainerCluster.RESERVED_URI_PREFIX + "/feedapi/"), is(true)); + assertThat(handlerMap.get("com.yahoo.vespa.http.server.FeedHandler").getServerBindings() + .contains(SystemBindingPattern.fromHttpPath("/reserved-for-internal-use/feedapi")), + is(true)); + assertThat(handlerMap.get("com.yahoo.vespa.http.server.FeedHandler").getServerBindings() + .contains(SystemBindingPattern.fromHttpPath("/reserved-for-internal-use/feedapi")), + is(true)); assertThat(handlerMap.get("com.yahoo.vespa.http.server.FeedHandler").getServerBindings().size(), equalTo(2)); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java index fdd7ae57f0f..6114449c948 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java @@ -241,7 +241,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { Element clusterElem = DomBuilderTest.parse( "<container id='default' version='1.0'>" + " <handler id='userRootHandler'>" + - " <binding>" + ROOT_HANDLER_BINDING + "</binding>" + + " <binding>" + ROOT_HANDLER_BINDING.patternString() + "</binding>" + " </handler>" + "</container>"); createModel(root, clusterElem); @@ -260,7 +260,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { Element clusterElem = DomBuilderTest.parse( "<container id='default' version='1.0'>" + " <handler id='userHandler'>" + - " <binding>" + STATE_HANDLER_BINDING_1 + "</binding>" + + " <binding>" + STATE_HANDLER_BINDING_1.patternString() + "</binding>" + " </handler>" + "</container>"); try { @@ -277,9 +277,9 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { createClusterWithJDiscHandler(); String discBindingsConfig = root.getConfig(JdiscBindingsConfig.class, "default").toString(); assertThat(discBindingsConfig, containsString("{discHandler}")); - assertThat(discBindingsConfig, containsString(".serverBindings[0] \"binding0\"")); - assertThat(discBindingsConfig, containsString(".serverBindings[1] \"binding1\"")); - assertThat(discBindingsConfig, containsString(".clientBindings[0] \"clientBinding\"")); + assertThat(discBindingsConfig, containsString(".serverBindings[0] \"http://*/binding0\"")); + assertThat(discBindingsConfig, containsString(".serverBindings[1] \"http://*/binding1\"")); + assertThat(discBindingsConfig, containsString(".clientBindings[0] \"http://*/clientBinding\"")); } @Test @@ -292,9 +292,9 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { Element clusterElem = DomBuilderTest.parse( "<container id='default' version='1.0'>", " <handler id='discHandler'>", - " <binding>binding0</binding>", - " <binding>binding1</binding>", - " <clientBinding>clientBinding</clientBinding>", + " <binding>http://*/binding0</binding>", + " <binding>http://*/binding1</binding>", + " <clientBinding>http://*/clientBinding</clientBinding>", " </handler>", "</container>"); @@ -340,16 +340,16 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { Element clusterElem = DomBuilderTest.parse( "<container id='default' version='1.0'>", " <processing>", - " <binding>binding0</binding>", - " <binding>binding1</binding>", + " <binding>http://*/binding0</binding>", + " <binding>http://*/binding1</binding>", " </processing>", "</container>"); createModel(root, clusterElem); String discBindingsConfig = root.getConfig(JdiscBindingsConfig.class, "default").toString(); - assertThat(discBindingsConfig, containsString(".serverBindings[0] \"binding0\"")); - assertThat(discBindingsConfig, containsString(".serverBindings[1] \"binding1\"")); + assertThat(discBindingsConfig, containsString(".serverBindings[0] \"http://*/binding0\"")); + assertThat(discBindingsConfig, containsString(".serverBindings[1] \"http://*/binding1\"")); assertThat(discBindingsConfig, not(containsString("/processing/*"))); } @@ -358,9 +358,9 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { createModelWithClientProvider(); String discBindingsConfig = root.getConfig(JdiscBindingsConfig.class, "default").toString(); assertThat(discBindingsConfig, containsString("{discClient}")); - assertThat(discBindingsConfig, containsString(".clientBindings[0] \"binding0\"")); - assertThat(discBindingsConfig, containsString(".clientBindings[1] \"binding1\"")); - assertThat(discBindingsConfig, containsString(".serverBindings[0] \"serverBinding\"")); + assertThat(discBindingsConfig, containsString(".clientBindings[0] \"http://*/binding0\"")); + assertThat(discBindingsConfig, containsString(".clientBindings[1] \"http://*/binding1\"")); + assertThat(discBindingsConfig, containsString(".serverBindings[0] \"http://*/serverBinding\"")); } @Test @@ -373,9 +373,9 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { Element clusterElem = DomBuilderTest.parse( "<container id='default' version='1.0'>" + " <client id='discClient'>" + - " <binding>binding0</binding>" + - " <binding>binding1</binding>" + - " <serverBinding>serverBinding</serverBinding>" + + " <binding>http://*/binding0</binding>" + + " <binding>http://*/binding1</binding>" + + " <serverBinding>http://*/serverBinding</serverBinding>" + " </client>" + "</container>" ); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/SearchBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/SearchBuilderTest.java index b2f9c805be1..c8564c5a273 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/SearchBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/SearchBuilderTest.java @@ -20,6 +20,8 @@ import static com.yahoo.test.Matchers.hasItemWithMethod; import static com.yahoo.vespa.model.container.search.ContainerSearch.QUERY_PROFILE_REGISTRY_CLASS; import static com.yahoo.vespa.model.container.xml.ContainerModelBuilder.SEARCH_HANDLER_BINDING; import static com.yahoo.vespa.model.container.xml.ContainerModelBuilder.SEARCH_HANDLER_CLASS; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.not; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -48,7 +50,7 @@ public class SearchBuilderTest extends ContainerModelBuilderTestBase { createModel(root, clusterElem); String discBindingsConfig = root.getConfig(JdiscBindingsConfig.class, "default").toString(); - assertTrue(discBindingsConfig.contains(GUIHandler.BINDING)); + assertThat(discBindingsConfig, containsString(GUIHandler.BINDING_PATH)); ApplicationContainerCluster cluster = (ApplicationContainerCluster)root.getChildren().get("default"); @@ -66,8 +68,8 @@ public class SearchBuilderTest extends ContainerModelBuilderTestBase { Element clusterElem = DomBuilderTest.parse( "<container id='default' version='1.0'>", " <search>", - " <binding>binding0</binding>", - " <binding>binding1</binding>", + " <binding>http://*/binding0</binding>", + " <binding>http://*/binding1</binding>", " </search>", nodesXml, "</container>"); @@ -75,9 +77,9 @@ public class SearchBuilderTest extends ContainerModelBuilderTestBase { createModel(root, clusterElem); String discBindingsConfig = root.getConfig(JdiscBindingsConfig.class, "default").toString(); - assertTrue(discBindingsConfig.contains(".serverBindings[0] \"binding0\"")); - assertTrue(discBindingsConfig.contains(".serverBindings[1] \"binding1\"")); - assertFalse(discBindingsConfig.contains("/search/*")); + assertThat(discBindingsConfig, containsString(".serverBindings[0] \"http://*/binding0\"")); + assertThat(discBindingsConfig, containsString(".serverBindings[1] \"http://*/binding1\"")); + assertThat(discBindingsConfig, not(containsString("/search/*"))); } @Test @@ -103,7 +105,7 @@ public class SearchBuilderTest extends ContainerModelBuilderTestBase { "<container id='default' version='1.0'>", " <search />", " <handler id='" + myHandler + "'>", - " <binding>" + SEARCH_HANDLER_BINDING + "</binding>", + " <binding>" + SEARCH_HANDLER_BINDING.patternString() + "</binding>", " </handler>", nodesXml, "</container>"); @@ -111,7 +113,7 @@ public class SearchBuilderTest extends ContainerModelBuilderTestBase { createModel(root, clusterElem); var discBindingsConfig = root.getConfig(JdiscBindingsConfig.class, "default"); - assertEquals(SEARCH_HANDLER_BINDING, discBindingsConfig.handlers(myHandler).serverBindings(0)); + assertEquals(SEARCH_HANDLER_BINDING.patternString(), discBindingsConfig.handlers(myHandler).serverBindings(0)); assertNull(discBindingsConfig.handlers(SEARCH_HANDLER_CLASS)); } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/HostSpec.java b/config-provisioning/src/main/java/com/yahoo/config/provision/HostSpec.java index fa684ff6046..6bc3809bd54 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/HostSpec.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/HostSpec.java @@ -45,15 +45,16 @@ public class HostSpec implements Comparable<HostSpec> { /** Create a host in a hosted system */ public HostSpec(String hostname, NodeResources realResources, - NodeResources advertisedResurces, + NodeResources advertisedResources, NodeResources requestedResources, ClusterMembership membership, Optional<Version> version, Optional<NetworkPorts> networkPorts, Optional<DockerImage> dockerImageRepo) { - this(hostname, List.of(), + this(hostname, + List.of(), realResources, - advertisedResurces, + advertisedResources, requestedResources, Optional.of(membership), version, diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializer.java b/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializer.java index 96b189cefe6..e4913643e8d 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializer.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializer.java @@ -6,7 +6,6 @@ import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostSpec; -import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeResources; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Cursor; @@ -16,7 +15,6 @@ import com.yahoo.slime.SlimeUtils; import java.io.IOException; import java.util.ArrayList; -import java.util.Collections; import java.util.LinkedHashSet; import java.util.List; import java.util.Optional; @@ -93,7 +91,7 @@ public class AllocatedHostsSerializer { object.setString(hostSpecDockerImageRepoKey, repo.repository()); }); }); - host.flavor().ifPresent(flavor -> toSlime(flavor, object)); // TODO: Remove this line after June 2020 + host.flavor().ifPresent(flavor -> toSlime(flavor, object)); // TODO: Remove this line when 7.272 has been released toSlime(host.realResources(), object.setObject(realResourcesKey)); toSlime(host.advertisedResources(), object.setObject(advertisedResourcesKey)); host.requestedResources().ifPresent(resources -> toSlime(resources, object.setObject(requestedResourcesKey))); @@ -124,25 +122,26 @@ public class AllocatedHostsSerializer { resourcesObject.setString(storageTypeKey, storageTypeToString(resources.storageType())); } - public static AllocatedHosts fromJson(byte[] json, Optional<NodeFlavors> nodeFlavors) { - return fromSlime(SlimeUtils.jsonToSlime(json).get(), nodeFlavors); + public static AllocatedHosts fromJson(byte[] json) { + return fromSlime(SlimeUtils.jsonToSlime(json).get()); } - public static AllocatedHosts fromSlime(Inspector inspector, Optional<NodeFlavors> nodeFlavors) { + public static AllocatedHosts fromSlime(Inspector inspector) { Inspector array = inspector.field(mappingKey); Set<HostSpec> hosts = new LinkedHashSet<>(); array.traverse((ArrayTraverser)(i, host) -> { - hosts.add(hostFromSlime(host.field(hostSpecKey), nodeFlavors)); + hosts.add(hostFromSlime(host.field(hostSpecKey))); }); return AllocatedHosts.withHosts(hosts); } - private static HostSpec hostFromSlime(Inspector object, Optional<NodeFlavors> nodeFlavors) { + private static HostSpec hostFromSlime(Inspector object) { + if (object.field(hostSpecMembershipKey).valid()) { // Hosted return new HostSpec(object.field(hostSpecHostNameKey).asString(), - nodeResourcesFromSlime(object.field(realResourcesKey), object, nodeFlavors), - nodeResourcesFromSlime(object.field(advertisedResourcesKey), object, nodeFlavors), - optionalNodeResourcesFromSlime(object.field(requestedResourcesKey)), // TODO: Make non-optional after June 2020 + nodeResourcesFromSlime(object.field(realResourcesKey)), + nodeResourcesFromSlime(object.field(advertisedResourcesKey)), + optionalNodeResourcesFromSlime(object.field(requestedResourcesKey)), // TODO: Make non-optional when we serialize NodeResources.unspecified() membershipFromSlime(object), optionalString(object.field(hostSpecCurrentVespaVersionKey)).map(com.yahoo.component.Version::new), NetworkPortsSerializer.fromSlime(object.field(hostSpecNetworkPortsKey)), @@ -162,13 +161,6 @@ public class AllocatedHostsSerializer { return aliases; } - private static Optional<Flavor> flavorFromSlime(Inspector object, Optional<NodeFlavors> nodeFlavors) { - if (object.field(flavorKey).valid() && nodeFlavors.isPresent() && nodeFlavors.get().exists(object.field(flavorKey).asString())) - return nodeFlavors.get().getFlavor(object.field(flavorKey).asString()); - else - return Optional.empty(); - } - private static NodeResources nodeResourcesFromSlime(Inspector resources) { return new NodeResources(resources.field(vcpuKey).asDouble(), resources.field(memoryKey).asDouble(), @@ -183,13 +175,6 @@ public class AllocatedHostsSerializer { return nodeResourcesFromSlime(resources); } - private static NodeResources nodeResourcesFromSlime(Inspector resources, Inspector parent, - Optional<NodeFlavors> nodeFlavors) { - if ( ! resources.valid()) // TODO: Remove the fallback using nodeFlavors after June 2020 - return flavorFromSlime(parent, nodeFlavors).map(f -> f.resources()).orElse(NodeResources.unspecified); - return nodeResourcesFromSlime(resources); - } - private static NodeResources.DiskSpeed diskSpeedFromSlime(Inspector diskSpeed) { switch (diskSpeed.asString()) { case "fast" : return NodeResources.DiskSpeed.fast; diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializerTest.java b/config-provisioning/src/test/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializerTest.java index 30810b79104..700bc389664 100644 --- a/config-provisioning/src/test/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializerTest.java +++ b/config-provisioning/src/test/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializerTest.java @@ -5,12 +5,9 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.DockerImage; -import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NetworkPorts; -import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeResources; -import com.yahoo.config.provisioning.FlavorsConfig; import org.junit.Test; import java.io.IOException; @@ -28,42 +25,45 @@ import static org.junit.Assert.assertEquals; */ public class AllocatedHostsSerializerTest { + private static final NodeResources smallSlowDiskSpeedNode = new NodeResources(0.5, 3.1, 4, 1, NodeResources.DiskSpeed.slow); + private static final NodeResources bigSlowDiskSpeedNode = new NodeResources(1.0, 6.2, 8, 2, NodeResources.DiskSpeed.slow); + private static final NodeResources anyDiskSpeedNode = new NodeResources(0.5, 3.1, 4, 1, NodeResources.DiskSpeed.any); + @Test public void testAllocatedHostsSerialization() throws IOException { - NodeFlavors configuredFlavors = configuredFlavorsFrom("C/12/45/100", 12, 45, 100, 50, Flavor.Type.BARE_METAL); Set<HostSpec> hosts = new LinkedHashSet<>(); hosts.add(new HostSpec("empty", List.of(), Optional.empty())); hosts.add(new HostSpec("with-aliases", List.of("alias1", "alias2"), Optional.empty())); hosts.add(new HostSpec("allocated", - NodeResources.unspecified(), - NodeResources.unspecified(), - NodeResources.unspecified(), + smallSlowDiskSpeedNode, + bigSlowDiskSpeedNode, + anyDiskSpeedNode, ClusterMembership.from("container/test/0/0", Version.fromString("6.73.1"), Optional.of(DockerImage.fromString("docker.foo.com:4443/vespa/bar"))), Optional.empty(), Optional.empty(), Optional.of(DockerImage.fromString("docker.foo.com:4443/vespa/bar")))); hosts.add(new HostSpec("flavor-from-resources-2", - new NodeResources(0.5, 3.1, 4, 1, NodeResources.DiskSpeed.slow), - new NodeResources(1.0, 6.2, 8, 2, NodeResources.DiskSpeed.slow), - new NodeResources(0.5, 3.1, 4, 1, NodeResources.DiskSpeed.any), + smallSlowDiskSpeedNode, + bigSlowDiskSpeedNode, + anyDiskSpeedNode, ClusterMembership.from("container/test/0/0", Version.fromString("6.73.1"), Optional.empty()), Optional.empty(), Optional.empty(), Optional.empty())); hosts.add(new HostSpec("with-version", - NodeResources.unspecified(), - NodeResources.unspecified(), - NodeResources.unspecified(), + smallSlowDiskSpeedNode, + bigSlowDiskSpeedNode, + anyDiskSpeedNode, ClusterMembership.from("container/test/0/0", Version.fromString("6.73.1"), Optional.empty()), Optional.of(Version.fromString("3.4.5")), Optional.empty(), Optional.empty())); hosts.add(new HostSpec("with-ports", - NodeResources.unspecified(), - NodeResources.unspecified(), - NodeResources.unspecified(), + smallSlowDiskSpeedNode, + bigSlowDiskSpeedNode, + anyDiskSpeedNode, ClusterMembership.from("container/test/0/0", Version.fromString("6.73.1"), Optional.empty()), Optional.empty(), @@ -71,11 +71,11 @@ public class AllocatedHostsSerializerTest { new NetworkPorts.Allocation(4567, "service2", "configId2", "suffix2")))), Optional.empty())); - assertAllocatedHosts(AllocatedHosts.withHosts(hosts), configuredFlavors); + assertAllocatedHosts(AllocatedHosts.withHosts(hosts)); } - private void assertAllocatedHosts(AllocatedHosts expectedHosts, NodeFlavors configuredFlavors) throws IOException { - AllocatedHosts deserializedHosts = fromJson(toJson(expectedHosts), Optional.of(configuredFlavors)); + private void assertAllocatedHosts(AllocatedHosts expectedHosts) throws IOException { + AllocatedHosts deserializedHosts = fromJson(toJson(expectedHosts)); assertEquals(expectedHosts, deserializedHosts); for (HostSpec expectedHost : expectedHosts.getHosts()) { @@ -99,17 +99,4 @@ public class AllocatedHostsSerializerTest { throw new IllegalArgumentException("No host " + hostname + " is present"); } - private NodeFlavors configuredFlavorsFrom(String flavorName, double cpu, double mem, double disk, double bandwidth, Flavor.Type type) { - FlavorsConfig.Builder b = new FlavorsConfig.Builder(); - FlavorsConfig.Flavor.Builder flavor = new FlavorsConfig.Flavor.Builder(); - flavor.name(flavorName); - flavor.minDiskAvailableGb(disk); - flavor.minCpuCores(cpu); - flavor.minMainMemoryAvailableGb(mem); - flavor.bandwidth(bandwidth); - flavor.environment(type.name()); - b.flavor(flavor); - return new NodeFlavors(b.build()); - } - } diff --git a/config/src/main/java/com/yahoo/vespa/config/benchmark/LoadTester.java b/config/src/main/java/com/yahoo/vespa/config/benchmark/LoadTester.java index 8a0f24bcd48..3a8d80e5ffe 100644 --- a/config/src/main/java/com/yahoo/vespa/config/benchmark/LoadTester.java +++ b/config/src/main/java/com/yahoo/vespa/config/benchmark/LoadTester.java @@ -43,11 +43,11 @@ import java.util.concurrent.ThreadLocalRandom; public class LoadTester { private static boolean debug = false; - private Transport transport = new Transport("rpc-client"); + private final Transport transport = new Transport("rpc-client"); protected Supervisor supervisor = new Supervisor(transport); private List<ConfigKey<?>> configs = new ArrayList<>(); private Map<ConfigDefinitionKey, Tuple2<String, String[]>> defs = new HashMap<>(); - private CompressionType compressionType = JRTConfigRequestFactory.getCompressionType(); + private final CompressionType compressionType = JRTConfigRequestFactory.getCompressionType(); /** * @param args command-line arguments @@ -149,7 +149,7 @@ public class LoadTester { return ret; } - private class Metrics { + private static class Metrics { long totBytes = 0; long totLatency = 0; @@ -214,9 +214,7 @@ public class LoadTester { Tuple2<String, String[]> defContent = defs.get(dKey); if (defContent == null && defs.size() > 0) { // Only complain if we actually did run with a def dir System.out.println("# No def found for " + dKey + ", not sending in request."); - }/* else { - System.out.println("# FOUND: "+dKey+" : "+ StringUtilities.implode(defContent, "\n")); - }*/ + } request = getRequest(ConfigKey.createFull(reqKey.getName(), reqKey.getConfigId(), reqKey.getNamespace(), defContent.first), defContent.second); if (debug) System.out.println("# Requesting: " + reqKey); long start = System.currentTimeMillis(); @@ -261,7 +259,7 @@ public class LoadTester { if (defContent == null) defContent = new String[0]; final long serverTimeout = 1000; return JRTClientConfigRequestV3.createWithParams(reqKey, DefContent.fromList(Arrays.asList(defContent)), - "unknown", "", 0, serverTimeout, Trace.createDummy(), + ConfigUtils.getCanonicalHostName(), "", 0, serverTimeout, Trace.createDummy(), compressionType, Optional.empty()); } @@ -269,4 +267,5 @@ public class LoadTester { return supervisor.connect(spec); } } + } diff --git a/config_test/pom.xml b/config_test/pom.xml index 0abc4e2728a..0d8b1912c61 100644 --- a/config_test/pom.xml +++ b/config_test/pom.xml @@ -25,7 +25,7 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-compiler-plugin</artifactId> - <version>3.6.1</version> + <version>3.8.1</version> <configuration> <source>1.8</source> <target>1.8</target> diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index b437e961104..eb268546580 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -90,6 +90,7 @@ import java.util.stream.Collectors; import static com.yahoo.config.model.api.container.ContainerServiceType.CLUSTERCONTROLLER_CONTAINER; import static com.yahoo.config.model.api.container.ContainerServiceType.CONTAINER; import static com.yahoo.config.model.api.container.ContainerServiceType.LOGSERVER_CONTAINER; +import static com.yahoo.vespa.curator.Curator.CompletionWaiter; import static com.yahoo.vespa.config.server.filedistribution.FileDistributionUtil.getFileReferencesOnDisk; import static com.yahoo.vespa.config.server.tenant.TenantRepository.HOSTED_VESPA_TENANT; import static com.yahoo.yolean.Exceptions.uncheck; @@ -344,6 +345,54 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye return Deployment.prepared(session, this, hostProvisioner, tenant, timeout, clock, false); } + public Transaction deactivateCurrentActivateNew(Session active, LocalSession prepared, boolean ignoreStaleSessionFailure) { + SessionRepository sessionRepository = tenantRepository.getTenant(prepared.getTenantName()).getSessionRepository(); + Transaction transaction = sessionRepository.createActivateTransaction(prepared); + if (active != null) { + checkIfActiveHasChanged(prepared, active, ignoreStaleSessionFailure); + checkIfActiveIsNewerThanSessionToBeActivated(prepared.getSessionId(), active.getSessionId()); + transaction.add(active.createDeactivateTransaction().operations()); + } + return transaction; + } + + static void checkIfActiveHasChanged(LocalSession session, Session currentActiveSession, boolean ignoreStaleSessionFailure) { + long activeSessionAtCreate = session.getActiveSessionAtCreate(); + log.log(Level.FINE, currentActiveSession.logPre() + "active session id at create time=" + activeSessionAtCreate); + if (activeSessionAtCreate == 0) return; // No active session at create + + long sessionId = session.getSessionId(); + long currentActiveSessionSessionId = currentActiveSession.getSessionId(); + log.log(Level.FINE, currentActiveSession.logPre() + "sessionId=" + sessionId + + ", current active session=" + currentActiveSessionSessionId); + if (currentActiveSession.isNewerThan(activeSessionAtCreate) && + currentActiveSessionSessionId != sessionId) { + String errMsg = currentActiveSession.logPre() + "Cannot activate session " + + sessionId + " because the currently active session (" + + currentActiveSessionSessionId + ") has changed since session " + sessionId + + " was created (was " + activeSessionAtCreate + " at creation time)"; + if (ignoreStaleSessionFailure) { + log.warning(errMsg + " (Continuing because of force.)"); + } else { + throw new ActivationConflictException(errMsg); + } + } + } + + private static boolean isValidSession(Session session) { + return session != null; + } + + // As of now, config generation is based on session id, and config generation must be a monotonically + // increasing number + static void checkIfActiveIsNewerThanSessionToBeActivated(long sessionId, long currentActiveSessionId) { + if (sessionId < currentActiveSessionId) { + throw new ActivationConflictException("It is not possible to activate session " + sessionId + + ", because it is older than current active session (" + + currentActiveSessionId + ")"); + } + } + // ---------------- Application operations ---------------------------------------------------------------- /** @@ -605,6 +654,17 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye // ---------------- Session operations ---------------------------------------------------------------- + + + public CompletionWaiter activate(LocalSession session, Session previousActiveSession, ApplicationId applicationId, boolean ignoreSessionStaleFailure) { + CompletionWaiter waiter = session.getSessionZooKeeperClient().createActiveWaiter(); + NestedTransaction transaction = new NestedTransaction(); + transaction.add(deactivateCurrentActivateNew(previousActiveSession, session, ignoreSessionStaleFailure)); + hostProvisioner.ifPresent(provisioner -> provisioner.activate(transaction, applicationId, session.getAllocatedHosts().getHosts())); + transaction.commit(); + return waiter; + } + /** * Gets the active Session for the given application id. * diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java index 01a95e8ecdd..8c2e6027691 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java @@ -9,9 +9,6 @@ import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.Provisioner; import java.util.logging.Level; -import com.yahoo.transaction.NestedTransaction; -import com.yahoo.transaction.Transaction; -import com.yahoo.vespa.config.server.ActivationConflictException; import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.ApplicationRepository.ActionTimer; import com.yahoo.vespa.config.server.TimeoutBudget; @@ -145,11 +142,7 @@ public class Deployment implements com.yahoo.config.provision.Deployment { CompletionWaiter waiter; try (Lock lock = tenant.getApplicationRepo().lock(applicationId)) { previousActiveSession = applicationRepository.getActiveSession(applicationId); - waiter = session.createActiveWaiter(); - NestedTransaction transaction = new NestedTransaction(); - transaction.add(deactivateCurrentActivateNew(previousActiveSession, session, ignoreSessionStaleFailure)); - hostProvisioner.ifPresent(provisioner -> provisioner.activate(transaction, applicationId, session.getAllocatedHosts().getHosts())); - transaction.commit(); + waiter = applicationRepository.activate(session, previousActiveSession, applicationId, ignoreSessionStaleFailure); } catch (RuntimeException e) { throw e; @@ -189,51 +182,4 @@ public class Deployment implements com.yahoo.config.provision.Deployment { } } - private static Transaction deactivateCurrentActivateNew(Session active, LocalSession prepared, boolean ignoreStaleSessionFailure) { - Transaction transaction = prepared.createActivateTransaction(); - if (isValidSession(active)) { - checkIfActiveHasChanged(prepared, active, ignoreStaleSessionFailure); - checkIfActiveIsNewerThanSessionToBeActivated(prepared.getSessionId(), active.getSessionId()); - transaction.add(active.createDeactivateTransaction().operations()); - } - return transaction; - } - - private static boolean isValidSession(Session session) { - return session != null; - } - - private static void checkIfActiveHasChanged(LocalSession session, Session currentActiveSession, boolean ignoreStaleSessionFailure) { - long activeSessionAtCreate = session.getActiveSessionAtCreate(); - log.log(Level.FINE, currentActiveSession.logPre() + "active session id at create time=" + activeSessionAtCreate); - if (activeSessionAtCreate == 0) return; // No active session at create - - long sessionId = session.getSessionId(); - long currentActiveSessionSessionId = currentActiveSession.getSessionId(); - log.log(Level.FINE, currentActiveSession.logPre() + "sessionId=" + sessionId + - ", current active session=" + currentActiveSessionSessionId); - if (currentActiveSession.isNewerThan(activeSessionAtCreate) && - currentActiveSessionSessionId != sessionId) { - String errMsg = currentActiveSession.logPre() + "Cannot activate session " + - sessionId + " because the currently active session (" + - currentActiveSessionSessionId + ") has changed since session " + sessionId + - " was created (was " + activeSessionAtCreate + " at creation time)"; - if (ignoreStaleSessionFailure) { - log.warning(errMsg + " (Continuing because of force.)"); - } else { - throw new ActivationConflictException(errMsg); - } - } - } - - // As of now, config generation is based on session id, and config generation must be a monotonically - // increasing number - private static void checkIfActiveIsNewerThanSessionToBeActivated(long sessionId, long currentActiveSessionId) { - if (sessionId < currentActiveSessionId) { - throw new ActivationConflictException("It is not possible to activate session " + sessionId + - ", because it is older than current active session (" + - currentActiveSessionId + ")"); - } - } - } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java index 96324ea4320..f842578b657 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java @@ -1,15 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.session; -import com.yahoo.config.application.api.ApplicationFile; -import com.yahoo.config.application.api.ApplicationMetaData; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.provision.TenantName; -import com.yahoo.path.Path; -import com.yahoo.transaction.Transaction; -import com.yahoo.vespa.config.server.application.TenantApplications; - -import static com.yahoo.vespa.curator.Curator.CompletionWaiter; /** * A LocalSession is a session that has been created locally on this configserver. A local session can be edited and @@ -22,65 +15,14 @@ import static com.yahoo.vespa.curator.Curator.CompletionWaiter; // TODO: Separate the "application store" and "session" aspects - the latter belongs in the HTTP layer -bratseth public class LocalSession extends Session { - protected final ApplicationPackage applicationPackage; - private final TenantApplications applicationRepo; - /** * Creates a session. This involves loading the application, validating it and distributing it. * * @param sessionId The session id for this session. */ public LocalSession(TenantName tenant, long sessionId, ApplicationPackage applicationPackage, - SessionZooKeeperClient sessionZooKeeperClient, - TenantApplications applicationRepo) { - super(tenant, sessionId, sessionZooKeeperClient); - this.applicationPackage = applicationPackage; - this.applicationRepo = applicationRepo; - } - - public ApplicationFile getApplicationFile(Path relativePath, Mode mode) { - if (mode.equals(Mode.WRITE)) { - markSessionEdited(); - } - return applicationPackage.getFile(relativePath); - } - - void setPrepared() { - setStatus(Session.Status.PREPARE); - } - - private Transaction createSetStatusTransaction(Status status) { - return sessionZooKeeperClient.createWriteStatusTransaction(status); - } - - private void setStatus(Session.Status newStatus) { - sessionZooKeeperClient.writeStatus(newStatus); - } - - public CompletionWaiter createActiveWaiter() { - return sessionZooKeeperClient.createActiveWaiter(); - } - - public Transaction createActivateTransaction() { - Transaction transaction = createSetStatusTransaction(Status.ACTIVATE); - transaction.add(applicationRepo.createPutTransaction(sessionZooKeeperClient.readApplicationId(), getSessionId()).operations()); - return transaction; + SessionZooKeeperClient sessionZooKeeperClient) { + super(tenant, sessionId, sessionZooKeeperClient, applicationPackage); } - private void markSessionEdited() { - setStatus(Session.Status.NEW); - } - - public long getActiveSessionAtCreate() { - return applicationPackage.getMetaData().getPreviousActiveGeneration(); - } - - public enum Mode { - READ, WRITE - } - - public ApplicationMetaData getMetaData() { return applicationPackage.getMetaData(); } - - public ApplicationPackage getApplicationPackage() { return applicationPackage; } - } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java index 9677c7cf20e..642ac33ab09 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.session; -import com.yahoo.config.application.api.ApplicationMetaData; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.TenantName; @@ -102,7 +101,7 @@ public class RemoteSession extends Session { KeeperException.NodeExistsException.class); Class<? extends Throwable> exceptionClass = e.getCause().getClass(); if (acceptedExceptions.contains(exceptionClass)) - log.log(Level.INFO, "Not able to notify completion for session " + getSessionId() + + log.log(Level.FINE, "Not able to notify completion for session " + getSessionId() + " (" + completionWaiter + ")," + " node " + (exceptionClass.equals(KeeperException.NoNodeException.class) ? "has been deleted" @@ -118,8 +117,4 @@ public class RemoteSession extends Session { transaction.close(); } - public ApplicationMetaData getMetaData() { - return sessionZooKeeperClient.loadApplicationPackage().getMetaData(); - } - } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java index a6818d1e43f..0fc85b5e51a 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java @@ -3,11 +3,15 @@ package com.yahoo.vespa.config.server.session; import com.yahoo.component.Version; import com.yahoo.config.FileReference; +import com.yahoo.config.application.api.ApplicationFile; +import com.yahoo.config.application.api.ApplicationMetaData; +import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.AthenzDomain; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.TenantName; +import com.yahoo.path.Path; import com.yahoo.transaction.Transaction; import com.yahoo.vespa.config.server.tenant.TenantRepository; @@ -27,13 +31,24 @@ public abstract class Session implements Comparable<Session> { private final long sessionId; protected final TenantName tenant; protected final SessionZooKeeperClient sessionZooKeeperClient; + protected final Optional<ApplicationPackage> applicationPackage; protected Session(TenantName tenant, long sessionId, SessionZooKeeperClient sessionZooKeeperClient) { this.tenant = tenant; this.sessionId = sessionId; this.sessionZooKeeperClient = sessionZooKeeperClient; + this.applicationPackage = Optional.empty(); } + protected Session(TenantName tenant, long sessionId, SessionZooKeeperClient sessionZooKeeperClient, + ApplicationPackage applicationPackage) { + this.tenant = tenant; + this.sessionId = sessionId; + this.sessionZooKeeperClient = sessionZooKeeperClient; + this.applicationPackage = Optional.of(applicationPackage); + } + + public final long getSessionId() { return sessionId; } @@ -42,11 +57,19 @@ public abstract class Session implements Comparable<Session> { return sessionZooKeeperClient.readStatus(); } + public SessionZooKeeperClient getSessionZooKeeperClient() { + return sessionZooKeeperClient; + } + @Override public String toString() { return "Session,id=" + sessionId; } + public long getActiveSessionAtCreate() { + return getMetaData().getPreviousActiveGeneration(); + } + /** * The status of this session. */ @@ -135,6 +158,31 @@ public abstract class Session implements Comparable<Session> { // Note: Assumes monotonically increasing session ids public boolean isNewerThan(long sessionId) { return getSessionId() > sessionId; } + public ApplicationMetaData getMetaData() { + return applicationPackage.isPresent() + ? applicationPackage.get().getMetaData() + : sessionZooKeeperClient.loadApplicationPackage().getMetaData(); + } + + public ApplicationPackage getApplicationPackage() { + return applicationPackage.orElseThrow(() -> new RuntimeException("No application package found for " + this)); + } + + public ApplicationFile getApplicationFile(Path relativePath, LocalSession.Mode mode) { + if (mode.equals(Session.Mode.WRITE)) { + markSessionEdited(); + } + return getApplicationPackage().getFile(relativePath); + } + + private void markSessionEdited() { + setStatus(Session.Status.NEW); + } + + void setStatus(Session.Status newStatus) { + sessionZooKeeperClient.writeStatus(newStatus); + } + @Override public int compareTo(Session rhs) { Long lhsId = getSessionId(); @@ -142,4 +190,8 @@ public abstract class Session implements Comparable<Session> { return lhsId.compareTo(rhsId); } + public enum Mode { + READ, WRITE + } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java index 389b661909e..6c4ef469be6 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java @@ -9,7 +9,6 @@ import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.application.provider.DeployData; import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.TenantName; import com.yahoo.io.IOUtils; import com.yahoo.path.Path; @@ -47,7 +46,6 @@ import java.time.Clock; import java.time.Duration; import java.time.Instant; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -166,7 +164,7 @@ public class SessionRepository { getSessionAppDir(sessionId), session.getApplicationPackage(), sessionZooKeeperClient) .getConfigChangeActions(); - session.setPrepared(); + setPrepared(session); waiter.awaitCompletion(params.getTimeoutBudget().timeLeft()); return actions; } @@ -317,8 +315,7 @@ public class SessionRepository { getRemoteSessions().forEach(this::sessionAdded); } - private synchronized void sessionsChanged() throws NumberFormatException { - List<Long> sessions = getSessionListFromDirectoryCache(directoryCache.getCurrentData()); + private synchronized void sessionsChanged(List<Long> sessions) throws NumberFormatException { checkForRemovedSessions(sessions); checkForAddedSessions(sessions); } @@ -418,14 +415,15 @@ public class SessionRepository { private void childEvent(CuratorFramework ignored, PathChildrenCacheEvent event) { zkWatcherExecutor.execute(() -> { log.log(Level.FINE, () -> "Got child event: " + event); + List<Long> sessions = getSessionListFromDirectoryCache(directoryCache.getCurrentData()); switch (event.getType()) { case CHILD_ADDED: - sessionsChanged(); - synchronizeOnNew(getSessionListFromDirectoryCache(Collections.singletonList(event.getData()))); + sessionsChanged(sessions); + synchronizeOnNew(sessions); break; case CHILD_REMOVED: case CONNECTION_RECONNECTED: - sessionsChanged(); + sessionsChanged(sessions); break; } }); @@ -489,7 +487,7 @@ public class SessionRepository { SessionZooKeeperClient sessionZKClient = createSessionZooKeeperClient(sessionId); sessionZKClient.createNewSession(clock.instant()); Curator.CompletionWaiter waiter = sessionZKClient.getUploadWaiter(); - LocalSession session = new LocalSession(tenantName, sessionId, applicationPackage, sessionZKClient, applicationRepo); + LocalSession session = new LocalSession(tenantName, sessionId, applicationPackage, sessionZKClient); waiter.awaitCompletion(timeoutBudget.timeLeft()); return session; } @@ -547,7 +545,7 @@ public class SessionRepository { ApplicationPackage applicationPackage = createApplicationPackage(applicationFile, applicationId, sessionId, currentlyActiveSessionId, false); SessionZooKeeperClient sessionZooKeeperClient = createSessionZooKeeperClient(sessionId); - return new LocalSession(tenantName, sessionId, applicationPackage, sessionZooKeeperClient, applicationRepo); + return new LocalSession(tenantName, sessionId, applicationPackage, sessionZooKeeperClient); } catch (Exception e) { throw new RuntimeException("Error creating session " + sessionId, e); } @@ -596,7 +594,7 @@ public class SessionRepository { File sessionDir = getAndValidateExistingSessionAppDir(sessionId); ApplicationPackage applicationPackage = FilesApplicationPackage.fromFile(sessionDir); SessionZooKeeperClient sessionZKClient = createSessionZooKeeperClient(sessionId); - return new LocalSession(tenantName, sessionId, applicationPackage, sessionZKClient, applicationRepo); + return new LocalSession(tenantName, sessionId, applicationPackage, sessionZKClient); } /** @@ -654,9 +652,8 @@ public class SessionRepository { private SessionZooKeeperClient createSessionZooKeeperClient(long sessionId) { String serverId = componentRegistry.getConfigserverConfig().serverId(); - Optional<NodeFlavors> nodeFlavors = componentRegistry.getZone().nodeFlavors(); Path sessionPath = getSessionPath(sessionId); - return new SessionZooKeeperClient(curator, componentRegistry.getConfigCurator(), sessionPath, serverId, nodeFlavors); + return new SessionZooKeeperClient(curator, componentRegistry.getConfigCurator(), sessionPath, serverId); } private File getAndValidateExistingSessionAppDir(long sessionId) { @@ -697,6 +694,20 @@ public class SessionRepository { return locksPath.append(String.valueOf(sessionId)); } + public Transaction createActivateTransaction(Session session) { + Transaction transaction = createSetStatusTransaction(session, Session.Status.ACTIVATE); + transaction.add(applicationRepo.createPutTransaction(session.sessionZooKeeperClient.readApplicationId(), session.getSessionId()).operations()); + return transaction; + } + + private Transaction createSetStatusTransaction(Session session, Session.Status status) { + return session.sessionZooKeeperClient.createWriteStatusTransaction(status); + } + + void setPrepared(Session session) { + session.setStatus(Session.Status.PREPARE); + } + private static class FileTransaction extends AbstractTransaction { public static FileTransaction from(FileOperation operation) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java index 1b9527f4376..cf1e07788ff 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java @@ -51,23 +51,21 @@ public class SessionZooKeeperClient { private final Path sessionPath; private final Path sessionStatusPath; private final String serverId; // hostname - private final Optional<NodeFlavors> nodeFlavors; - // Only for testing when cache loader does not need cache entries. + // Only for testing + // TODO: Remove, use the constructor below public SessionZooKeeperClient(Curator curator, Path sessionPath) { - this(curator, ConfigCurator.create(curator), sessionPath, "1", Optional.empty()); + this(curator, ConfigCurator.create(curator), sessionPath, "1"); } public SessionZooKeeperClient(Curator curator, ConfigCurator configCurator, Path sessionPath, - String serverId, - Optional<NodeFlavors> nodeFlavors) { + String serverId) { this.curator = curator; this.configCurator = configCurator; this.sessionPath = sessionPath; this.serverId = serverId; - this.nodeFlavors = nodeFlavors; this.sessionStatusPath = sessionPath.append(ConfigCurator.SESSIONSTATE_ZK_SUBPATH); } @@ -93,7 +91,7 @@ public class SessionZooKeeperClient { return createCompletionWaiter(PREPARE_BARRIER); } - Curator.CompletionWaiter createActiveWaiter() { + public Curator.CompletionWaiter createActiveWaiter() { return createCompletionWaiter(ACTIVE_BARRIER); } @@ -134,7 +132,7 @@ public class SessionZooKeeperClient { } public ApplicationPackage loadApplicationPackage() { - return new ZKApplicationPackage(configCurator, sessionPath, nodeFlavors); + return new ZKApplicationPackage(configCurator, sessionPath); } public ConfigDefinitionRepo getUserConfigDefinitions() { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java index 11cec9efd95..665f37759b4 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java @@ -13,7 +13,6 @@ import com.yahoo.config.codegen.DefParser; import com.yahoo.config.model.application.provider.PreGeneratedFileRegistry; import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.serialization.AllocatedHostsSerializer; import com.yahoo.io.IOUtils; import com.yahoo.io.reader.NamedReader; @@ -52,17 +51,17 @@ public class ZKApplicationPackage implements ApplicationPackage { public static final String allocatedHostsNode = "allocatedHosts"; private final ApplicationMetaData metaData; - public ZKApplicationPackage(ConfigCurator zk, Path sessionPath, Optional<NodeFlavors> nodeFlavors) { + public ZKApplicationPackage(ConfigCurator zk, Path sessionPath) { verifyAppPath(zk, sessionPath); zkApplication = new ZKApplication(zk, sessionPath); metaData = readMetaDataFromLiveApp(zkApplication); importFileRegistries(); - allocatedHosts = importAllocatedHosts(nodeFlavors); + allocatedHosts = importAllocatedHosts(); } - private Optional<AllocatedHosts> importAllocatedHosts(Optional<NodeFlavors> nodeFlavors) { + private Optional<AllocatedHosts> importAllocatedHosts() { if ( ! zkApplication.exists(ZKApplicationPackage.allocatedHostsNode)) return Optional.empty(); - return Optional.of(readAllocatedHosts(nodeFlavors)); + return Optional.of(readAllocatedHosts()); } /** @@ -70,9 +69,9 @@ public class ZKApplicationPackage implements ApplicationPackage { * * @return the allocated hosts at this node or empty if there is no data at this path */ - private AllocatedHosts readAllocatedHosts(Optional<NodeFlavors> nodeFlavors) { + private AllocatedHosts readAllocatedHosts() { try { - return AllocatedHostsSerializer.fromJson(zkApplication.getBytes(ZKApplicationPackage.allocatedHostsNode), nodeFlavors); + return AllocatedHostsSerializer.fromJson(zkApplication.getBytes(ZKApplicationPackage.allocatedHostsNode)); } catch (Exception e) { throw new RuntimeException("Unable to read allocated hosts", e); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java index 50a8cac2837..e42fb78f595 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java @@ -325,6 +325,10 @@ public class ApplicationRepositoryTest { { PrepareResult prepareResult = deployApp(testApp); + + assertNotNull(applicationRepository.getActiveSession(applicationId())); + assertNotNull(sessionRepository.getLocalSession(prepareResult.sessionId())); + try { applicationRepository.delete(applicationId(), Duration.ZERO); fail("Should have gotten an exception"); @@ -334,8 +338,7 @@ public class ApplicationRepositoryTest { // No active session or remote session (deleted in step above), but an exception was thrown above // A new delete should cleanup and be successful - RemoteSession activeSession = applicationRepository.getActiveSession(applicationId()); - assertNull(activeSession); + assertNull(applicationRepository.getActiveSession(applicationId())); assertNull(sessionRepository.getLocalSession(prepareResult.sessionId())); assertTrue(applicationRepository.delete(applicationId())); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java index dfc5649433a..deff0aba376 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClientTest.java @@ -165,7 +165,7 @@ public class ZooKeeperClientTest { Path hostsPath = app.append(ZKApplicationPackage.allocatedHostsNode); assertTrue(zk.exists(hostsPath.getAbsolute())); - AllocatedHosts deserialized = fromJson(zk.getBytes(hostsPath.getAbsolute()), Optional.empty()); + AllocatedHosts deserialized = fromJson(zk.getBytes(hostsPath.getAbsolute())); assertEquals(hosts, deserialized.getHosts()); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java index d2f09b802da..c6d241ee534 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java @@ -12,7 +12,7 @@ import com.yahoo.vespa.config.server.application.CompressedApplicationInputStrea import com.yahoo.vespa.config.server.application.OrchestratorMock; import com.yahoo.vespa.config.server.http.HttpErrorResponse; import com.yahoo.vespa.config.server.http.SessionHandlerTest; -import com.yahoo.vespa.config.server.session.LocalSession; +import com.yahoo.vespa.config.server.session.Session; import com.yahoo.vespa.config.server.tenant.TenantRepository; import org.junit.Before; import org.junit.Ignore; @@ -135,7 +135,7 @@ public class SessionCreateHandlerTest extends SessionHandlerTest { public void require_that_handler_unpacks_application() throws IOException { File outFile = CompressedApplicationInputStreamTest.createTarFile(); createHandler().handle(post(outFile)); - ApplicationFile applicationFile = applicationRepository.getApplicationFileFromSession(tenant, 2, "services.xml", LocalSession.Mode.READ); + ApplicationFile applicationFile = applicationRepository.getApplicationFileFromSession(tenant, 2, "services.xml", Session.Mode.READ); assertTrue(applicationFile.exists()); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java index a2ef6aeb578..cf01c9b6713 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java @@ -77,31 +77,21 @@ public class LocalSessionTest { } @Test - public void require_that_session_status_is_updated() throws Exception { - LocalSession session = createSession(TenantName.defaultName(), 3); - assertThat(session.getStatus(), is(Session.Status.NEW)); - doPrepare(session); - assertThat(session.getStatus(), is(Session.Status.PREPARE)); - session.createActivateTransaction().commit(); - assertThat(session.getStatus(), is(Session.Status.ACTIVATE)); - } - - @Test public void require_that_marking_session_modified_changes_status_to_new() throws Exception { LocalSession session = createSession(TenantName.defaultName(), 3); doPrepare(session); assertThat(session.getStatus(), is(Session.Status.PREPARE)); - session.getApplicationFile(Path.createRoot(), LocalSession.Mode.READ); + session.getApplicationFile(Path.createRoot(), Session.Mode.READ); assertThat(session.getStatus(), is(Session.Status.PREPARE)); - session.getApplicationFile(Path.createRoot(), LocalSession.Mode.WRITE); + session.getApplicationFile(Path.createRoot(), Session.Mode.WRITE); assertThat(session.getStatus(), is(Session.Status.NEW)); } @Test public void require_that_application_file_can_be_fetched() throws Exception { LocalSession session = createSession(TenantName.defaultName(), 3); - ApplicationFile f1 = session.getApplicationFile(Path.fromString("services.xml"), LocalSession.Mode.READ); - ApplicationFile f2 = session.getApplicationFile(Path.fromString("services2.xml"), LocalSession.Mode.READ); + ApplicationFile f1 = session.getApplicationFile(Path.fromString("services.xml"), Session.Mode.READ); + ApplicationFile f2 = session.getApplicationFile(Path.fromString("services2.xml"), Session.Mode.READ); assertTrue(f1.exists()); assertFalse(f2.exists()); } @@ -127,7 +117,7 @@ public class LocalSessionTest { zkClient.write(Collections.singletonMap(new Version(0, 0, 0), new MockFileRegistry())); TenantApplications applications = tenantRepository.getTenant(tenantName).getApplicationRepo(); applications.createApplication(applicationId()); - LocalSession session = new LocalSession(tenant, sessionId, FilesApplicationPackage.fromFile(testApp), zkc, applications); + LocalSession session = new LocalSession(tenant, sessionId, FilesApplicationPackage.fromFile(testApp), zkc); session.setApplicationId(applicationId()); return session; } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java index 4b4605cce7d..32d5d84f323 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java @@ -48,7 +48,7 @@ public class ZKApplicationPackageTest { Collections.singleton(new HostSpec("foo.yahoo.com", TEST_FLAVOR.get().resources(), TEST_FLAVOR.get().resources(), - NodeResources.unspecified(), + TEST_FLAVOR.get().resources(), ClusterMembership.from("container/test/0/0", Version.fromString("6.73.1"), Optional.of(DockerImage.fromString("docker.foo.com:4443/vespa/bar"))), Optional.of(Version.fromString("6.0.1")), Optional.empty(), @@ -67,7 +67,7 @@ public class ZKApplicationPackageTest { @Test public void testBasicZKFeed() throws IOException { feed(configCurator, new File(APP)); - ZKApplicationPackage zkApp = new ZKApplicationPackage(configCurator, Path.fromString("/0"), Optional.of(new MockNodeFlavors())); + ZKApplicationPackage zkApp = new ZKApplicationPackage(configCurator, Path.fromString("/0")); assertTrue(Pattern.compile(".*<slobroks>.*",Pattern.MULTILINE+Pattern.DOTALL).matcher(IOUtils.readAll(zkApp.getServices())).matches()); assertTrue(Pattern.compile(".*<alias>.*",Pattern.MULTILINE+Pattern.DOTALL).matcher(IOUtils.readAll(zkApp.getHosts())).matches()); assertTrue(Pattern.compile(".*<slobroks>.*",Pattern.MULTILINE+Pattern.DOTALL).matcher(IOUtils.readAll(zkApp.getFile(Path.fromString("services.xml")).createReader())).matches()); diff --git a/container-search/src/main/java/com/yahoo/prelude/Location.java b/container-search/src/main/java/com/yahoo/prelude/Location.java index f6228509bbb..3e9c2382f31 100644 --- a/container-search/src/main/java/com/yahoo/prelude/Location.java +++ b/container-search/src/main/java/com/yahoo/prelude/Location.java @@ -24,16 +24,9 @@ public class Location { private int y2 = 1; // center(x,y), radius - private int x = 1; - private int y = 1; - private int r = 1; - - // next three are now UNUSED - // ranking table, rank multiplier (scale) - // {0, 1} an int to make parsing and rendering the hit even simpler - private int tableId = 0; - private int s = 1; - private int replace = 0; + private int x = 0; + private int y = 0; + private int r = -1; private boolean renderCircle = false; private boolean renderRectangle = false; @@ -47,14 +40,14 @@ public class Location { return dimensions == l.dimensions && renderCircle == l.renderCircle && renderRectangle == l.renderRectangle - && aspect == l.aspect - && x1 == l.x1 - && x2 == l.x2 - && y1 == l.y1 - && y2 == l.y2 - && x == l.x - && y == l.y - && r == l.r; + && this.aspect == l.aspect + && this.x1 == l.x1 + && this.x2 == l.x2 + && this.y1 == l.y1 + && this.y2 == l.y2 + && this.x == l.x + && this.y == l.y + && this.r == l.r; } public boolean hasDimensions() { @@ -64,10 +57,10 @@ public class Location { if (hasDimensions() && dimensions != d) { throw new IllegalArgumentException("already has dimensions="+dimensions+", cannot change it to "+d); } - if (d == 1 || d == 2) { + if (d == 2) { dimensions = d; } else { - throw new IllegalArgumentException("Illegal location, dimensions must be 1 or 2, but was: "+d); + throw new IllegalArgumentException("Illegal location, dimensions must be 2, but was: "+d); } } public int getDimensions() { @@ -89,13 +82,13 @@ public class Location { if (px1 > px2) { throw new IllegalArgumentException("cannot have w > e"); } - x1 = px1; - x2 = px2; + this.x1 = px1; + this.x2 = px2; if (py1 > py2) { throw new IllegalArgumentException("cannot have s > n"); } - y1 = py1; - y2 = py2; + this.y1 = py1; + this.y2 = py2; renderRectangle = true; } @@ -104,12 +97,12 @@ public class Location { //no need to "optimize" for special cases, exactly 0, 30, 45, 60, or 90 degrees won't be input anyway double degrees = (double) y / 1000000d; if (degrees <= -90.0 || degrees >= +90.0) { - aspect = 0; + this.aspect = 0; return; } double radians = degrees * Math.PI / 180d; double cosLatRadians = Math.cos(radians); - aspect = (long) (cosLatRadians * 4294967295L); + this.aspect = (long) (cosLatRadians * 4294967295L); } public void setGeoCircle(double ns, double ew, double radius_in_degrees) { @@ -129,9 +122,9 @@ public class Location { if (radius_in_degrees < 0) { pr = -1; } - x = px; - y = py; - r = pr; + this.x = px; + this.y = py; + this.r = pr; renderCircle = true; adjustAspect(); } @@ -144,9 +137,9 @@ public class Location { if (radius_in_units < 0) { radius_in_units = -1; } - x = px; - y = py; - r = radius_in_units; + this.x = px; + this.y = py; + this.r = radius_in_units; renderCircle = true; } @@ -158,17 +151,12 @@ public class Location { String rectPart = rectangle.substring(1,endof); StringTokenizer tokens = new StringTokenizer(rectPart, ","); setDimensions(Integer.parseInt(tokens.nextToken())); - if (dimensions == 1) { - x1 = Integer.parseInt(tokens.nextToken()); - x2 = Integer.parseInt(tokens.nextToken()); - if (tokens.hasMoreTokens()) { - throw new IllegalArgumentException("Illegal location syntax: "+rectangle); - } - } else if (dimensions == 2) { - x1 = Integer.parseInt(tokens.nextToken()); - y1 = Integer.parseInt(tokens.nextToken()); - x2 = Integer.parseInt(tokens.nextToken()); - y2 = Integer.parseInt(tokens.nextToken()); + this.x1 = Integer.parseInt(tokens.nextToken()); + this.y1 = Integer.parseInt(tokens.nextToken()); + this.x2 = Integer.parseInt(tokens.nextToken()); + this.y2 = Integer.parseInt(tokens.nextToken()); + if (tokens.hasMoreTokens()) { + throw new IllegalArgumentException("Illegal location syntax: "+rectangle); } renderRectangle = true; String theRest = rectangle.substring(endof+1).trim(); @@ -185,34 +173,24 @@ public class Location { String circlePart = circle.substring(1,endof); StringTokenizer tokens = new StringTokenizer(circlePart, ","); setDimensions(Integer.parseInt(tokens.nextToken())); - x = Integer.parseInt(tokens.nextToken()); - if (dimensions == 2) { - y = Integer.parseInt(tokens.nextToken()); - } - r = Integer.parseInt(tokens.nextToken()); + this.x = Integer.parseInt(tokens.nextToken()); + this.y = Integer.parseInt(tokens.nextToken()); + this.r = Integer.parseInt(tokens.nextToken()); Integer.parseInt(tokens.nextToken()); // was "tableId" - Integer.parseInt(tokens.nextToken()); // was "scale" (multiplier) + Integer.parseInt(tokens.nextToken()); // was "scale" Integer.parseInt(tokens.nextToken()); // was "replace" - - if (dimensions == 1) { - if (tokens.hasMoreTokens()) { - throw new IllegalArgumentException("Illegal location syntax: "+circle); - } - } - else { - if (tokens.hasMoreTokens()) { - String aspectToken = tokens.nextToken(); - if (aspectToken.equalsIgnoreCase("CalcLatLon")) { - adjustAspect(); - } else { - try { - aspect = Long.parseLong(aspectToken); - } catch (NumberFormatException nfe) { - throw new IllegalArgumentException("Aspect "+aspectToken+" for location must be an integer or 'CalcLatLon' for automatic aspect calculation.", nfe); - } - if (aspect > 4294967295L || aspect < 0) { - throw new IllegalArgumentException("Aspect "+aspect+" for location parameter must be less than 4294967296 (2^32)"); - } + if (tokens.hasMoreTokens()) { + String aspectToken = tokens.nextToken(); + if (aspectToken.equalsIgnoreCase("CalcLatLon")) { + adjustAspect(); + } else { + try { + aspect = Long.parseLong(aspectToken); + } catch (NumberFormatException nfe) { + throw new IllegalArgumentException("Aspect "+aspectToken+" for location must be an integer or 'CalcLatLon' for automatic aspect calculation.", nfe); + } + if (aspect > 4294967295L || aspect < 0) { + throw new IllegalArgumentException("Aspect "+aspect+" for location parameter must be less than 4294967296 (2^32)"); } } } @@ -261,28 +239,20 @@ public class Location { } if (renderRectangle) { ser.append("[").append(dimensions).append(","); - if (dimensions == 1) { - ser.append(x1).append(","). - append(x2); - } - else { - ser.append(x1).append(","). - append(y1).append(","). - append(x2).append(","). - append(y2); - } + ser.append(x1).append(","). + append(y1).append(","). + append(x2).append(","). + append(y2); ser.append("]"); } if (renderCircle) { - ser.append("(").append(dimensions).append(",").append(x); - if (dimensions == 2) { - ser.append(",").append(y); - } - ser.append(",").append(forBackend ? backendRadius() : r). - append(",").append(tableId). - append(",").append(s). - append(",").append(replace); - if (dimensions == 2 && aspect != 0) { + ser.append("(").append(dimensions).append(","). + append(this.x).append(",").append(this.y); + ser.append(",").append(forBackend ? backendRadius() : this.r). + append(",").append(0). // was "tableId" + append(",").append(1). // was "scale" + append(",").append(0); // was "replace" + if (aspect != 0) { ser.append(",").append(aspect); } ser.append(")"); @@ -296,7 +266,7 @@ public class Location { */ public int getBoundingWidth() { if (renderCircle) { - return r * 2; + return this.r * 2; } else { return x2 - x1; } @@ -308,7 +278,7 @@ public class Location { */ public int getBoundingHeight() { if (renderCircle) { - return r * 2; + return this.r * 2; } else { return y2 - y1; } @@ -370,11 +340,11 @@ public class Location { **/ public double degRadius() { checkGeoCircle(); - return (r < 0) ? -1.0 : (0.000001 * r); + return (this.r < 0) ? -1.0 : (0.000001 * this.r); } private int backendRadius() { - return (r < 0) ? -1 : r; + return (this.r < 0) ? -1 : this.r; } /** diff --git a/container-search/src/main/java/com/yahoo/prelude/query/BoolItem.java b/container-search/src/main/java/com/yahoo/prelude/query/BoolItem.java index 27045629780..542df9d4b8b 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/BoolItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/BoolItem.java @@ -1,6 +1,8 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.query; +import com.yahoo.processing.IllegalInputException; + import java.nio.ByteBuffer; /** @@ -59,7 +61,7 @@ public class BoolItem extends TermItem { switch (stringValue.toLowerCase()) { case "true" : return true; case "false" : return false; - default: throw new IllegalArgumentException("Expected 'true' or 'false', got '" + stringValue + "'"); + default: throw new IllegalInputException("Expected 'true' or 'false', got '" + stringValue + "'"); } } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/CompositeItem.java b/container-search/src/main/java/com/yahoo/prelude/query/CompositeItem.java index 64f759dcf9c..4609edb7446 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/CompositeItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/CompositeItem.java @@ -46,7 +46,7 @@ public abstract class CompositeItem extends Item { Item possibleCycle = i.next(); if (this == possibleCycle) { - throw new QueryException("Cannot add " + item + " to " + this + " as it would create a cycle"); + throw new IllegalArgumentException("Cannot add " + item + " to " + this + " as it would create a cycle"); } else if (possibleCycle instanceof CompositeItem) { ensureNotInSubtree((CompositeItem) possibleCycle); } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/GeoLocationItem.java b/container-search/src/main/java/com/yahoo/prelude/query/GeoLocationItem.java index 8202c8fb279..ba5270e7af7 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/GeoLocationItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/GeoLocationItem.java @@ -22,7 +22,7 @@ public class GeoLocationItem extends TermItem { public GeoLocationItem(Location location) { this(location, location.getAttribute()); if (! location.hasAttribute()) { - throw new IllegalArgumentException("missing attribute on location: "+location); + throw new IllegalArgumentException("Missing attribute on location: " + location); } } @@ -34,13 +34,14 @@ public class GeoLocationItem extends TermItem { public GeoLocationItem(Location location, String fieldName) { super(fieldName, false); if (location.hasAttribute() && ! location.getAttribute().equals(fieldName)) { - throw new IllegalArgumentException("inconsistent attribute on location: "+location.getAttribute()+" versus fieldName: "+fieldName); + throw new IllegalArgumentException("Inconsistent attribute on location: " + location.getAttribute() + + " versus fieldName: " + fieldName); } if (! location.isGeoCircle()) { - throw new IllegalArgumentException("GeoLocationItem only supports Geo Circles, got: "+location); + throw new IllegalArgumentException("GeoLocationItem only supports Geo Circles, got: " + location); } if (location.hasBoundingBox()) { - throw new IllegalArgumentException("GeoLocationItem does not support bounding box yet, got: "+location); + throw new IllegalArgumentException("GeoLocationItem does not support bounding box, got: " + location); } this.location = new Location(location.toString()); this.location.setAttribute(null); // keep this in (superclass) indexName only diff --git a/container-search/src/main/java/com/yahoo/prelude/query/IntItem.java b/container-search/src/main/java/com/yahoo/prelude/query/IntItem.java index 714e8f9cb5e..1591d31f749 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/IntItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/IntItem.java @@ -111,7 +111,7 @@ public class IntItem extends TermItem { } catch (IllegalArgumentException e) { throw new IllegalArgumentException("'" + expression + "' is not an int item expression: " + - "Expected NUMBER, '<'NUMBER, '>'NUMBER or ('['|'<')NUMBER;NUMBER(;NUMBER)?(']'|'>')", e); + "Expected NUMBER, '<'NUMBER, '>'NUMBER or ('['|'<')NUMBER;NUMBER(;NUMBER)?(']'|'>')", e); } } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/QueryException.java b/container-search/src/main/java/com/yahoo/prelude/query/QueryException.java index 34898827c2e..7d029d69077 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/QueryException.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/QueryException.java @@ -4,10 +4,11 @@ package com.yahoo.prelude.query; /** * Runtime exception to mark errors in query parsing. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen + * @deprecated no methods throw this */ +@Deprecated // TODO: Remove on Vespa 8 public class QueryException extends RuntimeException { - private static final long serialVersionUID = -2975856668328596533L; public QueryException(String message) { super(message); @@ -16,4 +17,5 @@ public class QueryException extends RuntimeException { public QueryException(String message, Throwable cause) { super(message, cause); } + } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/RegExpItem.java b/container-search/src/main/java/com/yahoo/prelude/query/RegExpItem.java index 2a5c6135d71..ec3744306ed 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/RegExpItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/RegExpItem.java @@ -7,9 +7,10 @@ import java.util.regex.Pattern; /** * Match a field with the contained regular expression. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ public class RegExpItem extends TermItem { + private String expression; private Pattern regexp; diff --git a/container-search/src/main/java/com/yahoo/prelude/query/SegmentItem.java b/container-search/src/main/java/com/yahoo/prelude/query/SegmentItem.java index 1c3eb261f90..f70bf8021ff 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/SegmentItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/SegmentItem.java @@ -102,7 +102,7 @@ public abstract class SegmentItem extends CompositeItem implements BlockItem { } private void dontAdd() { - throw new QueryException("Tried to add item to an immutable segment."); + throw new IllegalArgumentException("Tried to add item to an immutable segment."); } public Item removeItem(int index) { @@ -120,7 +120,7 @@ public abstract class SegmentItem extends CompositeItem implements BlockItem { } private void dontRemove() { - throw new QueryException("Tried to remove an item from an immutable segment."); + throw new IllegalArgumentException("Tried to remove an item from an immutable segment."); } // TODO: Add a getItemIterator which is safe for immutability diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/AdvancedParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/AdvancedParser.java index e3d1b280a5a..e2dc5df5242 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/AdvancedParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/AdvancedParser.java @@ -206,7 +206,7 @@ public class AdvancedParser extends StructuredParser { if (!tokens.currentIs(LBRACE)) return 0; tokens.skip(LBRACE); if (!tokens.currentIsNoIgnore(NUMBER)) throw new IllegalArgumentException("Expected an integer argument"); - int distance=Integer.valueOf(tokens.next().image); + int distance = Integer.valueOf(tokens.next().image); if (!tokens.skip(Token.Kind.RBRACE)) throw new IllegalArgumentException("Expected a right brace following the argument"); return distance; } diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/PhraseMatcher.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/PhraseMatcher.java index f49e49c1771..be33c0ee9e1 100644 --- a/container-search/src/main/java/com/yahoo/prelude/querytransform/PhraseMatcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/querytransform/PhraseMatcher.java @@ -64,9 +64,9 @@ public class PhraseMatcher { * @throws IllegalArgumentException if FSA is null */ public PhraseMatcher(FSA phraseAutomatonFSA,boolean ignorePluralForm) { - if(phraseAutomatonFSA==null) throw new IllegalArgumentException("FSA is null"); - this.ignorePluralForm=ignorePluralForm; - phraseFSA=phraseAutomatonFSA; + if (phraseAutomatonFSA == null) throw new NullPointerException("FSA is null"); + this.ignorePluralForm = ignorePluralForm; + phraseFSA = phraseAutomatonFSA; } public boolean isEmpty() { return phraseFSA == null; } diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java index 318912eab04..eb31b75cd6f 100644 --- a/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java @@ -359,7 +359,7 @@ public class StemmingSearcher extends Searcher { default: throw new IllegalArgumentException("Unknown segmenting rule: " + current.getSegmentingRule() + ". This is a bug in Vespa, as the implementation has gotten out of sync." + - " Please create a ticket as soon as possible."); + " Please create an issue."); } } diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/PosSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/PosSearcher.java index 37561d3a0f5..7063a14a389 100644 --- a/container-search/src/main/java/com/yahoo/prelude/searcher/PosSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/searcher/PosSearcher.java @@ -75,7 +75,7 @@ public class PosSearcher extends Searcher { loc.setAttribute(posAttribute); try { - if (ll == null && xy == null && bb != null) { + if (ll == null && xy == null) { parseBoundingBox(bb, loc); } else { if (ll != null && xy != null) { diff --git a/container-search/src/main/java/com/yahoo/search/Query.java b/container-search/src/main/java/com/yahoo/search/Query.java index 9e3f6d20e36..4995927f7a2 100644 --- a/container-search/src/main/java/com/yahoo/search/Query.java +++ b/container-search/src/main/java/com/yahoo/search/Query.java @@ -10,7 +10,6 @@ import com.yahoo.fs4.MapEncoder; import java.util.logging.Level; import com.yahoo.prelude.fastsearch.DocumentDatabase; import com.yahoo.prelude.query.Highlight; -import com.yahoo.prelude.query.QueryException; import com.yahoo.prelude.query.textualrepresentation.TextualQueryRepresentation; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.dispatch.Dispatcher; @@ -413,11 +412,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { if (field.getType() == FieldType.genericQueryProfileType) { // Generic map CompoundName fullName = prefix.append(field.getName()); for (Map.Entry<String, Object> entry : originalProperties.listProperties(fullName, context).entrySet()) { - try { - properties().set(fullName.append(entry.getKey()), entry.getValue(), context); - } catch (IllegalArgumentException e) { - throw new QueryException("Invalid request parameter", e); - } + properties().set(fullName.append(entry.getKey()), entry.getValue(), context); } } else if (field.getType() instanceof QueryProfileFieldType) { // Nested arguments @@ -427,11 +422,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { CompoundName fullName = prefix.append(field.getName()); Object value = originalProperties.get(fullName, context); if (value != null) { - try { - properties().set(fullName, value, context); - } catch (IllegalArgumentException e) { - throw new QueryException("Invalid request parameter", e); - } + properties().set(fullName, value, context); } } } @@ -440,13 +431,8 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { /** Calls properties.set on all entries in requestMap */ private void setPropertiesFromRequestMap(Map<String, String> requestMap, Properties properties, boolean ignoreSelect) { for (var entry : requestMap.entrySet()) { - try { - if (ignoreSelect && entry.getKey().equals(Select.SELECT)) continue; - properties.set(entry.getKey(), entry.getValue(), requestMap); - } - catch (IllegalArgumentException e) { - throw new QueryException("Invalid request parameter", e); - } + if (ignoreSelect && entry.getKey().equals(Select.SELECT)) continue; + properties.set(entry.getKey(), entry.getValue(), requestMap); } } diff --git a/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java b/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java index 60c5d42c531..0c38b38179d 100644 --- a/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java @@ -14,6 +14,7 @@ import com.yahoo.concurrent.CopyOnWriteHashMap; import com.yahoo.errorhandling.Results; import com.yahoo.errorhandling.Results.Builder; import com.yahoo.prelude.IndexFacts; +import com.yahoo.processing.IllegalInputException; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.Query; import com.yahoo.search.Result; @@ -545,8 +546,8 @@ public class FederationSearcher extends ForkingSearcher { private ComponentSpecification asSourceSpec(String source) { try { return new ComponentSpecification(source); - } catch(Exception e) { - throw new IllegalArgumentException("The source ref '" + source + "' used for federation is not valid.", e); + } catch (Exception e) { + throw new IllegalInputException("The source ref '" + source + "' used for federation is not valid.", e); } } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/GroupingQueryParser.java b/container-search/src/main/java/com/yahoo/search/grouping/GroupingQueryParser.java index 9924a05bb46..b9e0825ab03 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/GroupingQueryParser.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/GroupingQueryParser.java @@ -12,8 +12,14 @@ import com.yahoo.search.grouping.request.GroupingOperation; import com.yahoo.search.query.Select; import com.yahoo.search.searchchain.Execution; import com.yahoo.search.searchchain.PhaseNames; +import com.yahoo.processing.IllegalInputException; -import java.util.*; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.TimeZone; /** * This searcher is responsible for turning the "select" parameter into a corresponding {@link GroupingRequest}. It will @@ -35,19 +41,23 @@ public class GroupingQueryParser extends Searcher { @Override public Result search(Query query, Execution execution) { - String reqParam = query.properties().getString(PARAM_REQUEST); - if (reqParam == null) { + try { + String reqParam = query.properties().getString(PARAM_REQUEST); + if (reqParam == null) return execution.search(query); + + List<Continuation> continuations = getContinuations(query.properties().getString(PARAM_CONTINUE)); + TimeZone zone = getTimeZone(query.properties().getString(PARAM_TIMEZONE, "utc")); + for (GroupingOperation op : GroupingOperation.fromStringAsList(reqParam)) { + GroupingRequest grpRequest = GroupingRequest.newInstance(query); + grpRequest.setRootOperation(op); + grpRequest.setTimeZone(zone); + grpRequest.continuations().addAll(continuations); + } return execution.search(query); } - List<Continuation> continuations = getContinuations(query.properties().getString(PARAM_CONTINUE)); - TimeZone zone = getTimeZone(query.properties().getString(PARAM_TIMEZONE, "utc")); - for (GroupingOperation op : GroupingOperation.fromStringAsList(reqParam)) { - GroupingRequest grpRequest = GroupingRequest.newInstance(query); - grpRequest.setRootOperation(op); - grpRequest.setTimeZone(zone); - grpRequest.continuations().addAll(continuations); + catch (IllegalArgumentException e) { + throw new IllegalInputException(e); } - return execution.search(query); } private List<Continuation> getContinuations(String param) { diff --git a/container-search/src/main/java/com/yahoo/search/grouping/GroupingValidator.java b/container-search/src/main/java/com/yahoo/search/grouping/GroupingValidator.java index 06b030dbc78..cd8578cd728 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/GroupingValidator.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/GroupingValidator.java @@ -5,6 +5,7 @@ import com.google.inject.Inject; import com.yahoo.component.chain.dependencies.After; import com.yahoo.component.chain.dependencies.Before; import com.yahoo.component.chain.dependencies.Provides; +import com.yahoo.processing.IllegalInputException; import com.yahoo.search.grouping.request.AttributeMapLookupValue; import com.yahoo.vespa.config.search.AttributesConfig; import com.yahoo.container.QrSearchersConfig; @@ -80,14 +81,14 @@ public class GroupingValidator extends Searcher { AttributesConfig.Attribute keyAttribute = attributes.get(keyAttributeName); AttributesConfig.Attribute keySourceAttribute = attributes.get(keySourceAttributeName); if (!keySourceAttribute.datatype().equals(keyAttribute.datatype())) { - throw new IllegalArgumentException("Grouping request references key source attribute '" + - keySourceAttributeName + "' with data type '" + keySourceAttribute.datatype() + - "' that is different than data type '" + keyAttribute.datatype() + "' of key attribute '" + - keyAttributeName + "'"); + throw new IllegalInputException("Grouping request references key source attribute '" + + keySourceAttributeName + "' with data type '" + keySourceAttribute.datatype() + + "' that is different than data type '" + keyAttribute.datatype() + "' of key attribute '" + + keyAttributeName + "'"); } if (!keySourceAttribute.collectiontype().equals(AttributesConfig.Attribute.Collectiontype.Enum.SINGLE)) { - throw new IllegalArgumentException("Grouping request references key source attribute '" + - keySourceAttributeName + "' which is not of single value type"); + throw new IllegalInputException("Grouping request references key source attribute '" + + keySourceAttributeName + "' which is not of single value type"); } } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/AddFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/AddFunction.java index 420861d2f6c..60805aacd5f 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/AddFunction.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/AddFunction.java @@ -38,9 +38,9 @@ public class AddFunction extends FunctionNode { /** * Constructs a new instance of this class from a list of arguments. * - * @param args The arguments to pass to the constructor. - * @return The created instance. - * @throws IllegalArgumentException Thrown if the number of arguments is less than 2. + * @param args the arguments to pass to the constructor. + * @return the created instance. + * @throws IllegalArgumentException thrown if the number of arguments is less than 2. */ public static AddFunction newInstance(List<GroupingExpression> args) { if (args.size() < 2) { diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/BucketResolver.java b/container-search/src/main/java/com/yahoo/search/grouping/request/BucketResolver.java index c36c8af5c34..6c6b20973ab 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/BucketResolver.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/BucketResolver.java @@ -24,9 +24,9 @@ public class BucketResolver { * Pushes the given expression onto this bucket resolver. Once all buckets have been pushed using this method, call * {@link #resolve(GroupingExpression)} to retrieve to combined grouping expression. * - * @param val The expression to push. - * @param inclusive Whether or not the value is inclusive or not. - * @throws IllegalArgumentException Thrown if the expression is incompatible. + * @param val the expression to push + * @param inclusive whether or not the value is inclusive or not + * @throws IllegalArgumentException thrown if the expression is incompatible */ public BucketResolver push(ConstantValue<?> val, boolean inclusive) { if (prev == null) { diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/GroupingOperation.java b/container-search/src/main/java/com/yahoo/search/grouping/request/GroupingOperation.java index c825f3c61de..8b73fa01128 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/GroupingOperation.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/GroupingOperation.java @@ -228,8 +228,8 @@ public abstract class GroupingOperation extends GroupingNode { * method verifies the input level against the operation type, and recursively resolves the level of all argument * expressions. * - * @param level The level of the input data. - * @throws IllegalArgumentException Thrown if a contained expression is invalid for the given level. + * @param level the level of the input data + * @throws IllegalArgumentException thrown if a contained expression is invalid for the given level */ public void resolveLevel(int level) { if (groupBy != null) { @@ -322,12 +322,7 @@ public abstract class GroupingOperation extends GroupingNode { return this; } - /** - * Return the accuracy of this. - * - * @return The accuracy value. - * @see #setAccuracy(double) - */ + /** Return the accuracy of this. */ public double getAccuracy() { return accuracy; } @@ -335,8 +330,8 @@ public abstract class GroupingOperation extends GroupingNode { /** * Adds an expression to the order-by clause of this operation. * - * @param exp The expressions to add to this. - * @return This, to allow chaining. + * @param exp the expressions to add to this + * @return this, to allow chaining */ public GroupingOperation addOrderBy(GroupingExpression exp) { orderBy.add(exp); @@ -346,11 +341,11 @@ public abstract class GroupingOperation extends GroupingNode { /** * Convenience method to call {@link #addOrderBy(GroupingExpression)} for each element in the given list. * - * @param lst The list of expressions to add. - * @return This, to allow chaining. + * @param list the list of expressions to add + * @return this, to allow chaining */ - public GroupingOperation addOrderBy(List<GroupingExpression> lst) { - for (GroupingExpression exp : lst) { + public GroupingOperation addOrderBy(List<GroupingExpression> list) { + for (GroupingExpression exp : list) { addOrderBy(exp); } return this; @@ -359,7 +354,7 @@ public abstract class GroupingOperation extends GroupingNode { /** * Returns the number of expressions in the order-by clause of this. * - * @return The expression count. + * @return the expression count */ public int getNumOrderBy() { return orderBy.size(); @@ -368,9 +363,9 @@ public abstract class GroupingOperation extends GroupingNode { /** * Returns the group-by expression at the given index. * - * @param i The index of the expression to return. - * @return The expression at the given index. - * @throws IndexOutOfBoundsException If the index is out of range. + * @param i the index of the expression to return + * @return the expression at the given index + * @throws IndexOutOfBoundsException if the index is out of range */ public GroupingExpression getOrderBy(int i) { return orderBy.get(i); @@ -379,7 +374,7 @@ public abstract class GroupingOperation extends GroupingNode { /** * Returns an immutable view to the order-by clause of this. * - * @return The expression list. + * @return the expression list */ public List<GroupingExpression> getOrderBy() { return Collections.unmodifiableList(orderBy); @@ -388,8 +383,8 @@ public abstract class GroupingOperation extends GroupingNode { /** * Adds an expression to the output clause of this operation. * - * @param exp The expressions to add to this. - * @return This, to allow chaining. + * @param exp the expressions to add to this + * @return this, to allow chaining */ public GroupingOperation addOutput(GroupingExpression exp) { outputs.add(exp); @@ -399,8 +394,8 @@ public abstract class GroupingOperation extends GroupingNode { /** * Convenience method to call {@link #addOutput(GroupingExpression)} for each element in the given list. * - * @param lst The list of expressions to add. - * @return This, to allow chaining. + * @param lst the list of expressions to add + * @return this, to allow chaining */ public GroupingOperation addOutputs(List<GroupingExpression> lst) { for (GroupingExpression exp : lst) { @@ -412,7 +407,7 @@ public abstract class GroupingOperation extends GroupingNode { /** * Returns the number of expressions in the output clause of this. * - * @return The expression count. + * @return the expression count */ public int getNumOutputs() { return outputs.size(); @@ -421,9 +416,9 @@ public abstract class GroupingOperation extends GroupingNode { /** * Returns the output expression at the given index. * - * @param i The index of the expression to return. - * @return The expression at the given index. - * @throws IndexOutOfBoundsException If the index is out of range. + * @param i the index of the expression to return + * @return the expression at the given index + * @throws IndexOutOfBoundsException If the index is out of range */ public GroupingExpression getOutput(int i) { return outputs.get(i); @@ -432,7 +427,7 @@ public abstract class GroupingOperation extends GroupingNode { /** * Returns an immutable view to the output clause of this. * - * @return The expression list. + * @return the expression list */ public List<GroupingExpression> getOutputs() { return Collections.unmodifiableList(outputs); @@ -443,8 +438,8 @@ public abstract class GroupingOperation extends GroupingNode { * during expression evaluation to give the dispatch-node more data to consider when selecting the N groups that are * to be evaluated further. * - * @param precision The precision to set. - * @return This, to allow chaining. + * @param precision the precision to set + * @return this, to allow chaining * @see #setMax(int) */ public GroupingOperation setPrecision(int precision) { @@ -452,11 +447,7 @@ public abstract class GroupingOperation extends GroupingNode { return this; } - /** - * Returns the precision clause of this. - * - * @return The precision. - */ + /** Returns the precision clause of this. */ public int getPrecision() { return precision; } @@ -464,11 +455,11 @@ public abstract class GroupingOperation extends GroupingNode { /** * Assigns a string as the where clause of this operation. * - * @param str The string to assign to this. - * @return This, to allow chaining. + * @param string the string to assign to this + * @return this, to allow chaining */ - public GroupingOperation setWhere(String str) { - where = str; + public GroupingOperation setWhere(String string) { + where = string; return this; } @@ -590,9 +581,9 @@ public abstract class GroupingOperation extends GroupingNode { * Convenience method to call {@link #fromStringAsList(String)} and assert that the list contains exactly one * grouping operation. * - * @param str The string to parse. - * @return A grouping operation that corresponds to the string. - * @throws IllegalArgumentException Thrown if the string could not be parsed as a single operation. + * @param str the string to parse + * @return a grouping operation that corresponds to the string + * @throws IllegalArgumentException thrown if the string could not be parsed as a single operation */ public static GroupingOperation fromString(String str) { List<GroupingOperation> lst = fromStringAsList(str); @@ -606,15 +597,15 @@ public abstract class GroupingOperation extends GroupingNode { * Parses the given string as a list of grouping operations. This method never returns null, it either returns a * list of valid grouping requests or it throws an exception. * - * @param str The string to parse. - * @return A list of grouping operations that corresponds to the string. - * @throws IllegalArgumentException Thrown if the string could not be parsed. + * @param string the string to parse + * @return a list of grouping operations that corresponds to the string + * @throws IllegalArgumentException thrown if the string could not be parsed */ - public static List<GroupingOperation> fromStringAsList(String str) { - if (str == null || str.trim().length() == 0) { + public static List<GroupingOperation> fromStringAsList(String string) { + if (string == null || string.trim().length() == 0) { return Collections.emptyList(); } - GroupingParserInput input = new GroupingParserInput(str); + GroupingParserInput input = new GroupingParserInput(string); try { return new GroupingParser(input).requestList(); } catch (ParseException | TokenMgrException e) { diff --git a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java index c658d404adb..b0c8fbba059 100644 --- a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java +++ b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java @@ -21,8 +21,8 @@ import com.yahoo.language.Linguistics; import java.util.logging.Level; import com.yahoo.net.HostName; import com.yahoo.net.UriTools; -import com.yahoo.prelude.query.QueryException; import com.yahoo.prelude.query.parser.ParseException; +import com.yahoo.processing.IllegalInputException; import com.yahoo.processing.rendering.Renderer; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.query.context.QueryContext; @@ -61,7 +61,6 @@ import java.util.concurrent.Executor; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; -import java.util.logging.Level; import java.util.logging.Logger; /** @@ -229,10 +228,8 @@ public class SearchHandler extends LoggingRequestHandler { try { try { return handleBody(request); - } catch (QueryException e) { - return (e.getCause() instanceof IllegalArgumentException) - ? invalidParameterResponse(request, e) - : illegalQueryResponse(request, e); + } catch (IllegalInputException e) { + return illegalQueryResponse(request, e); } catch (RuntimeException e) { // Make sure we generate a valid response even on unexpected errors log.log(Level.WARNING, "Failed handling " + request, e); return internalServerErrorResponse(request, e); @@ -263,10 +260,6 @@ public class SearchHandler extends LoggingRequestHandler { return new HttpSearchResponse(getHttpResponseStatus(request, result), result, query, renderer); } - private HttpResponse invalidParameterResponse(HttpRequest request, RuntimeException e) { - return errorResponse(request, ErrorMessage.createInvalidQueryParameter(Exceptions.toMessageString(e))); - } - private HttpResponse illegalQueryResponse(HttpRequest request, RuntimeException e) { return errorResponse(request, ErrorMessage.createIllegalQuery(Exceptions.toMessageString(e))); } @@ -275,7 +268,6 @@ public class SearchHandler extends LoggingRequestHandler { return errorResponse(request, ErrorMessage.createInternalServerError(Exceptions.toMessageString(e))); } - private HttpSearchResponse handleBody(HttpRequest request) { Map<String, String> requestMap = requestMapFromRequest(request); @@ -450,19 +442,15 @@ public class SearchHandler extends LoggingRequestHandler { + Exceptions.toMessageString(e)); log.log(Level.FINE, error::getDetailedMessage); return new Result(query, error); + } catch (IllegalInputException e) { + ErrorMessage error = ErrorMessage.createBadRequest("Invalid request [" + request + "]: " + + Exceptions.toMessageString(e)); + log.log(Level.FINE, error::getDetailedMessage); + return new Result(query, error); } catch (IllegalArgumentException e) { - if ("Comparison method violates its general contract!".equals(e.getMessage())) { - // This is an error in application components or Vespa code - log(request, query, e); - return new Result(query, ErrorMessage.createUnspecifiedError("Failed searching: " + - Exceptions.toMessageString(e), e)); - } - else { - ErrorMessage error = ErrorMessage.createBadRequest("Invalid search request [" + request + "]: " - + Exceptions.toMessageString(e)); - log.log(Level.FINE, error::getDetailedMessage); - return new Result(query, error); - } + log(request, query, e); + return new Result(query, ErrorMessage.createUnspecifiedError("Failed: " + + Exceptions.toMessageString(e), e)); } catch (LinkageError | StackOverflowError e) { // LinkageError should have been an Exception in an OSGi world - typical bundle dependency issue problem // StackOverflowError is recoverable @@ -472,7 +460,7 @@ public class SearchHandler extends LoggingRequestHandler { return new Result(query, error); } catch (Exception e) { log(request, query, e); - return new Result(query, ErrorMessage.createUnspecifiedError("Failed searching: " + + return new Result(query, ErrorMessage.createUnspecifiedError("Failed: " + Exceptions.toMessageString(e), e)); } } @@ -579,8 +567,8 @@ public class SearchHandler extends LoggingRequestHandler { byte[] byteArray = IOUtils.readBytes(request.getData(), 1 << 20); inspector = SlimeUtils.jsonToSlime(byteArray).get(); if (inspector.field("error_message").valid()) { - throw new QueryException("Illegal query: " + inspector.field("error_message").asString() + " at: '" + - new String(inspector.field("offending_input").asData(), StandardCharsets.UTF_8) + "'"); + throw new IllegalInputException("Illegal query: " + inspector.field("error_message").asString() + " at: '" + + new String(inspector.field("offending_input").asData(), StandardCharsets.UTF_8) + "'"); } } catch (IOException e) { @@ -593,9 +581,9 @@ public class SearchHandler extends LoggingRequestHandler { requestMap.putAll(request.propertyMap()); if (requestMap.containsKey("yql") && (requestMap.containsKey("select.where") || requestMap.containsKey("select.grouping")) ) - throw new QueryException("Illegal query: Query contains both yql and select parameter"); + throw new IllegalInputException("Illegal query: Query contains both yql and select parameter"); if (requestMap.containsKey("query") && (requestMap.containsKey("select.where") || requestMap.containsKey("select.grouping")) ) - throw new QueryException("Illegal query: Query contains both query and select parameter"); + throw new IllegalInputException("Illegal query: Query contains both query and select parameter"); return requestMap; } diff --git a/container-search/src/main/java/com/yahoo/search/pagetemplates/PageTemplateSearcher.java b/container-search/src/main/java/com/yahoo/search/pagetemplates/PageTemplateSearcher.java index 0ec04bf99de..2074fce19bd 100644 --- a/container-search/src/main/java/com/yahoo/search/pagetemplates/PageTemplateSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/pagetemplates/PageTemplateSearcher.java @@ -2,9 +2,9 @@ package com.yahoo.search.pagetemplates; import com.google.inject.Inject; -import com.yahoo.component.ComponentId; import com.yahoo.component.chain.dependencies.Provides; import com.yahoo.component.provider.ComponentRegistry; +import com.yahoo.processing.IllegalInputException; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; @@ -13,8 +13,6 @@ import com.yahoo.search.pagetemplates.config.PageTemplateConfigurer; import com.yahoo.search.pagetemplates.engine.Organizer; import com.yahoo.search.pagetemplates.engine.Resolution; import com.yahoo.search.pagetemplates.engine.Resolver; -import com.yahoo.search.pagetemplates.engine.resolvers.DeterministicResolver; -import com.yahoo.search.pagetemplates.engine.resolvers.RandomResolver; import com.yahoo.search.pagetemplates.engine.resolvers.ResolverRegistry; import com.yahoo.search.pagetemplates.model.Choice; import com.yahoo.search.pagetemplates.model.PageElement; @@ -23,7 +21,13 @@ import com.yahoo.processing.request.CompoundName; import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.searchchain.Execution; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.Map; /** * Enables page optimization templates. @@ -107,7 +111,7 @@ public class PageTemplateSearcher extends Searcher { @Override public Result search(Query query, Execution execution) { // Pre execution: Choose template and sources - List<PageElement> pages=selectPageTemplates(query); + List<PageElement> pages = selectPageTemplates(query); if (pages.isEmpty()) return execution.search(query); // Bypass if no page template chosen addSources(pages,query); @@ -115,12 +119,12 @@ public class PageTemplateSearcher extends Searcher { query.properties().set(pagePageTemplateListName, pages); // Execute - Result result=execution.search(query); + Result result = execution.search(query); // Post execution: Resolve choices and organize the result as dictated by the resolved template - Choice pageTemplateChoice=Choice.createSingletons(pages); - Resolution resolution=selectResolver(query).resolve(pageTemplateChoice,query,result); - organizer.organize(pageTemplateChoice,resolution,result); + Choice pageTemplateChoice = Choice.createSingletons(pages); + Resolution resolution = selectResolver(query).resolve(pageTemplateChoice, query, result); + organizer.organize(pageTemplateChoice, resolution, result); return result; } @@ -132,23 +136,23 @@ public class PageTemplateSearcher extends Searcher { // Determine the list of page template ids @SuppressWarnings("unchecked") List<String> pageIds = (List<String>) query.properties().get(pageIdListName); - if (pageIds==null) { - String pageIdString=query.properties().getString(pageIdName,"").trim(); - if (pageIdString.length()>0) - pageIds=Arrays.asList(pageIdString.split(" ")); + if (pageIds == null) { + String pageIdString = query.properties().getString(pageIdName,"").trim(); + if (pageIdString.length() > 0) + pageIds = Arrays.asList(pageIdString.split(" ")); } // If none set, just return the default or null if none - if (pageIds==null) { + if (pageIds == null) { PageElement defaultPage=templateRegistry.getComponent("default"); - return (defaultPage==null ? Collections.<PageElement>emptyList() : Collections.singletonList(defaultPage)); + return (defaultPage == null ? Collections.<PageElement>emptyList() : Collections.singletonList(defaultPage)); } // Resolve the id list to page templates - List<PageElement> pages=new ArrayList<>(pageIds.size()); + List<PageElement> pages = new ArrayList<>(pageIds.size()); for (String pageId : pageIds) { - PageTemplate page=templateRegistry.getComponent(pageId); - if (page==null) + PageTemplate page = templateRegistry.getComponent(pageId); + if (page == null) query.errors().add(ErrorMessage.createInvalidQueryParameter("Could not resolve requested page template '" + pageId + "'")); else @@ -159,17 +163,17 @@ public class PageTemplateSearcher extends Searcher { } private Resolver selectResolver(Query query) { - String resolverId=query.properties().getString(pageResolverName); - if (resolverId==null) return resolverRegistry.defaultResolver(); - Resolver resolver=resolverRegistry.getComponent(resolverId); - if (resolver==null) throw new IllegalArgumentException("No page template resolver '" + resolverId + "'"); + String resolverId = query.properties().getString(pageResolverName); + if (resolverId == null) return resolverRegistry.defaultResolver(); + Resolver resolver = resolverRegistry.getComponent(resolverId); + if (resolver == null) throw new IllegalInputException("No page template resolver '" + resolverId + "'"); return resolver; } /** Sets query.getModel().getSources() to the right value and add source parameters specified in templates */ - private void addSources(List<PageElement> pages,Query query) { + private void addSources(List<PageElement> pages, Query query) { // Determine all wanted sources - Set<Source> pageSources=new HashSet<>(); + Set<Source> pageSources = new HashSet<>(); for (PageElement page : pages) pageSources.addAll(((PageTemplate)page).getSources()); @@ -177,34 +181,34 @@ public class PageTemplateSearcher extends Searcher { if (query.getModel().getSources().size() > 0) { // Add properties if the source list is set explicitly, but do not modify otherwise - addParametersForIncludedSources(pageSources,query); + addParametersForIncludedSources(pageSources, query); return; } if (pageSources.contains(Source.any)) { - IntentModel intentModel=IntentModel.getFrom(query); - if (intentModel!=null) { + IntentModel intentModel = IntentModel.getFrom(query); + if (intentModel != null) { query.getModel().getSources().addAll(intentModel.getSourceNames()); - addPageTemplateSources(pageSources,query); + addPageTemplateSources(pageSources, query); } // otherwise leave empty to search all } else { // Let the page templates decide - addPageTemplateSources(pageSources,query); + addPageTemplateSources(pageSources, query); } } private void addPageTemplateSources(Set<Source> pageSources,Query query) { for (Source pageSource : pageSources) { - if (pageSource==Source.any) continue; + if (pageSource == Source.any) continue; query.getModel().getSources().add(pageSource.getName()); addParameters(pageSource,query); } } - private void addParametersForIncludedSources(Set<Source> sources,Query query) { + private void addParametersForIncludedSources(Set<Source> sources, Query query) { for (Source source : sources) { - if (source.parameters().size()>0 && query.getModel().getSources().contains(source.getName())) + if (source.parameters().size() > 0 && query.getModel().getSources().contains(source.getName())) addParameters(source,query); } } @@ -220,8 +224,8 @@ public class PageTemplateSearcher extends Searcher { * is not supported. (Same parameter sets in multiple templates is supported, * and will be just one entry in this set). */ - private void addErrorIfSameSourceMultipleTimes(List<PageElement> pages,Set<Source> sources,Query query) { - Set<String> sourceNames=new HashSet<>(); + private void addErrorIfSameSourceMultipleTimes(List<PageElement> pages, Set<Source> sources, Query query) { + Set<String> sourceNames = new HashSet<>(); for (Source source : sources) { if (sourceNames.contains(source.getName())) query.errors().add(ErrorMessage.createInvalidQueryParameter( diff --git a/container-search/src/main/java/com/yahoo/search/pagetemplates/PlaceholderMappingVisitor.java b/container-search/src/main/java/com/yahoo/search/pagetemplates/PlaceholderMappingVisitor.java index c29e9615fe8..c37ea5667c0 100644 --- a/container-search/src/main/java/com/yahoo/search/pagetemplates/PlaceholderMappingVisitor.java +++ b/container-search/src/main/java/com/yahoo/search/pagetemplates/PlaceholderMappingVisitor.java @@ -19,16 +19,16 @@ import java.util.Map; */ class PlaceholderMappingVisitor extends PageTemplateVisitor { - private Map<String, MapChoice> placeholderIdToChoice=new LinkedHashMap<>(); + private final Map<String, MapChoice> placeholderIdToChoice = new LinkedHashMap<>(); @Override public void visit(MapChoice mapChoice) { - List<String> placeholderIds=mapChoice.placeholderIds(); + List<String> placeholderIds = mapChoice.placeholderIds(); for (String placeholderId : placeholderIds) { - MapChoice existingChoice=placeholderIdToChoice.put(placeholderId,mapChoice); - if (existingChoice!=null) + MapChoice existingChoice = placeholderIdToChoice.put(placeholderId,mapChoice); + if (existingChoice != null) throw new IllegalArgumentException("placeholder id '" + placeholderId + "' is referenced by both " + - mapChoice + " and " + existingChoice + ": Only one reference is allowed"); + mapChoice + " and " + existingChoice + ": Only one reference is allowed"); } } diff --git a/container-search/src/main/java/com/yahoo/search/pagetemplates/PlaceholderReferenceCreatingVisitor.java b/container-search/src/main/java/com/yahoo/search/pagetemplates/PlaceholderReferenceCreatingVisitor.java index e8870f4f11a..5ef507201cb 100644 --- a/container-search/src/main/java/com/yahoo/search/pagetemplates/PlaceholderReferenceCreatingVisitor.java +++ b/container-search/src/main/java/com/yahoo/search/pagetemplates/PlaceholderReferenceCreatingVisitor.java @@ -1,9 +1,10 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.pagetemplates; -import com.yahoo.search.pagetemplates.model.*; +import com.yahoo.search.pagetemplates.model.MapChoice; +import com.yahoo.search.pagetemplates.model.PageTemplateVisitor; +import com.yahoo.search.pagetemplates.model.Placeholder; -import java.util.HashMap; import java.util.Map; /** @@ -14,16 +15,16 @@ import java.util.Map; */ class PlaceholderReferenceCreatingVisitor extends PageTemplateVisitor { - private Map<String, MapChoice> placeholderIdToChoice=new HashMap<>(); + private final Map<String, MapChoice> placeholderIdToChoice; public PlaceholderReferenceCreatingVisitor(Map<String, MapChoice> placeholderIdToChoice) { - this.placeholderIdToChoice=placeholderIdToChoice; + this.placeholderIdToChoice = placeholderIdToChoice; } @Override public void visit(Placeholder placeholder) { - MapChoice choice=placeholderIdToChoice.get(placeholder.getId()); - if (choice==null) + MapChoice choice = placeholderIdToChoice.get(placeholder.getId()); + if (choice == null) throw new IllegalArgumentException(placeholder + " is not referenced by any choice"); placeholder.setValueContainer(choice); } diff --git a/container-search/src/main/java/com/yahoo/search/pagetemplates/config/PageTemplateXMLReader.java b/container-search/src/main/java/com/yahoo/search/pagetemplates/config/PageTemplateXMLReader.java index beffd12b22a..0359432a819 100644 --- a/container-search/src/main/java/com/yahoo/search/pagetemplates/config/PageTemplateXMLReader.java +++ b/container-search/src/main/java/com/yahoo/search/pagetemplates/config/PageTemplateXMLReader.java @@ -38,18 +38,18 @@ public class PageTemplateXMLReader { * @throws RuntimeException if <code>directory</code> is not a readable directory, or if there is some error in the XML */ public PageTemplateRegistry read(String directory) { - List<NamedReader> pageReaders=new ArrayList<>(); + List<NamedReader> pageReaders = new ArrayList<>(); try { - File dir=new File(directory); - if ( !dir.isDirectory() ) throw new IllegalArgumentException("Could not read page templates: '" + - directory + "' is not a valid directory."); + File dir = new File(directory); + if ( ! dir.isDirectory() ) throw new IllegalArgumentException("Could not read page templates: '" + + directory + "' is not a valid directory."); for (File file : sortFiles(dir)) { if ( ! file.getName().endsWith(".xml")) continue; - pageReaders.add(new NamedReader(file.getName(),new FileReader(file))); + pageReaders.add(new NamedReader(file.getName(), new FileReader(file))); } - return read(pageReaders,true); + return read(pageReaders, true); } catch (IOException e) { throw new IllegalArgumentException("Could not read page templates from '" + directory + "'",e); @@ -67,18 +67,18 @@ public class PageTemplateXMLReader { * @throws RuntimeException if <code>fileName</code> is not a readable file, or if there is some error in the XML */ public PageTemplate readFile(String fileName) { - NamedReader pageReader=null; + NamedReader pageReader = null; try { - File file=new File(fileName); - pageReader=new NamedReader(fileName,new FileReader(file)); - String firstName=file.getName().substring(0,file.getName().length()-4); - return read(Collections.singletonList(pageReader),true).getComponent(firstName); + File file = new File(fileName); + pageReader = new NamedReader(fileName,new FileReader(file)); + String firstName = file.getName().substring(0, file.getName().length() - 4); + return read(Collections.singletonList(pageReader), true).getComponent(firstName); } catch (IOException e) { - throw new IllegalArgumentException("Could not read the page template '" + fileName + "'",e); + throw new IllegalArgumentException("Could not read the page template '" + fileName + "'", e); } finally { - if (pageReader!=null) + if (pageReader != null) try { pageReader.close(); } catch (IOException e) { } } } @@ -130,11 +130,11 @@ public class PageTemplateXMLReader { } /** Throws an exception if the name is not corresponding to the id */ - private void validateFileName(final String actualName,ComponentId id,String artifactName) { - String expectedCanonicalFileName=id.toFileName(); - String fileName=new File(actualName).getName(); - fileName=stripXmlEnding(fileName); - String canonicalFileName=ComponentId.fromFileName(fileName).toFileName(); + private void validateFileName(String actualName, ComponentId id, String artifactName) { + String expectedCanonicalFileName = id.toFileName(); + String fileName = new File(actualName).getName(); + fileName = stripXmlEnding(fileName); + String canonicalFileName = ComponentId.fromFileName(fileName).toFileName(); if ( ! canonicalFileName.equals(expectedCanonicalFileName)) throw new IllegalArgumentException("The file name of " + artifactName + " '" + id + "' must be '" + expectedCanonicalFileName + ".xml' but was '" + actualName + "'"); @@ -144,14 +144,14 @@ public class PageTemplateXMLReader { if (!fileName.endsWith(".xml")) throw new IllegalArgumentException("'" + fileName + "' should have a .xml ending"); else - return fileName.substring(0,fileName.length()-4); + return fileName.substring(0, fileName.length() - 4); } private void readPages() { for (Map.Entry<ComponentId,Element> pageElement : pageElementsByPageId.entrySet()) { try { - PageTemplate page=registry.getComponent(pageElement.getValue().getAttribute("id")); - readPageContent(pageElement.getValue(),page); + PageTemplate page = registry.getComponent(pageElement.getValue().getAttribute("id")); + readPageContent(pageElement.getValue(), page); } catch (IllegalArgumentException e) { throw new IllegalArgumentException("Could not read page template '" + pageElement.getKey() + "'",e); @@ -159,16 +159,16 @@ public class PageTemplateXMLReader { } } - private void readPageContent(Element pageElement,PageTemplate page) { + private void readPageContent(Element pageElement, PageTemplate page) { if (page.isFrozen()) return; // Already read - Section rootSection=new Section(page.getId().toString()); - readSection(pageElement,rootSection); + Section rootSection = new Section(page.getId().toString()); + readSection(pageElement, rootSection); page.setSection(rootSection); page.freeze(); } /** Fills a section with attributes and sub-elements from a "section" or "page" element */ - private Section readSection(Element sectionElement,Section section) { + private Section readSection(Element sectionElement, Section section) { section.setLayout(Layout.fromString(sectionElement.getAttribute("layout"))); section.setRegion(sectionElement.getAttribute("region")); section.setOrder(Sorting.fromString(sectionElement.getAttribute("order"))); @@ -198,10 +198,10 @@ public class PageTemplateXMLReader { /** Reads the direct descendant elements of an include */ private List<PageElement> readInclude(Element element) { - PageTemplate included=registry.getComponent(element.getAttribute("idref")); - if (included==null) + PageTemplate included = registry.getComponent(element.getAttribute("idref")); + if (included == null) throw new IllegalArgumentException("Could not find page template '" + element.getAttribute("idref")); - readPageContent(pageElementsByPageId.get(included.getId()),included); + readPageContent(pageElementsByPageId.get(included.getId()), included); return included.getSection().elements(Section.class); } @@ -223,9 +223,9 @@ public class PageTemplateXMLReader { } private List<Source> readSourceAttribute(Element sectionElement) { - List<Source> sources=new ArrayList<>(); - String sourceAttributeString=sectionElement.getAttribute("source"); - if (sourceAttributeString!=null) { + List<Source> sources = new ArrayList<>(); + String sourceAttributeString = sectionElement.getAttribute("source"); + if (sourceAttributeString != null) { for (String sourceName : sourceAttributeString.split(" ")) { if (sourceName.isEmpty()) continue; if ("*".equals(sourceName)) diff --git a/container-search/src/main/java/com/yahoo/search/pagetemplates/engine/Organizer.java b/container-search/src/main/java/com/yahoo/search/pagetemplates/engine/Organizer.java index 3e6e82a5584..051103cba06 100644 --- a/container-search/src/main/java/com/yahoo/search/pagetemplates/engine/Organizer.java +++ b/container-search/src/main/java/com/yahoo/search/pagetemplates/engine/Organizer.java @@ -9,9 +9,7 @@ import com.yahoo.search.query.Sorting; import com.yahoo.search.result.*; import java.util.ArrayList; -import java.util.Iterator; import java.util.List; -import java.util.Map; /** * Reorganizes and prunes a result as prescribed by a resolved template. @@ -29,13 +27,13 @@ public class Organizer { * @param result the result to organize */ public void organize(Choice templateChoice, Resolution resolution, Result result) { - PageTemplate template=(PageTemplate)templateChoice.get(resolution.getResolution(templateChoice)).get(0); - SectionHitGroup sectionGroup =toGroup(template.getSection(),resolution,result); - ErrorHit errors=result.hits().getErrorHit(); + PageTemplate template = (PageTemplate)templateChoice.get(resolution.getResolution(templateChoice)).get(0); + SectionHitGroup sectionGroup = toGroup(template.getSection(), resolution, result); + ErrorHit errors = result.hits().getErrorHit(); // transfer state from existing hit sectionGroup.setQuery(result.hits().getQuery()); - if (errors!=null && errors instanceof DefaultErrorHit) + if (errors instanceof DefaultErrorHit) sectionGroup.add((DefaultErrorHit)errors); result.hits().forEachField((name, value) -> sectionGroup.setField(name, value)); result.setHits(sectionGroup); diff --git a/container-search/src/main/java/com/yahoo/search/pagetemplates/engine/Resolution.java b/container-search/src/main/java/com/yahoo/search/pagetemplates/engine/Resolution.java index e0a3821e10c..f36ebe56e21 100644 --- a/container-search/src/main/java/com/yahoo/search/pagetemplates/engine/Resolution.java +++ b/container-search/src/main/java/com/yahoo/search/pagetemplates/engine/Resolution.java @@ -40,10 +40,10 @@ public class Resolution { * been resolved in this */ public int getResolution(Choice choice) { - if (choice.alternatives().size()==1) return 0; + if (choice.alternatives().size() == 1) return 0; if (choice.isEmpty()) throw new IllegalArgumentException("Cannot return a resolution of empty " + choice); - Integer resolution=choiceResolutions.get(choice); - if (resolution==null) throw new IllegalArgumentException(this + " has no resolution of " + choice); + Integer resolution = choiceResolutions.get(choice); + if (resolution == null) throw new IllegalArgumentException(this + " has no resolution of " + choice); return resolution; } diff --git a/container-search/src/main/java/com/yahoo/search/pagetemplates/result/PageTemplatesXmlRenderer.java b/container-search/src/main/java/com/yahoo/search/pagetemplates/result/PageTemplatesXmlRenderer.java index f2e2e1b034d..47914792da8 100644 --- a/container-search/src/main/java/com/yahoo/search/pagetemplates/result/PageTemplatesXmlRenderer.java +++ b/container-search/src/main/java/com/yahoo/search/pagetemplates/result/PageTemplatesXmlRenderer.java @@ -271,10 +271,10 @@ public class PageTemplatesXmlRenderer extends AsynchronousSectionedRenderer<Resu private Result getResult() { try { - return (Result) getResponse(); + return (Result)getResponse(); } catch (ClassCastException e) { - throw new IllegalArgumentException("PageTemplatesXmlRenderer attempted used outside a search context, got a " + - getResponse().getClass().getName()); + throw new IllegalStateException("PageTemplatesXmlRenderer attempted used outside a search context, got a " + + getResponse().getClass().getName()); } } diff --git a/container-search/src/main/java/com/yahoo/search/query/Model.java b/container-search/src/main/java/com/yahoo/search/query/Model.java index f06aab09a3d..637873aa375 100644 --- a/container-search/src/main/java/com/yahoo/search/query/Model.java +++ b/container-search/src/main/java/com/yahoo/search/query/Model.java @@ -7,6 +7,7 @@ import com.yahoo.language.LocaleFactory; import com.yahoo.prelude.query.CompositeItem; import com.yahoo.prelude.query.Item; import com.yahoo.prelude.query.TaggableItem; +import com.yahoo.processing.IllegalInputException; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.Query; import com.yahoo.search.query.parser.Parsable; @@ -236,10 +237,14 @@ public class Model implements Cloneable { */ public QueryTree getQueryTree() { if (queryTree == null) { - Parser parser = ParserFactory.newInstance(type, ParserEnvironment.fromExecutionContext(execution.context())); - queryTree = parser.parse(Parsable.fromQueryModel(this)); - if (parent.getTraceLevel() >= 2) { - parent.trace("Query parsed to: " + parent.yqlRepresentation(), 2); + try { + Parser parser = ParserFactory.newInstance(type, ParserEnvironment.fromExecutionContext(execution.context())); + queryTree = parser.parse(Parsable.fromQueryModel(this)); + if (parent.getTraceLevel() >= 2) + parent.trace("Query parsed to: " + parent.yqlRepresentation(), 2); + } + catch (IllegalArgumentException e) { + throw new IllegalInputException("Failed parsing query", e); } } return queryTree; diff --git a/container-search/src/main/java/com/yahoo/search/query/ParameterParser.java b/container-search/src/main/java/com/yahoo/search/query/ParameterParser.java index d358fa06977..3addc1fdb4e 100644 --- a/container-search/src/main/java/com/yahoo/search/query/ParameterParser.java +++ b/container-search/src/main/java/com/yahoo/search/query/ParameterParser.java @@ -1,6 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.query; +import com.yahoo.processing.IllegalInputException; + import static com.yahoo.container.util.Util.quote; /** @@ -25,13 +27,8 @@ public class ParameterParser { * representation cannot be parsed as a number followed optionally by time unit */ public static Long asMilliSeconds(Object value, Long defaultValue) { - if (value == null) { - return defaultValue; - } - if (value instanceof Number) { - Number n = (Number) value; - return Long.valueOf(n.longValue() * 1000L); - } + if (value == null) return defaultValue; + if (value instanceof Number) return ((Number)value).longValue() * 1000L; return parseTime(value.toString()); } @@ -43,7 +40,7 @@ public class ParameterParser { double multiplier = parseUnit(time.substring(unitOffset)); return (long) (measure * multiplier); } catch (RuntimeException e) { - throw new IllegalArgumentException("Error parsing " + quote(time), e); + throw new IllegalInputException("Error parsing " + quote(time), e); } } @@ -58,7 +55,7 @@ public class ParameterParser { } } if (unitOffset == 0) { - throw new NumberFormatException("Invalid number " + quote(time)); + throw new IllegalInputException("Invalid number " + quote(time)); } return unitOffset; } diff --git a/container-search/src/main/java/com/yahoo/search/query/SelectParser.java b/container-search/src/main/java/com/yahoo/search/query/SelectParser.java index 0d9acea7643..30d741f465c 100644 --- a/container-search/src/main/java/com/yahoo/search/query/SelectParser.java +++ b/container-search/src/main/java/com/yahoo/search/query/SelectParser.java @@ -2,6 +2,7 @@ package com.yahoo.search.query; import com.google.common.base.Preconditions; +import com.yahoo.processing.IllegalInputException; import com.yahoo.collections.LazyMap; import com.yahoo.geo.DistanceParser; import com.yahoo.geo.ParsedDegree; @@ -27,7 +28,6 @@ import com.yahoo.prelude.query.OrItem; import com.yahoo.prelude.query.PhraseItem; import com.yahoo.prelude.query.PredicateQueryItem; import com.yahoo.prelude.query.PrefixItem; -import com.yahoo.prelude.query.QueryException; import com.yahoo.prelude.query.RangeItem; import com.yahoo.prelude.query.RankItem; import com.yahoo.prelude.query.RegExpItem; @@ -72,10 +72,6 @@ import static com.yahoo.search.yql.YqlParser.AND_SEGMENTING; import static com.yahoo.search.yql.YqlParser.ANNOTATIONS; import static com.yahoo.search.yql.YqlParser.APPROXIMATE; import static com.yahoo.search.yql.YqlParser.ASCENDING_HITS_ORDER; -import static com.yahoo.search.yql.YqlParser.BOUNDS; -import static com.yahoo.search.yql.YqlParser.BOUNDS_LEFT_OPEN; -import static com.yahoo.search.yql.YqlParser.BOUNDS_OPEN; -import static com.yahoo.search.yql.YqlParser.BOUNDS_RIGHT_OPEN; import static com.yahoo.search.yql.YqlParser.CONNECTION_ID; import static com.yahoo.search.yql.YqlParser.CONNECTION_WEIGHT; import static com.yahoo.search.yql.YqlParser.CONNECTIVITY; @@ -83,7 +79,6 @@ import static com.yahoo.search.yql.YqlParser.DEFAULT_TARGET_NUM_HITS; import static com.yahoo.search.yql.YqlParser.DESCENDING_HITS_ORDER; import static com.yahoo.search.yql.YqlParser.DISTANCE; import static com.yahoo.search.yql.YqlParser.DOT_PRODUCT; -import static com.yahoo.search.yql.YqlParser.END_ANCHOR; import static com.yahoo.search.yql.YqlParser.EQUIV; import static com.yahoo.search.yql.YqlParser.FILTER; import static com.yahoo.search.yql.YqlParser.GEO_LOCATION; @@ -109,7 +104,6 @@ import static com.yahoo.search.yql.YqlParser.RANKED; import static com.yahoo.search.yql.YqlParser.SAME_ELEMENT; import static com.yahoo.search.yql.YqlParser.SCORE_THRESHOLD; import static com.yahoo.search.yql.YqlParser.SIGNIFICANCE; -import static com.yahoo.search.yql.YqlParser.START_ANCHOR; import static com.yahoo.search.yql.YqlParser.STEM; import static com.yahoo.search.yql.YqlParser.SUBSTRING; import static com.yahoo.search.yql.YqlParser.SUFFIX; @@ -117,7 +111,6 @@ import static com.yahoo.search.yql.YqlParser.TARGET_HITS; import static com.yahoo.search.yql.YqlParser.TARGET_NUM_HITS; import static com.yahoo.search.yql.YqlParser.THRESHOLD_BOOST_FACTOR; import static com.yahoo.search.yql.YqlParser.UNIQUE_ID; -import static com.yahoo.search.yql.YqlParser.URI; import static com.yahoo.search.yql.YqlParser.USE_POSITION_DATA; import static com.yahoo.search.yql.YqlParser.USER_INPUT_LANGUAGE; import static com.yahoo.search.yql.YqlParser.WAND; @@ -169,13 +162,18 @@ public class SelectParser implements Parser { private QueryTree buildTree() { Inspector inspector = SlimeUtils.jsonToSlime(this.query.getSelect().getWhereString()).get(); if (inspector.field("error_message").valid()) { - throw new QueryException("Illegal query: " + inspector.field("error_message").asString() + + throw new IllegalInputException("Illegal query: " + inspector.field("error_message").asString() + " at: '" + new String(inspector.field("offending_input").asData(), StandardCharsets.UTF_8) + "'"); } - Item root = walkJson(inspector); - connectItems(); - return new QueryTree(root); + try { + Item root = walkJson(inspector); + connectItems(); + return new QueryTree(root); + } + catch (IllegalArgumentException e) { + throw new IllegalInputException("Illegal JSON query", e); + } } private Item walkJson(Inspector inspector) { @@ -229,8 +227,8 @@ public class SelectParser implements Parser { private List<String> toGroupingRequests(String groupingJson) { Inspector inspector = SlimeUtils.jsonToSlime(groupingJson).get(); if (inspector.field("error_message").valid()) { - throw new QueryException("Illegal query: " + inspector.field("error_message").asString() + - " at: '" + new String(inspector.field("offending_input").asData(), StandardCharsets.UTF_8) + "'"); + throw new IllegalInputException("Illegal query: " + inspector.field("error_message").asString() + + " at: '" + new String(inspector.field("offending_input").asData(), StandardCharsets.UTF_8) + "'"); } List<String> operations = new ArrayList<>(); @@ -1199,9 +1197,10 @@ public class SelectParser implements Parser { private void connectItems() { for (ConnectedItem entry : connectedItems) { TaggableItem to = identifiedItems.get(entry.toId); - Preconditions.checkNotNull(to, - "Item '%s' was specified to connect to item with ID %s, which does not " - + "exist in the query.", entry.fromItem, entry.toId); + if (to == null) + throw new IllegalArgumentException("Item '" + entry.fromItem + + "' was specified to connect to item with ID " + entry.toId + + ", which does not exist in the query."); entry.fromItem.setConnectivity((Item) to, entry.weight); } } diff --git a/container-search/src/main/java/com/yahoo/search/query/Sorting.java b/container-search/src/main/java/com/yahoo/search/query/Sorting.java index 6518dcd5b6d..0a0ae65d524 100644 --- a/container-search/src/main/java/com/yahoo/search/query/Sorting.java +++ b/container-search/src/main/java/com/yahoo/search/query/Sorting.java @@ -3,6 +3,7 @@ package com.yahoo.search.query; import com.ibm.icu.text.Collator; import com.ibm.icu.util.ULocale; +import com.yahoo.processing.IllegalInputException; import com.yahoo.text.Utf8; import java.nio.ByteBuffer; @@ -88,7 +89,7 @@ public class Sorting implements Cloneable { } else if (STRENGTH_IDENTICAL.equalsIgnoreCase(s)) { strength = UcaSorter.Strength.IDENTICAL; } else { - throw new IllegalArgumentException("Unknown collation strength: '" + s + "'"); + throw new IllegalInputException("Unknown collation strength: '" + s + "'"); } sorter = new UcaSorter(sortString.substring(startPar+1, commaPos), sortString.substring(commaPos+1, commaopt), strength); } else { @@ -99,9 +100,9 @@ public class Sorting implements Cloneable { } } else { if (funcName.isEmpty()) { - throw new IllegalArgumentException("No sort function specified"); + throw new IllegalInputException("No sort function specified"); } else { - throw new IllegalArgumentException("Unknown sort function '" + funcName + "'"); + throw new IllegalInputException("Unknown sort function '" + funcName + "'"); } } } else { @@ -196,7 +197,7 @@ public class Sorting implements Cloneable { if (legalAttributeName.matcher(fieldName).matches()) { this.fieldName = fieldName; } else { - throw new IllegalArgumentException("Illegal attribute name '" + fieldName + "' for sorting. Requires '" + legalAttributeName.pattern() + "'"); + throw new IllegalInputException("Illegal attribute name '" + fieldName + "' for sorting. Requires '" + legalAttributeName.pattern() + "'"); } } diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfile.java b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfile.java index b6b03d37da8..6008b046d1a 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfile.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfile.java @@ -4,6 +4,7 @@ package com.yahoo.search.query.profile; import com.google.common.collect.ImmutableList; import com.yahoo.component.ComponentId; import com.yahoo.component.provider.FreezableSimpleComponent; +import com.yahoo.processing.IllegalInputException; import com.yahoo.processing.request.CompoundName; import com.yahoo.processing.request.Properties; import com.yahoo.search.query.profile.compiled.CompiledQueryProfile; @@ -451,7 +452,7 @@ public class QueryProfile extends FreezableSimpleComponent implements Cloneable setNode(name, value, null, binding, registry); } catch (IllegalArgumentException e) { - throw new IllegalArgumentException("Could not set '" + name + "' to '" + value + "'", e); + throw new IllegalInputException("Could not set '" + name + "' to '" + value + "'", e); } } @@ -688,15 +689,15 @@ public class QueryProfile extends FreezableSimpleComponent implements Cloneable FieldDescription fieldDescription = type.getField(localName); if (fieldDescription == null) { if (type.isStrict()) - throw new IllegalArgumentException("'" + localName + "' is not declared in " + type + ", and the type is strict"); + throw new IllegalInputException("'" + localName + "' is not declared in " + type + ", and the type is strict"); return value; } if (registry == null && (fieldDescription.getType() instanceof QueryProfileFieldType)) - throw new IllegalArgumentException("A registry was not passed: Query profile references is not supported"); + throw new IllegalInputException("A registry was not passed: Query profile references is not supported"); Object convertedValue = fieldDescription.getType().convertFrom(value, registry); if (convertedValue == null) - throw new IllegalArgumentException("'" + value + "' is not a " + fieldDescription.getType().toInstanceDescription()); + throw new IllegalInputException("'" + value + "' is not a " + fieldDescription.getType().toInstanceDescription()); return convertedValue; } diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java index d199ee44c9c..41272d695ac 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java @@ -2,6 +2,7 @@ package com.yahoo.search.query.profile; import com.yahoo.collections.Pair; +import com.yahoo.processing.IllegalInputException; import com.yahoo.processing.request.CompoundName; import com.yahoo.processing.request.properties.PropertyMap; import com.yahoo.protect.Validator; @@ -107,7 +108,7 @@ public class QueryProfileProperties extends Properties { String localName = name.get(i); FieldDescription fieldDescription = type.getField(localName); if (fieldDescription == null && type.isStrict()) - throw new IllegalArgumentException("'" + localName + "' is not declared in " + type + ", and the type is strict"); + throw new IllegalInputException("'" + localName + "' is not declared in " + type + ", and the type is strict"); // TODO: In addition to strictness, check legality along the way @@ -115,7 +116,7 @@ public class QueryProfileProperties extends Properties { if (i == name.size() - 1) { // at the end of the path, check the assignment type value = fieldDescription.getType().convertFrom(value, profile.getRegistry()); if (value == null) - throw new IllegalArgumentException("'" + value + "' is not a " + + throw new IllegalInputException("'" + value + "' is not a " + fieldDescription.getType().toInstanceDescription()); } else if (fieldDescription.getType() instanceof QueryProfileFieldType) { @@ -129,12 +130,12 @@ public class QueryProfileProperties extends Properties { if (value instanceof String && value.toString().startsWith("ref:")) { if (profile.getRegistry() == null) - throw new IllegalArgumentException("Runtime query profile references does not work when the " + + throw new IllegalInputException("Runtime query profile references does not work when the " + "QueryProfileProperties are constructed without a registry"); String queryProfileId = value.toString().substring(4); value = profile.getRegistry().findQueryProfile(queryProfileId); if (value == null) - throw new IllegalArgumentException("Query profile '" + queryProfileId + "' is not found"); + throw new IllegalInputException("Query profile '" + queryProfileId + "' is not found"); } if (value instanceof CompiledQueryProfile) { // this will be due to one of the two clauses above @@ -149,7 +150,7 @@ public class QueryProfileProperties extends Properties { } } catch (IllegalArgumentException e) { - throw new IllegalArgumentException("Could not set '" + name + "' to '" + value + "'", e); + throw new IllegalInputException("Could not set '" + name + "' to '" + value + "'", e); } } diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/Binding.java b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/Binding.java index 8638a99172f..e873c80add1 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/Binding.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/Binding.java @@ -33,7 +33,6 @@ public class Binding implements Comparable<Binding> { private final int hashCode; - @SuppressWarnings("unchecked") public static final Binding nullBinding = new Binding(Integer.MAX_VALUE, Collections.<String,String>emptyMap()); public static Binding createFrom(DimensionBinding dimensionBinding) { diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldDescription.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldDescription.java index 6c30f1a8b05..daab5f6a378 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldDescription.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldDescription.java @@ -97,7 +97,8 @@ public class FieldDescription implements Comparable<FieldDescription> { this.type = type; // Forbidden until we can figure out the right semantics - if (name.isCompound() && ! aliases.isEmpty()) throw new IllegalArgumentException("Aliases are not allowed with compound names"); + if (name.isCompound() && ! aliases.isEmpty()) + throw new IllegalArgumentException("Aliases are not allowed with compound names"); this.aliases = ImmutableList.copyOf(aliases); this.mandatory = mandatory; diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java index d5f88ba99ec..6186f588a21 100644 --- a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java +++ b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.query.properties; +import com.yahoo.processing.IllegalInputException; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.Query; @@ -317,7 +318,7 @@ public class QueryProperties extends Properties { if (e.getMessage() != null && e.getMessage().startsWith("Could not set")) throw e; else - throw new IllegalArgumentException("Could not set '" + key + "' to '" + value + "'", e); + throw new IllegalInputException("Could not set '" + key + "' to '" + value + "'", e); } } @@ -364,8 +365,8 @@ public class QueryProperties extends Properties { } private void throwIllegalParameter(String key,String namespace) { - throw new IllegalArgumentException("'" + key + "' is not a valid property in '" + namespace + - "'. See the query api for valid keys starting by '" + namespace + "'."); + throw new IllegalInputException("'" + key + "' is not a valid property in '" + namespace + + "'. See the query api for valid keys starting by '" + namespace + "'."); } @Override diff --git a/container-search/src/main/java/com/yahoo/search/query/ranking/MatchPhase.java b/container-search/src/main/java/com/yahoo/search/query/ranking/MatchPhase.java index 794247863bf..72a6533e946 100644 --- a/container-search/src/main/java/com/yahoo/search/query/ranking/MatchPhase.java +++ b/container-search/src/main/java/com/yahoo/search/query/ranking/MatchPhase.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.query.ranking; +import com.yahoo.processing.IllegalInputException; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.query.Ranking; import com.yahoo.search.query.profile.types.FieldDescription; @@ -84,8 +85,9 @@ public class MatchPhase implements Cloneable { public void setMaxFilterCoverage(double maxFilterCoverage) { if ((maxFilterCoverage < 0.0) || (maxFilterCoverage > 1.0)) { - throw new IllegalArgumentException("maxFilterCoverage must be in the range [0.0, 1.0]. It is " + maxFilterCoverage); + throw new IllegalInputException("maxFilterCoverage must be in the range [0.0, 1.0]. It is " + maxFilterCoverage); } + this.maxFilterCoverage = maxFilterCoverage; } diff --git a/container-search/src/main/java/com/yahoo/search/query/ranking/Matching.java b/container-search/src/main/java/com/yahoo/search/query/ranking/Matching.java index fb3f2acfadd..14e4e006b39 100644 --- a/container-search/src/main/java/com/yahoo/search/query/ranking/Matching.java +++ b/container-search/src/main/java/com/yahoo/search/query/ranking/Matching.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.query.ranking; +import com.yahoo.processing.IllegalInputException; import com.yahoo.search.query.Ranking; import com.yahoo.search.query.profile.types.FieldDescription; import com.yahoo.search.query.profile.types.QueryProfileType; @@ -59,7 +60,7 @@ public class Matching implements Cloneable { public void setTermwiselimit(double value) { if ((value < 0.0) || (value > 1.0)) { - throw new IllegalArgumentException("termwiselimit must be in the range [0.0, 1.0]. It is " + value); + throw new IllegalInputException("termwiselimit must be in the range [0.0, 1.0]. It is " + value); } termwiseLimit = value; } diff --git a/container-search/src/main/java/com/yahoo/search/query/ranking/SoftTimeout.java b/container-search/src/main/java/com/yahoo/search/query/ranking/SoftTimeout.java index 0d47ef77ce5..43c26692221 100644 --- a/container-search/src/main/java/com/yahoo/search/query/ranking/SoftTimeout.java +++ b/container-search/src/main/java/com/yahoo/search/query/ranking/SoftTimeout.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.query.ranking; +import com.yahoo.processing.IllegalInputException; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.query.Ranking; import com.yahoo.search.query.profile.types.FieldDescription; @@ -54,7 +55,7 @@ public class SoftTimeout implements Cloneable { /** Override the adaptive factor determined on the content nodes */ public void setFactor(double factor) { if ((factor < 0.0) || (factor > 1.0)) { - throw new IllegalArgumentException("factor must be in the range [0.0, 1.0], got " + factor); + throw new IllegalInputException("factor must be in the range [0.0, 1.0], got " + factor); } this.factor = factor; } @@ -64,7 +65,7 @@ public class SoftTimeout implements Cloneable { /** Override the tail cost factor determined on the content nodes */ public void setTailcost(double tailcost) { if ((tailcost < 0.0) || (tailcost > 1.0)) { - throw new IllegalArgumentException("tailcost must be in the range [0.0, 1.0], got " + tailcost); + throw new IllegalInputException("tailcost must be in the range [0.0, 1.0], got " + tailcost); } this.tailcost = tailcost; } diff --git a/container-search/src/main/java/com/yahoo/search/query/rewrite/rewriters/GenericExpansionRewriter.java b/container-search/src/main/java/com/yahoo/search/query/rewrite/rewriters/GenericExpansionRewriter.java index 794217e688f..9a2374da4c9 100644 --- a/container-search/src/main/java/com/yahoo/search/query/rewrite/rewriters/GenericExpansionRewriter.java +++ b/container-search/src/main/java/com/yahoo/search/query/rewrite/rewriters/GenericExpansionRewriter.java @@ -76,9 +76,8 @@ public class GenericExpansionRewriter extends QueryRewriteSearcher { HashMap<String, File> fileList) { logger = Logger.getLogger(GenericExpansionRewriter.class.getName()); FSA fsa = (FSA)rewriterDicts.get(GENERIC_EXPAND_DICT); - if(fsa==null) { - RewriterUtils.error(logger, "Error retrieving FSA dictionary: " + - GENERIC_EXPAND_DICT); + if (fsa==null) { + RewriterUtils.error(logger, "Error retrieving FSA dictionary: " + GENERIC_EXPAND_DICT); return false; } // Create Phrase Matcher diff --git a/container-search/src/main/java/com/yahoo/search/query/textserialize/item/ItemContext.java b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/ItemContext.java index e739608387d..b6c886c91d7 100644 --- a/container-search/src/main/java/com/yahoo/search/query/textserialize/item/ItemContext.java +++ b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/ItemContext.java @@ -12,6 +12,7 @@ import java.util.Map; * @author Tony Vaagenes */ public class ItemContext { + private class Connectivity { final String id; final double strength; @@ -43,7 +44,8 @@ public class ItemContext { private Item getItem(String id) { Item item = itemById.get(id); if (item == null) - throw new IllegalArgumentException("No item with id '" + id + "'."); + throw new IllegalArgumentException("No item with id '" + id + "'"); return item; } + } diff --git a/container-search/src/main/java/com/yahoo/search/query/textserialize/item/TermConverter.java b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/TermConverter.java index 04c01d7acc1..503987af027 100644 --- a/container-search/src/main/java/com/yahoo/search/query/textserialize/item/TermConverter.java +++ b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/TermConverter.java @@ -10,6 +10,7 @@ import com.yahoo.search.query.textserialize.serializer.ItemIdMapper; * @author Tony Vaagenes */ public abstract class TermConverter implements ItemFormConverter { + @Override public Object formToItem(String name, ItemArguments arguments, ItemContext context) { ensureOnlyOneChild(arguments); @@ -22,7 +23,6 @@ public abstract class TermConverter implements ItemFormConverter { abstract TermItem newTermItem(String word); - private void ensureOnlyOneChild(ItemArguments arguments) { if (arguments.children.size() != 1) { throw new IllegalArgumentException("Expected exactly one argument, got '" + @@ -50,4 +50,5 @@ public abstract class TermConverter implements ItemFormConverter { } protected abstract String getValue(TermItem item); + } diff --git a/container-search/src/main/java/com/yahoo/search/query/textserialize/item/TypeCheck.java b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/TypeCheck.java index b7843a300dc..70eba9d8b50 100644 --- a/container-search/src/main/java/com/yahoo/search/query/textserialize/item/TypeCheck.java +++ b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/TypeCheck.java @@ -7,6 +7,7 @@ import com.yahoo.protect.Validator; * @author Tony Vaagenes */ public class TypeCheck { + public static void ensureInstanceOf(Object object, Class<?> c) { Validator.ensureInstanceOf(expectationString(c.getName(), object.getClass().getSimpleName()), object, c); @@ -24,4 +25,5 @@ public class TypeCheck { private static String expectationString(String expected, String got) { return "Expected " + expected + ", but got " + got; } + } diff --git a/container-search/src/main/java/com/yahoo/search/query/textserialize/serializer/Serializer.java b/container-search/src/main/java/com/yahoo/search/query/textserialize/serializer/Serializer.java index d4e499bab79..d849e744184 100644 --- a/container-search/src/main/java/com/yahoo/search/query/textserialize/serializer/Serializer.java +++ b/container-search/src/main/java/com/yahoo/search/query/textserialize/serializer/Serializer.java @@ -14,6 +14,7 @@ import static com.yahoo.search.query.textserialize.item.ListUtil.first; * @author Tony Vaagenes */ class Serializer { + static String serialize(Object child, ItemIdMapper itemIdMapper) { if (child instanceof DispatchForm) { return ((DispatchForm) child).serialize(itemIdMapper); @@ -76,4 +77,5 @@ class Serializer { static String serializeItem(Item item, ItemIdMapper itemIdMapper) { return ItemExecutorRegistry.getByType(item.getItemType()).itemToForm(item, itemIdMapper).serialize(itemIdMapper); } + } diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/BooleanAttributeParser.java b/container-search/src/main/java/com/yahoo/search/querytransform/BooleanAttributeParser.java index fd76cc4397a..532de1270da 100644 --- a/container-search/src/main/java/com/yahoo/search/querytransform/BooleanAttributeParser.java +++ b/container-search/src/main/java/com/yahoo/search/querytransform/BooleanAttributeParser.java @@ -14,10 +14,10 @@ import java.math.BigInteger; * a 64-bit hex number <code>0x1234</code> or a list of bits <code>[0, 2, 43, * 22, ...]</code>. * - * @author <a href="mailto:magnarn@yahoo-inc.com">Magnar Nedland</a> - * @since 5.1.15 + * @author Magnar Nedland */ abstract class BooleanAttributeParser extends SimpleMapParser { + private boolean isMap = true; @Override diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/BooleanSearcher.java b/container-search/src/main/java/com/yahoo/search/querytransform/BooleanSearcher.java index f77301f587c..de01773c27c 100644 --- a/container-search/src/main/java/com/yahoo/search/querytransform/BooleanSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/querytransform/BooleanSearcher.java @@ -4,6 +4,7 @@ package com.yahoo.search.querytransform; import com.yahoo.component.chain.dependencies.After; import com.yahoo.component.chain.dependencies.Provides; import com.yahoo.prelude.query.PredicateQueryItem; +import com.yahoo.processing.IllegalInputException; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.Query; import com.yahoo.search.Result; @@ -26,6 +27,7 @@ import static com.yahoo.yolean.Exceptions.toMessageString; @After({ STEMMING, ACCENT_REMOVAL }) @Provides(BooleanSearcher.PREDICATE) public class BooleanSearcher extends Searcher { + private static final CompoundName FIELD = new CompoundName("boolean.field"); private static final CompoundName ATTRIBUTES = new CompoundName("boolean.attributes"); private static final CompoundName RANGE_ATTRIBUTES = new CompoundName("boolean.rangeAttributes"); @@ -61,7 +63,11 @@ public class BooleanSearcher extends Searcher { } catch (TokenMgrException e) { return new Result(query, ErrorMessage.createInvalidQueryParameter(toMessageString(e))); } - } else { + catch (IllegalArgumentException e) { + throw new IllegalInputException("Failed boolean search on field '" + fieldName + "'", e); + } + } + else { if (query.isTraceable(5)) { query.trace("BooleanSearcher: Nothing added to query", false, 5); } @@ -79,7 +85,9 @@ public class BooleanSearcher extends Searcher { } static public class PredicateValueAttributeParser extends BooleanAttributeParser { - private PredicateQueryItem item; + + private final PredicateQueryItem item; + public PredicateValueAttributeParser(PredicateQueryItem item) { this.item = item; } @@ -93,10 +101,13 @@ public class BooleanSearcher extends Searcher { protected void addAttribute(String attribute, String value, BigInteger subQueryMask) { item.addFeature(attribute, value, subQueryMask.longValue()); } + } static private class PredicateRangeAttributeParser extends BooleanAttributeParser { - private PredicateQueryItem item; + + private final PredicateQueryItem item; + public PredicateRangeAttributeParser(PredicateQueryItem item) { this.item = item; } @@ -110,5 +121,7 @@ public class BooleanSearcher extends Searcher { protected void addAttribute(String attribute, String value, BigInteger subQueryMask) { item.addRangeFeature(attribute, Long.parseLong(value), subQueryMask.longValue()); } + } + } diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/WandSearcher.java b/container-search/src/main/java/com/yahoo/search/querytransform/WandSearcher.java index 0b1387a16a2..f3804af1a9e 100644 --- a/container-search/src/main/java/com/yahoo/search/querytransform/WandSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/querytransform/WandSearcher.java @@ -4,6 +4,7 @@ package com.yahoo.search.querytransform; import com.yahoo.prelude.Index; import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.query.*; +import com.yahoo.processing.IllegalInputException; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.Query; import com.yahoo.search.Result; @@ -92,7 +93,7 @@ public class WandSearcher extends Searcher { private WandType resolveWandType(IndexFacts.Session indexFacts, Query query) { Index index = indexFacts.getIndex(fieldName); if (index.isNull()) { - throw new IllegalArgumentException("Field '" + fieldName + "' was not found in " + indexFacts); + throw new IllegalInputException("Field '" + fieldName + "' was not found in " + indexFacts); } else { return WandType.create(query.properties().getString(WAND_TYPE, "vespa")); } @@ -100,15 +101,15 @@ public class WandSearcher extends Searcher { private int resolveHeapSize(Query query) { String defaultHeapSize = "100"; - return Integer.valueOf(query.properties().getString(WAND_HEAP_SIZE, defaultHeapSize)); + return Integer.parseInt(query.properties().getString(WAND_HEAP_SIZE, defaultHeapSize)); } private double resolveScoreThreshold(Query query) { - return Double.valueOf(query.properties().getString(WAND_SCORE_THRESHOLD, "0")); + return Double.parseDouble(query.properties().getString(WAND_SCORE_THRESHOLD, "0")); } private double resolveThresholdBoostFactor(Query query) { - return Double.valueOf(query.properties().getString(WAND_THRESHOLD_BOOST_FACTOR, "1")); + return Double.parseDouble(query.properties().getString(WAND_THRESHOLD_BOOST_FACTOR, "1")); } public boolean hasValidData() { @@ -166,7 +167,7 @@ public class WandSearcher extends Searcher { } else if (inputs.getWandType().equals(WandType.DOT_PRODUCT)) { return populate(new DotProductItem(inputs.getFieldName()), inputs.getTokens()); } - throw new IllegalArgumentException("Unknown type '" + inputs.getWandType() + "'"); + throw new IllegalInputException("Unknown type '" + inputs.getWandType() + "'"); } private CompositeItem populate(CompositeItem parent, String fieldName, Map<String,Integer> tokens) { @@ -195,10 +196,12 @@ public class WandSearcher extends Searcher { } private static class IntegerMapParser extends MapParser<Integer> { + @Override protected Integer parseValue(String s) { return Integer.parseInt(s); } + } } diff --git a/container-search/src/main/java/com/yahoo/search/rendering/RendererRegistry.java b/container-search/src/main/java/com/yahoo/search/rendering/RendererRegistry.java index 783045babf4..9540bc20bc5 100644 --- a/container-search/src/main/java/com/yahoo/search/rendering/RendererRegistry.java +++ b/container-search/src/main/java/com/yahoo/search/rendering/RendererRegistry.java @@ -4,6 +4,7 @@ package com.yahoo.search.rendering; import com.yahoo.component.ComponentId; import com.yahoo.component.ComponentSpecification; import com.yahoo.component.provider.ComponentRegistry; +import com.yahoo.processing.IllegalInputException; import com.yahoo.processing.rendering.Renderer; import com.yahoo.search.Result; import com.yahoo.search.pagetemplates.result.PageTemplatesXmlRenderer; @@ -103,8 +104,8 @@ public final class RendererRegistry extends ComponentRegistry<com.yahoo.processi com.yahoo.processing.rendering.Renderer<Result> renderer = getComponent(format); if (renderer == null) - throw new IllegalArgumentException("No renderer with id or alias '" + format + "'. " + - "Available renderers are: [" + rendererNames() + "]."); + throw new IllegalInputException("No renderer with id or alias '" + format + "'. " + + "Available renderers are: [" + rendererNames() + "]."); return renderer; } diff --git a/container-search/src/main/java/com/yahoo/search/rendering/XmlRenderer.java b/container-search/src/main/java/com/yahoo/search/rendering/XmlRenderer.java index 62ee16993fd..f3b0cd3a61e 100644 --- a/container-search/src/main/java/com/yahoo/search/rendering/XmlRenderer.java +++ b/container-search/src/main/java/com/yahoo/search/rendering/XmlRenderer.java @@ -352,8 +352,8 @@ public final class XmlRenderer extends AsynchronousSectionedRenderer<Result> { try { return (Result) getResponse(); } catch (ClassCastException e) { - throw new IllegalArgumentException("XmlRenderer attempted used outside a search context, got a " + - getResponse().getClass().getName()); + throw new IllegalStateException("XmlRenderer attempted used outside a search context, got a " + + getResponse().getClass().getName()); } } diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/model/federation/LocalProviderSpec.java b/container-search/src/main/java/com/yahoo/search/searchchain/model/federation/LocalProviderSpec.java index 4c36ca9b4da..8db690678bb 100644 --- a/container-search/src/main/java/com/yahoo/search/searchchain/model/federation/LocalProviderSpec.java +++ b/container-search/src/main/java/com/yahoo/search/searchchain/model/federation/LocalProviderSpec.java @@ -56,7 +56,7 @@ public class LocalProviderSpec { this.cacheSize = cacheSize; if (clusterName == null) - throw new IllegalArgumentException("Missing cluster name."); + throw new IllegalArgumentException("Missing cluster name"); } public static boolean includesType(String type) { diff --git a/container-search/src/main/java/com/yahoo/search/searchers/InputCheckingSearcher.java b/container-search/src/main/java/com/yahoo/search/searchers/InputCheckingSearcher.java index 3c0453f8900..5e15a8ba14b 100644 --- a/container-search/src/main/java/com/yahoo/search/searchers/InputCheckingSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/searchers/InputCheckingSearcher.java @@ -19,11 +19,13 @@ import com.yahoo.prelude.query.Item; import com.yahoo.prelude.query.PhraseItem; import com.yahoo.prelude.query.TermItem; import com.yahoo.prelude.query.WordItem; +import com.yahoo.processing.IllegalInputException; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.searchchain.Execution; +import com.yahoo.yolean.Exceptions; /** * Check whether the query tree seems to be "well formed". In other words, run heurestics against @@ -50,10 +52,8 @@ public class InputCheckingSearcher extends Searcher { public Result search(Query query, Execution execution) { try { checkQuery(query); - } catch (IllegalArgumentException e) { - if (log.isLoggable(Level.FINE)) { - log.log(Level.FINE, "Rejected query \"" + query.toString() + "\" on cause of: " + e.getMessage()); - } + } catch (IllegalInputException e) { + log.log(Level.FINE, () -> "Rejected query '" + query.toString() + "' on cause of: " + Exceptions.toMessageString(e)); return new Result(query, ErrorMessage.createIllegalQuery(e.getMessage())); } return execution.search(query); @@ -92,8 +92,9 @@ public class InputCheckingSearcher extends Searcher { repeatedCount++; if (repeatedCount >= MAX_REPEATED_CONSECUTIVE_TERMS_IN_PHRASE) { repeatedConsecutiveTermsInPhraseRejections.add(); - throw new IllegalArgumentException("More than " + MAX_REPEATED_CONSECUTIVE_TERMS_IN_PHRASE + - " ocurrences of term '" + current + "' in a row detected in phrase : " + phrase.toString()); + throw new IllegalInputException("More than " + MAX_REPEATED_CONSECUTIVE_TERMS_IN_PHRASE + + " occurrences of term '" + current + + "' in a row detected in phrase : " + phrase.toString()); } } else { repeatedCount = 0; @@ -125,8 +126,8 @@ public class InputCheckingSearcher extends Searcher { if (count != null) { if (count.get() >= MAX_REPEATED_TERMS_IN_PHRASE) { repeatedTermsInPhraseRejections.add(); - throw new IllegalArgumentException("Phrase contains more than " + MAX_REPEATED_TERMS_IN_PHRASE + - " occurrences of term '" + current + "' in phrase : " + phrase.toString()); + throw new IllegalInputException("Phrase contains more than " + MAX_REPEATED_TERMS_IN_PHRASE + + " occurrences of term '" + current + "' in phrase : " + phrase.toString()); } count.inc(); } else { @@ -169,8 +170,8 @@ public class InputCheckingSearcher extends Searcher { return; } utfRejections.add(); - throw new IllegalArgumentException("The user input has been determined to be double encoded UTF-8." - + " Please investigate whether this is a false positive."); + throw new IllegalInputException("The user input has been determined to be double encoded UTF-8." + + " Please investigate whether this is a false positive."); } private int countSingleCharacterUserTerms(Item queryItem) { diff --git a/container-search/src/main/java/com/yahoo/search/statistics/PeakQpsSearcher.java b/container-search/src/main/java/com/yahoo/search/statistics/PeakQpsSearcher.java index 108e46fb68e..6d4fcdaf8fb 100644 --- a/container-search/src/main/java/com/yahoo/search/statistics/PeakQpsSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/statistics/PeakQpsSearcher.java @@ -137,8 +137,8 @@ public class PeakQpsSearcher extends Searcher { useMetaHit = false; propertyName = null; } else { - throw new IllegalArgumentException("Config definition out of sync with implementation." + - " No way to create output for method " + method + "."); + throw new IllegalStateException("Config definition out of sync with implementation." + + " No way to create output for method " + method + "."); } } diff --git a/container-search/src/main/java/com/yahoo/search/yql/MinimalQueryInserter.java b/container-search/src/main/java/com/yahoo/search/yql/MinimalQueryInserter.java index 8b2438e3ec2..b863440ea5b 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/MinimalQueryInserter.java +++ b/container-search/src/main/java/com/yahoo/search/yql/MinimalQueryInserter.java @@ -5,6 +5,7 @@ import com.google.common.annotations.Beta; import com.google.inject.Inject; import com.yahoo.language.Linguistics; import com.yahoo.language.simple.SimpleLinguistics; +import com.yahoo.processing.IllegalInputException; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; @@ -28,6 +29,7 @@ import java.util.logging.Logger; * * @author Steinar Knutsen */ +// TODO: The query model should do this @Beta @Provides(MinimalQueryInserter.EXTERNAL_YQL) @Before(PhaseNames.TRANSFORMED_QUERY) @@ -40,19 +42,22 @@ public class MinimalQueryInserter extends Searcher { private static final CompoundName MAX_HITS = new CompoundName("maxHits"); private static final CompoundName MAX_OFFSET = new CompoundName("maxOffset"); - private static Logger log = Logger.getLogger(MinimalQueryInserter.class.getName()); + private static final Logger log = Logger.getLogger(MinimalQueryInserter.class.getName()); @Inject public MinimalQueryInserter(Linguistics linguistics) { // Warmup is needed to avoid a large 400ms init cost during first execution of yql code. warmup(linguistics); } + public MinimalQueryInserter() { this(new SimpleLinguistics()); } + static boolean warmup() { return warmup(new SimpleLinguistics()); } + private static boolean warmup(Linguistics linguistics) { Query query = new Query("search/?yql=select%20*%20from%20sources%20where%20title%20contains%20'xyz';"); Result result = insertQuery(query, new ParserEnvironment().setLinguistics(linguistics)); @@ -67,6 +72,18 @@ public class MinimalQueryInserter extends Searcher { return true; } + @Override + public Result search(Query query, Execution execution) { + try { + if (query.properties().get(YQL) == null) return execution.search(query); + Result result = insertQuery(query, ParserEnvironment.fromExecutionContext(execution.context())); + return (result == null) ? execution.search(query) : result; + } + catch (IllegalArgumentException e) { + throw new IllegalInputException("Illegal YQL query", e); + } + } + private static Result insertQuery(Query query, ParserEnvironment env) { YqlParser parser = (YqlParser) ParserFactory.newInstance(Query.Type.YQL, env); parser.setQueryParser(false); @@ -116,12 +133,4 @@ public class MinimalQueryInserter extends Searcher { return null; } - @Override - public Result search(Query query, Execution execution) { - if (query.properties().get(YQL) == null) return execution.search(query); - - Result result = insertQuery(query, ParserEnvironment.fromExecutionContext(execution.context())); - return (result == null) ? execution.search(query) : result; - } - } diff --git a/container-search/src/main/java/com/yahoo/search/yql/OperatorNode.java b/container-search/src/main/java/com/yahoo/search/yql/OperatorNode.java index e3ef4b5d4df..f529de32fe9 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/OperatorNode.java +++ b/container-search/src/main/java/com/yahoo/search/yql/OperatorNode.java @@ -162,7 +162,7 @@ final class OperatorNode<T extends Operator> { } // we are aware only of types used in our logical operator trees -- OperatorNode, List, and constant values - private static final Function<Object, Object> COPY = new Function<Object, Object>() { + private static final Function<Object, Object> COPY = new Function<>() { @Override public Object apply(Object input) { if (input instanceof List) { diff --git a/container-search/src/main/java/com/yahoo/search/yql/ProgramParser.java b/container-search/src/main/java/com/yahoo/search/yql/ProgramParser.java index a270d935a97..234558f5fd2 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/ProgramParser.java +++ b/container-search/src/main/java/com/yahoo/search/yql/ProgramParser.java @@ -139,7 +139,7 @@ final class ProgramParser { int charPositionInLine, @NotNull String msg, @Nullable RecognitionException e) { - throw new ProgramCompileException(new Location(programName, line, charPositionInLine), msg); + throw new ProgramCompileException(new Location(programName, line, charPositionInLine), "%s", msg); } }); @@ -156,7 +156,7 @@ final class ProgramParser { int charPositionInLine, @NotNull String msg, @Nullable RecognitionException e) { - throw new ProgramCompileException(new Location(programName, line, charPositionInLine), msg); + throw new ProgramCompileException(new Location(programName, line, charPositionInLine), "%s", msg); } }); @@ -622,19 +622,19 @@ final class ProgramParser { } private OperatorNode<SequenceOperator> convertMerge(List<Merge_componentContext> mergeComponentList, Scope scope) { - Preconditions.checkArgument(mergeComponentList != null); - List<OperatorNode<SequenceOperator>> sources = Lists.newArrayListWithExpectedSize(mergeComponentList.size()); - for (Merge_componentContext mergeComponent:mergeComponentList) { - Select_statementContext selectContext = mergeComponent.select_statement(); - Source_statementContext sourceContext = mergeComponent.source_statement(); - if (selectContext != null) { - sources.add(convertQuery(selectContext, scope.getRoot())); - } else { - sources.add(convertQuery(sourceContext, scope.getRoot())); - } - } - return OperatorNode.create(SequenceOperator.MERGE, sources); - } + Preconditions.checkArgument(mergeComponentList != null); + List<OperatorNode<SequenceOperator>> sources = Lists.newArrayListWithExpectedSize(mergeComponentList.size()); + for (Merge_componentContext mergeComponent:mergeComponentList) { + Select_statementContext selectContext = mergeComponent.select_statement(); + Source_statementContext sourceContext = mergeComponent.source_statement(); + if (selectContext != null) { + sources.add(convertQuery(selectContext, scope.getRoot())); + } else { + sources.add(convertQuery(sourceContext, scope.getRoot())); + } + } + return OperatorNode.create(SequenceOperator.MERGE, sources); + } private OperatorNode<SequenceOperator> convertQuery(ParseTree node, Scope scope) { if (node instanceof Select_statementContext @@ -642,7 +642,7 @@ final class ProgramParser { || node instanceof Update_statementContext || node instanceof Delete_statementContext) { return convertSelectOrInsertOrUpdateOrDelete(node, scope.getRoot()); - } else if (node instanceof Source_statementContext) { //for pipe + } else if (node instanceof Source_statementContext) { // for pipe Source_statementContext sourceStatementContext = (Source_statementContext)node; return convertPipe(sourceStatementContext.query_statement(), sourceStatementContext.pipeline_step(), scope); } else if (node instanceof Merge_statementContext) { diff --git a/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java b/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java index 22328fb026e..a38e48fd89d 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java +++ b/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java @@ -142,7 +142,7 @@ public class VespaSerializer { escape(((WordItem) current).getIndexedString(), destination).append('"'); } else { throw new IllegalArgumentException("Serializing of " + current.getClass().getSimpleName() - + " in segment AND expressions not implemented, please report this as a bug."); + + " in segment AND expressions not implemented, please report this as a bug."); } } } @@ -643,7 +643,7 @@ public class VespaSerializer { WordAlternativesSerializer.serialize(destination, (WordAlternativesItem) current, false); } else { throw new IllegalArgumentException("Serializing of " + current.getClass().getSimpleName() + - " in phrases not implemented, please report this as a bug."); + " in phrases not implemented, please report this as a bug."); } } destination.append(')'); @@ -1158,7 +1158,7 @@ public class VespaSerializer { Serializer doIt = dispatch.get(item.getClass()); if (doIt == null) { - throw new IllegalArgumentException(item.getClass() + " not supported for YQL+ marshalling."); + throw new IllegalArgumentException(item.getClass() + " not supported for YQL marshalling."); } if (state.peekFirst() != null && state.peekFirst().subItems > 0) { diff --git a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java index 31a22fd3b58..3875018609e 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java +++ b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java @@ -67,6 +67,7 @@ import com.yahoo.prelude.query.WeakAndItem; import com.yahoo.prelude.query.WeightedSetItem; import com.yahoo.prelude.query.WordAlternativesItem; import com.yahoo.prelude.query.WordItem; +import com.yahoo.processing.IllegalInputException; import com.yahoo.search.Query; import com.yahoo.search.grouping.Continuation; import com.yahoo.search.grouping.request.GroupingOperation; @@ -318,10 +319,10 @@ public class YqlParser implements Parser { private void connectItems() { for (ConnectedItem entry : connectedItems) { TaggableItem to = identifiedItems.get(entry.toId); - Preconditions.checkNotNull(to, - "Item '%s' was specified to connect to item with ID %s, which does not " - + "exist in the query.", entry.fromItem, - entry.toId); + if (to == null) + throw new IllegalArgumentException("Item '" + entry.fromItem + + "' was specified to connect to item with ID " + entry.toId + + ", which does not exist in the query."); entry.fromItem.setConnectivity((Item) to, entry.weight); } } @@ -782,7 +783,7 @@ public class YqlParser implements Parser { try { ast = new ProgramParser().parse("query", currentlyParsing.getQuery()); } catch (Exception e) { - throw new IllegalArgumentException(e); + throw new IllegalInputException(e); } assertHasOperator(ast, StatementOperator.PROGRAM); Preconditions.checkArgument(ast.getArguments().length == 1, diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcher.java b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcher.java index 8bd7bdac8a5..d06c4cd6f01 100644 --- a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcher.java +++ b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcher.java @@ -163,7 +163,7 @@ public class VdsStreamingSearcher extends VespaBackEndSearcher { query.toString(), elapsedMillis / 1000.0))); } return new Result(query, ErrorMessage.createTimeout(e.getMessage())); - } catch (InterruptedException|IllegalArgumentException e) { + } catch (InterruptedException | IllegalArgumentException e) { return new Result(query, ErrorMessage.createBackendCommunicationError(e.getMessage())); } diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java index 2b0d8adc8ef..b05ddfa3bb2 100644 --- a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java +++ b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java @@ -346,9 +346,9 @@ class VdsVisitor extends VisitorDataHandler implements Visitor { log.log(Level.FINE, "VdsVisitor completed successfully for " + query + " with selection " + params.getDocumentSelection()); } } else { - throw new IllegalArgumentException("Query failed: " // TODO: Is it necessary to use a runtime exception? - + params.getControlHandler().getResult().code + ": " - + params.getControlHandler().getResult().message); + throw new IllegalArgumentException("Query failed: " + + params.getControlHandler().getResult().code + ": " + + params.getControlHandler().getResult().message); } } @@ -425,7 +425,8 @@ class VdsVisitor extends VisitorDataHandler implements Visitor { BufferSerializer buf = new BufferSerializer( new GrowableByteBuffer(ByteBuffer.wrap(value)) ); newGrouping.deserialize(buf); if (buf.getBuf().hasRemaining()) { - throw new IllegalArgumentException("Failed deserializing grouping. There are still data left. Position = " + buf.position() + ", limit = " + buf.getBuf().limit()); + throw new IllegalArgumentException("Failed deserializing grouping. There is still data left. " + + "Position = " + buf.position() + ", limit = " + buf.getBuf().limit()); } synchronized (groupingMap) { diff --git a/container-search/src/test/java/com/yahoo/prelude/query/ItemsCommonStuffTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/ItemsCommonStuffTestCase.java index a080ae653e5..204ccff7fb3 100644 --- a/container-search/src/test/java/com/yahoo/prelude/query/ItemsCommonStuffTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/query/ItemsCommonStuffTestCase.java @@ -42,7 +42,7 @@ public class ItemsCommonStuffTestCase { a.addItem(as); try { as.addItem(a); - } catch (QueryException e) { + } catch (IllegalArgumentException e) { caught = true; } assertTrue(caught); @@ -51,7 +51,7 @@ public class ItemsCommonStuffTestCase { as.addItem(a); try { a.addItem(as); - } catch (QueryException e) { + } catch (IllegalArgumentException e) { caught = true; } assertTrue(caught); @@ -235,21 +235,21 @@ public class ItemsCommonStuffTestCase { boolean caught = false; try { as.removeItem(firstItem); - } catch (QueryException e) { + } catch (IllegalArgumentException e) { caught = true; } assertTrue(caught); caught = false; try { as.addItem(new WordItem("puppy")); - } catch (QueryException e) { + } catch (IllegalArgumentException e) { caught= true; } assertTrue(caught); caught = false; try { as.addItem(1, new WordItem("kvalp")); - } catch (QueryException e) { + } catch (IllegalArgumentException e) { caught = true; } assertTrue(caught); diff --git a/container-search/src/test/java/com/yahoo/prelude/test/QueryTestCase.java b/container-search/src/test/java/com/yahoo/prelude/test/QueryTestCase.java index 4b2ced1b771..a04834b261c 100644 --- a/container-search/src/test/java/com/yahoo/prelude/test/QueryTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/test/QueryTestCase.java @@ -8,25 +8,21 @@ import com.yahoo.prelude.Index; import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.IndexModel; import com.yahoo.prelude.SearchDefinition; -import com.yahoo.prelude.query.QueryException; import com.yahoo.prelude.query.WordItem; import com.yahoo.search.Query; import com.yahoo.search.query.Sorting; import com.yahoo.search.searchchain.Execution; import com.yahoo.yolean.Exceptions; import org.hamcrest.Matcher; -import org.junit.Before; import org.junit.Ignore; import org.junit.Test; -import java.util.Collections; import java.util.List; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.anyOf; import static org.junit.Assert.*; -import static org.junit.Assume.assumeTrue; /** * Tests for query class @@ -376,7 +372,7 @@ public class QueryTestCase { try { newQuery(queryString); fail("Above statement should throw"); - } catch (QueryException e) { + } catch (IllegalArgumentException e) { // As expected. assertThat(Exceptions.toMessageString(e), expectedErrorMessage); } diff --git a/container-search/src/test/java/com/yahoo/search/grouping/UniqueGroupingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/UniqueGroupingSearcherTestCase.java index 8ff03b35b60..86ab96a2197 100644 --- a/container-search/src/test/java/com/yahoo/search/grouping/UniqueGroupingSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/grouping/UniqueGroupingSearcherTestCase.java @@ -2,7 +2,6 @@ package com.yahoo.search.grouping; import com.yahoo.component.chain.Chain; -import com.yahoo.prelude.query.QueryException; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; @@ -41,18 +40,19 @@ public class UniqueGroupingSearcherTestCase { new MockResultProvider(0, false)); assertEquals(0, result.hits().size()); } + @Test public void testIllegalSortingSpec() { try { search("?query=foo&unique=fingerprint&sorting=-1", new MockResultProvider(0, true).addGroupList(new GroupList("fingerprint"))); fail("Above statement should throw"); - } catch (QueryException e) { + } catch (IllegalArgumentException e) { // As expected. assertThat( Exceptions.toMessageString(e), containsString( - "Invalid request parameter: Could not set 'ranking.sorting' to '-1': " + + "Could not set 'ranking.sorting' to '-1': " + "Illegal attribute name '1' for sorting. Requires '[\\[]*[a-zA-Z_][\\.a-zA-Z0-9_-]*[\\]]*'")); } } diff --git a/container-search/src/test/java/com/yahoo/search/handler/test/JSONSearchHandlerTestCase.java b/container-search/src/test/java/com/yahoo/search/handler/test/JSONSearchHandlerTestCase.java index 272092b6fc0..3cca053d0e5 100644 --- a/container-search/src/test/java/com/yahoo/search/handler/test/JSONSearchHandlerTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/handler/test/JSONSearchHandlerTestCase.java @@ -169,7 +169,7 @@ public class JSONSearchHandlerTestCase { String response = responseHandler.readAll(); assertThat(responseHandler.getStatus(), is(400)); assertThat(response, containsString("offset")); - assertThat(response, containsString("\"code\":" + com.yahoo.container.protect.Error.INVALID_QUERY_PARAMETER.code)); + assertThat(response, containsString("\"code\":" + com.yahoo.container.protect.Error.ILLEGAL_QUERY.code)); } @Test diff --git a/container-search/src/test/java/com/yahoo/search/handler/test/SearchHandlerTestCase.java b/container-search/src/test/java/com/yahoo/search/handler/test/SearchHandlerTestCase.java index c96af2ed4d1..7d03e065fa9 100644 --- a/container-search/src/test/java/com/yahoo/search/handler/test/SearchHandlerTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/handler/test/SearchHandlerTestCase.java @@ -201,7 +201,7 @@ public class SearchHandlerTestCase { String response = responseHandler.readAll(); assertThat(responseHandler.getStatus(), is(400)); assertThat(response, containsString("offset")); - assertThat(response, containsString("\"code\":" + com.yahoo.container.protect.Error.INVALID_QUERY_PARAMETER.code)); + assertThat(response, containsString("\"code\":" + com.yahoo.container.protect.Error.ILLEGAL_QUERY.code)); } @Test diff --git a/container-search/src/test/java/com/yahoo/search/query/MatchingTestCase.java b/container-search/src/test/java/com/yahoo/search/query/MatchingTestCase.java index 462e5284972..3b6b3c7d7c6 100644 --- a/container-search/src/test/java/com/yahoo/search/query/MatchingTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/MatchingTestCase.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.query; -import com.yahoo.prelude.query.QueryException; import com.yahoo.search.Query; import org.junit.Test; @@ -12,9 +11,10 @@ import static org.junit.Assert.assertEquals; * @author baldersheim */ public class MatchingTestCase { + @Test public void testDefaultsInQuery() { - Query query=new Query("?query=test"); + Query query = new Query("?query=test"); assertNull(query.getRanking().getMatching().getTermwiseLimit()); assertNull(query.getRanking().getMatching().getNumThreadsPerSearch()); assertNull(query.getRanking().getMatching().getNumSearchPartitions()); @@ -24,7 +24,7 @@ public class MatchingTestCase { @Test public void testQueryOverride() { - Query query=new Query("?query=test&ranking.matching.termwiselimit=0.7&ranking.matching.numthreadspersearch=17&ranking.matching.numsearchpartitions=13&ranking.matching.minhitsperthread=3"); + Query query = new Query("?query=test&ranking.matching.termwiselimit=0.7&ranking.matching.numthreadspersearch=17&ranking.matching.numsearchpartitions=13&ranking.matching.minhitsperthread=3"); assertEquals(Double.valueOf(0.7), query.getRanking().getMatching().getTermwiseLimit()); assertEquals(Integer.valueOf(17), query.getRanking().getMatching().getNumThreadsPerSearch()); assertEquals(Integer.valueOf(13), query.getRanking().getMatching().getNumSearchPartitions()); @@ -40,16 +40,17 @@ public class MatchingTestCase { private void verifyException(String key, String value) { try { new Query("?query=test&ranking.matching."+key+"="+value); - assertFalse(true); - } catch (QueryException e) { - assertEquals("Invalid request parameter", e.getMessage()); - assertEquals("Could not set 'ranking.matching." + key + "' to '" + value +"'", e.getCause().getMessage()); - assertEquals(key + " must be in the range [0.0, 1.0]. It is " + value, e.getCause().getCause().getMessage()); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("Could not set 'ranking.matching." + key + "' to '" + value +"'", e.getMessage()); + assertEquals(key + " must be in the range [0.0, 1.0]. It is " + value, e.getCause().getMessage()); } } + @Test public void testLimits() { verifyException("termwiselimit", "-0.1"); verifyException("termwiselimit", "1.1"); } + } diff --git a/container-search/src/test/java/com/yahoo/search/query/SoftTimeoutTestCase.java b/container-search/src/test/java/com/yahoo/search/query/SoftTimeoutTestCase.java index dff6d4c26c3..a15038ff7e2 100644 --- a/container-search/src/test/java/com/yahoo/search/query/SoftTimeoutTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/SoftTimeoutTestCase.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.query; -import com.yahoo.prelude.query.QueryException; import com.yahoo.search.Query; import org.junit.Test; import static org.junit.Assert.*; @@ -10,9 +9,10 @@ import static org.junit.Assert.*; * @author baldersheim */ public class SoftTimeoutTestCase { + @Test public void testDefaultsInQuery() { - Query query=new Query("?query=test"); + Query query = new Query("?query=test"); assertTrue(query.getRanking().getSoftTimeout().getEnable()); assertNull(query.getRanking().getSoftTimeout().getFactor()); assertNull(query.getRanking().getSoftTimeout().getTailcost()); @@ -20,7 +20,7 @@ public class SoftTimeoutTestCase { @Test public void testQueryOverride() { - Query query=new Query("?query=test&ranking.softtimeout.factor=0.7&ranking.softtimeout.tailcost=0.3"); + Query query = new Query("?query=test&ranking.softtimeout.factor=0.7&ranking.softtimeout.tailcost=0.3"); assertTrue(query.getRanking().getSoftTimeout().getEnable()); assertEquals(Double.valueOf(0.7), query.getRanking().getSoftTimeout().getFactor()); assertEquals(Double.valueOf(0.3), query.getRanking().getSoftTimeout().getTailcost()); @@ -49,13 +49,13 @@ public class SoftTimeoutTestCase { private void verifyException(String key, String value) { try { new Query("?query=test&ranking.softtimeout."+key+"="+value); - assertFalse(true); - } catch (QueryException e) { - assertEquals("Invalid request parameter", e.getMessage()); - assertEquals("Could not set 'ranking.softtimeout." + key + "' to '" + value +"'", e.getCause().getMessage()); - assertEquals(key + " must be in the range [0.0, 1.0], got " + value, e.getCause().getCause().getMessage()); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("Could not set 'ranking.softtimeout." + key + "' to '" + value +"'", e.getMessage()); + assertEquals(key + " must be in the range [0.0, 1.0], got " + value, e.getCause().getMessage()); } } + @Test public void testLimits() { verifyException("factor", "-0.1"); @@ -63,4 +63,5 @@ public class SoftTimeoutTestCase { verifyException("tailcost", "-0.1"); verifyException("tailcost", "1.1"); } + } diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/types/test/NativePropertiesTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/types/test/NativePropertiesTestCase.java index 224b0c82339..2ced2ba9323 100644 --- a/container-search/src/test/java/com/yahoo/search/query/profile/types/test/NativePropertiesTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/profile/types/test/NativePropertiesTestCase.java @@ -3,7 +3,6 @@ package com.yahoo.search.query.profile.types.test; import com.yahoo.jdisc.http.HttpRequest.Method; import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.prelude.query.QueryException; import com.yahoo.search.Query; import com.yahoo.search.query.profile.QueryProfile; import com.yahoo.search.query.profile.types.QueryProfileType; @@ -23,22 +22,22 @@ public class NativePropertiesTestCase { @Test public void testNativeInStrict() { - QueryProfileType strictType=new QueryProfileType("strict"); + QueryProfileType strictType = new QueryProfileType("strict"); strictType.setStrict(true); - QueryProfile strict=new QueryProfile("profile"); + QueryProfile strict = new QueryProfile("profile"); strict.setType(strictType); try { new Query(HttpRequest.createTestRequest("?hits=10&tracelevel=5", Method.GET), strict.compile(null)); fail("Above statement should throw"); - } catch (QueryException e) { + } catch (IllegalArgumentException e) { // As expected. } try { new Query(HttpRequest.createTestRequest("?notnative=5", Method.GET), strict.compile(null)); fail("Above statement should throw"); - } catch (QueryException e) { + } catch (IllegalArgumentException e) { // As expected. assertThat( Exceptions.toMessageString(e), diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/types/test/QueryProfileTypeTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/types/test/QueryProfileTypeTestCase.java index ecebbead866..a0bf0972281 100644 --- a/container-search/src/test/java/com/yahoo/search/query/profile/types/test/QueryProfileTypeTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/profile/types/test/QueryProfileTypeTestCase.java @@ -2,9 +2,7 @@ package com.yahoo.search.query.profile.types.test; import com.yahoo.component.ComponentId; -import com.yahoo.component.provider.ComponentRegistry; import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.prelude.query.QueryException; import com.yahoo.tensor.Tensor; import com.yahoo.yolean.Exceptions; import com.yahoo.search.Query; @@ -434,7 +432,7 @@ public class QueryProfileTypeTestCase { com.yahoo.jdisc.http.HttpRequest.Method.GET), profile.compile(null)); fail("Above statement should throw"); - } catch (QueryException e) { + } catch (IllegalArgumentException e) { // As expected. assertThat(Exceptions.toMessageString(e), containsString("Could not set 'myUserQueryProfile.nondeclared' to 'someValue': 'nondeclared' is not declared in query profile type 'userStrict', and the type is strict")); @@ -549,7 +547,7 @@ public class QueryProfileTypeTestCase { com.yahoo.jdisc.http.HttpRequest.Method.GET), cRegistry.getComponent("topMap")); fail("Above statement should throw"); - } catch (QueryException e) { + } catch (IllegalArgumentException e) { // As expected. assertThat(Exceptions.toMessageString(e), containsString("Could not set 'subMap.typeProfile.someValue' to 'value': 'someValue' is not declared in query profile type 'testtypeStrict', and the type is strict")); diff --git a/container-search/src/test/java/com/yahoo/search/querytransform/test/SortingDegraderTestCase.java b/container-search/src/test/java/com/yahoo/search/querytransform/test/SortingDegraderTestCase.java index 5a14cff1818..ab65e258477 100644 --- a/container-search/src/test/java/com/yahoo/search/querytransform/test/SortingDegraderTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/querytransform/test/SortingDegraderTestCase.java @@ -6,7 +6,6 @@ import com.yahoo.prelude.Index; import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.IndexModel; import com.yahoo.prelude.SearchDefinition; -import com.yahoo.prelude.query.QueryException; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; @@ -16,8 +15,6 @@ import com.yahoo.search.querytransform.SortingDegrader; import com.yahoo.search.searchchain.Execution; import org.junit.Test; -import java.util.Collections; - import static org.junit.Assert.*; /** @@ -67,12 +64,10 @@ public class SortingDegraderTestCase { try { Query query = new Query("?ranking.sorting=-a1%20-a2&ranking.matchPhase.maxFilterCoverage=37"); assertTrue(false); - } catch (QueryException qe) { - assertEquals("Invalid request parameter", qe.getMessage()); - Throwable setE = qe.getCause(); - assertTrue(setE instanceof IllegalArgumentException); - assertEquals("Could not set 'ranking.matchPhase.maxFilterCoverage' to '37'", setE.getMessage()); - Throwable rootE = setE.getCause(); + } catch (IllegalArgumentException qe) { + assertTrue(qe instanceof IllegalArgumentException); + assertEquals("Could not set 'ranking.matchPhase.maxFilterCoverage' to '37'", qe.getMessage()); + Throwable rootE = qe.getCause(); assertTrue(rootE instanceof IllegalArgumentException); assertEquals("maxFilterCoverage must be in the range [0.0, 1.0]. It is 37.0", rootE.getMessage()); } diff --git a/container-search/src/test/java/com/yahoo/search/searchers/test/InputCheckingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/searchers/test/InputCheckingSearcherTestCase.java index aa507d38be5..9bf5b654a6d 100644 --- a/container-search/src/test/java/com/yahoo/search/searchers/test/InputCheckingSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/searchers/test/InputCheckingSearcherTestCase.java @@ -85,7 +85,7 @@ public class InputCheckingSearcherTestCase { assertNull(r.hits().getErrorHit()); r = execution.search(new Query("/search/?query=%22a.b.0.0.0.0.0.0.c%22")); assertNotNull(r.hits().getErrorHit()); - assertEquals("More than 5 ocurrences of term '0' in a row detected in phrase : \"a b 0 0 0 0 0 0 c\"", + assertEquals("More than 5 occurrences of term '0' in a row detected in phrase : \"a b 0 0 0 0 0 0 c\"", r.hits().getErrorHit().errorIterator().next().getDetailedMessage()); r = execution.search(new Query("/search/?query=a.b.0.0.0.1.0.0.0.c")); assertNull(r.hits().getErrorHit()); @@ -97,7 +97,7 @@ public class InputCheckingSearcherTestCase { assertNull(r.hits().getErrorHit()); r = execution.search(new Query("/search/?query=%22a.b.0.0.0.0.0.0.c%22")); assertNotNull(r.hits().getErrorHit()); - assertEquals("More than 5 ocurrences of term '0' in a row detected in phrase : \"a b 0 0 0 0 0 0 c\"", + assertEquals("More than 5 occurrences of term '0' in a row detected in phrase : \"a b 0 0 0 0 0 0 c\"", r.hits().getErrorHit().errorIterator().next().getDetailedMessage()); r = execution.search(new Query("/search/?query=%22a.b.0.0.0.1.0.0.0.c%22")); assertNull(r.hits().getErrorHit()); diff --git a/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java b/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java index 3a67245e912..a42a2ee55aa 100644 --- a/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java @@ -23,7 +23,6 @@ import com.yahoo.prelude.query.IndexedItem; import com.yahoo.prelude.query.IntItem; import com.yahoo.prelude.query.Item; import com.yahoo.prelude.query.OrItem; -import com.yahoo.prelude.query.QueryException; import com.yahoo.prelude.query.RankItem; import com.yahoo.prelude.query.WordItem; import com.yahoo.processing.request.CompoundName; @@ -47,7 +46,6 @@ import java.io.UnsupportedEncodingException; import java.net.URLEncoder; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -289,7 +287,7 @@ public class QueryTestCase { try { new Query(httpEncode("/search?timeout=nalle")); fail("Above statement should throw"); - } catch (QueryException e) { + } catch (IllegalArgumentException e) { // As expected. assertTrue(Exceptions.toMessageString(e).contains("Could not set 'timeout' to 'nalle': Error parsing 'nalle': Invalid number 'nalle'")); } @@ -865,7 +863,7 @@ public class QueryTestCase { root.addItem(child); fail("Expected exception"); } - catch (QueryException e) { + catch (IllegalArgumentException e) { assertEquals("Cannot add OR (AND ) to (AND ) as it would create a cycle", e.getMessage()); } @@ -875,7 +873,7 @@ public class QueryTestCase { child.addItem(root); fail("Expected exception"); } - catch (QueryException e) { + catch (IllegalArgumentException e) { assertEquals("Cannot add (AND (OR )) to (OR ) as it would create a cycle", e.getMessage()); } } diff --git a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java index 1b8324c9de8..f5e22e30f45 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java @@ -449,8 +449,8 @@ public class YqlParserTestCase { "title contains ([{\"id\": 1, \"connectivity\": {\"id\": 4, \"weight\": 7.0}}]\"madonna\") " + "and title contains ([{\"id\": 2}]\"saint\") " + "and title contains ([{\"id\": 3}]\"angel\");", - new NullPointerException("Item 'title:madonna' was specified to connect to item with ID 4, " + - "which does not exist in the query.")); + new IllegalArgumentException("Item 'title:madonna' was specified to connect to item with ID 4, " + + "which does not exist in the query.")); } @Test diff --git a/container-search/src/test/java/com/yahoo/select/SelectTestCase.java b/container-search/src/test/java/com/yahoo/select/SelectTestCase.java index f297fd69f24..c802eb18c0f 100644 --- a/container-search/src/test/java/com/yahoo/select/SelectTestCase.java +++ b/container-search/src/test/java/com/yahoo/select/SelectTestCase.java @@ -14,6 +14,7 @@ import com.yahoo.prelude.query.SuffixItem; import com.yahoo.prelude.query.WeakAndItem; import com.yahoo.prelude.query.WordAlternativesItem; import com.yahoo.prelude.query.WordItem; +import com.yahoo.processing.IllegalInputException; import com.yahoo.search.Query; import com.yahoo.search.grouping.GroupingRequest; import com.yahoo.search.grouping.request.AllOperation; @@ -423,7 +424,7 @@ public class SelectTestCase { assertParseFail("{ \"and\": [ {\"contains\" : { \"children\" : [\"title\", \"madonna\"], \"attributes\" : {\"id\": 1, \"connectivity\": {\"id\": 4, \"weight\": 7.0}} } }, " + "{ \"contains\" : { \"children\" : [\"title\", \"saint\"], \"attributes\" : {\"id\": 2} } }, " + "{ \"contains\" : { \"children\" : [\"title\", \"angel\"], \"attributes\" : {\"id\": 3} } } ] }", - new NullPointerException("Item 'title:madonna' was specified to connect to item with ID 4, " + + new IllegalArgumentException("Item 'title:madonna' was specified to connect to item with ID 4, " + "which does not exist in the query.")); } @@ -618,7 +619,7 @@ public class SelectTestCase { parseWhere("{ \"range\" : { \"children\":[ \"foo\", { \">=\" : 0, \"<=\" : 1 }], \"attributes\" : {\"hitLimit\": 38, \"ascending\": true, \"descending\": false} } }"); } catch (IllegalArgumentException e) { assertTrue("Expected information about abuse of settings.", - e.getMessage().contains("both ascending and descending ordering set")); + e.getCause().getMessage().contains("both ascending and descending ordering set")); gotExceptionFromParse = true; } assertTrue(gotExceptionFromParse); @@ -775,12 +776,14 @@ public class SelectTestCase { private void assertParseFail(String where, Throwable expectedException) { try { parseWhere(where).toString(); - } catch (Throwable t) { - assertEquals(expectedException.getClass(), t.getClass()); - assertEquals(expectedException.getMessage(), t.getMessage()); - return; + fail("Parse succeeded: " + where); + } catch (Throwable outer) { + assertEquals(IllegalInputException.class, outer.getClass()); + assertEquals("Illegal JSON query", outer.getMessage()); + Throwable cause = outer.getCause(); + assertEquals(expectedException.getClass(), cause.getClass()); + assertEquals(expectedException.getMessage(), cause.getMessage()); } - fail("Parse succeeded: " + where); } private void assertRootClass(String where, Class<? extends Item> expectedRootClass) { diff --git a/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsVisitorTestCase.java b/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsVisitorTestCase.java index 7841b6f715c..4bfbd91eb89 100644 --- a/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsVisitorTestCase.java +++ b/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsVisitorTestCase.java @@ -130,7 +130,7 @@ public class VdsVisitorTestCase { traceLevel = 100; summary = "fancysummary"; profile = "fancyprofile"; - location = "(1,10000,2000,0,1,0)"; + location = "(2,10000,2000,0,0,1,0)"; sortSpec = "+surname -yearofbirth"; rankProperties = "rankfeature.something=2"; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/DeploymentStatistics.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/DeploymentStatistics.java index 99cf7542d53..df018d64748 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/DeploymentStatistics.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/DeploymentStatistics.java @@ -84,13 +84,14 @@ public class DeploymentStatistics { for (Deployment deployment : instance.productionDeployments().values()) allVersions.add(deployment.version()); - JobList failing = status.jobs().failing(); + JobList failing = status.jobs().failing() + .not().withStatus(RunStatus.outOfCapacity) + .not().withStatus(RunStatus.aborted); - // Add all unsuccessful runs for failing jobs as any run may have resulted in an incomplete deployment + // Add all unsuccessful runs for failing production jobs as any run may have resulted in an incomplete deployment // where a subset of nodes have upgraded. failing.not().failingApplicationChange() - .not().withStatus(RunStatus.outOfCapacity) - .not().withStatus(RunStatus.aborted) + .production() .mapToList(JobStatus::runs) .forEach(runs -> runs.descendingMap().values().stream() .dropWhile(run -> ! run.hasEnded()) @@ -101,9 +102,17 @@ public class DeploymentStatistics { failingUpgrade.get(run.versions().targetPlatform()).add(run); })); + // Add only the last failing run for test jobs. + failing.not().failingApplicationChange() + .not().production() + .lastCompleted().asList() + .forEach(run -> { + failingUpgrade.putIfAbsent(run.versions().targetPlatform(), new ArrayList<>()); + failingUpgrade.get(run.versions().targetPlatform()).add(run); + }); + + // Add only the last failing for instances failing only an application change, i.e., no upgrade. failing.failingApplicationChange() - .concat(failing.withStatus(RunStatus.outOfCapacity)) - .concat(failing.withStatus(RunStatus.aborted)) .lastCompleted().asList() .forEach(run -> { otherFailing.putIfAbsent(run.versions().targetPlatform(), new ArrayList<>()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporterTest.java index 157941bd092..dcb7a6dd42b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporterTest.java @@ -162,6 +162,10 @@ public class DeploymentIssueReporterTest { reporter.maintain(); assertTrue("We get a platform issue when confidence is broken", issues.platformIssue()); assertFalse("No deployment issue is filed for app2, which has a version upgrade failure.", issues.isOpenFor(app2.application().id())); + + app2.runJob(systemTest); + tester.controllerTester().upgradeSystem(version); + assertEquals(VespaVersion.Confidence.low, tester.controller().versionStatus().systemVersion().get().confidence()); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java index 44e172d9f93..3ea9d038d99 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java @@ -385,6 +385,86 @@ public class VersionStatusTest { } @Test + public void testConfidenceWithLingeringVersions() { + DeploymentTester tester = new DeploymentTester().atMondayMorning(); + Version version0 = new Version("6.2"); + tester.controllerTester().upgradeSystem(version0); + tester.upgrader().maintain(); + var appPackage = new ApplicationPackageBuilder().region("us-west-1").region("us-east-3").upgradePolicy("canary"); + + var canary0 = tester.newDeploymentContext("tenant1", "canary0", "default") + .submit(appPackage.build()) + .deploy(); + + assertEquals("All applications running on this version: High", + Confidence.high, confidence(tester.controller(), version0)); + + // New version is released + Version version1 = new Version("6.3"); + tester.controllerTester().upgradeSystem(version1); + tester.upgrader().maintain(); + tester.triggerJobs(); + + // App upgrades to the new version and fails + canary0.failDeployment(systemTest); + canary0.abortJob(stagingTest); + tester.controllerTester().computeVersionStatus(); + assertEquals("One canary failed: Broken", + Confidence.broken, confidence(tester.controller(), version1)); + + // New version is released + Version version2 = new Version("6.4"); + tester.controllerTester().upgradeSystem(version2); + tester.upgrader().maintain(); + assertEquals("Confidence remains unchanged for version1 until app overrides old tests: Broken", + Confidence.broken, confidence(tester.controller(), version1)); + assertEquals("Confidence defaults to low for version with no applications", + Confidence.low, confidence(tester.controller(), version2)); + assertEquals(version2, canary0.instance().change().platform().orElseThrow()); + + canary0.failDeployment(systemTest); + canary0.abortJob(stagingTest); + tester.controllerTester().computeVersionStatus(); + assertFalse("Previous version should be forgotten, as canary only had test jobs run on it", + tester.controller().versionStatus().versions().stream().anyMatch(version -> version.versionNumber().equals(version1))); + + // App succeeds with tests, but fails production deployment + canary0.runJob(systemTest) + .runJob(stagingTest) + .failDeployment(productionUsWest1); + + assertEquals("One canary failed: Broken", + Confidence.broken, confidence(tester.controller(), version2)); + + // A new version is released, and the app again fails production deployment. + Version version3 = new Version("6.5"); + tester.controllerTester().upgradeSystem(version3); + tester.upgrader().maintain(); + assertEquals("Confidence remains unchanged for version2: Broken", + Confidence.broken, confidence(tester.controller(), version2)); + assertEquals("Confidence defaults to low for version with no applications", + Confidence.low, confidence(tester.controller(), version3)); + assertEquals(version3, canary0.instance().change().platform().orElseThrow()); + + canary0.runJob(systemTest) + .runJob(stagingTest) + .failDeployment(productionUsWest1); + tester.controllerTester().computeVersionStatus(); + assertEquals("Confidence remains unchanged for version2: Broken", + Confidence.broken, confidence(tester.controller(), version2)); + assertEquals("Canary broken, so confidence for version3: Broken", + Confidence.broken, confidence(tester.controller(), version3)); + + // App succeeds production deployment, clearing failure on version2 + canary0.runJob(productionUsWest1); + tester.controllerTester().computeVersionStatus(); + assertFalse("Previous version should be forgotten, as canary only had test jobs run on it", + tester.controller().versionStatus().versions().stream().anyMatch(version -> version.versionNumber().equals(version2))); + assertEquals("Canary OK, but not done upgrading, so confidence for version3: Low", + Confidence.low, confidence(tester.controller(), version3)); + } + + @Test public void testConfidenceOverride() { DeploymentTester tester = new DeploymentTester(); Version version0 = new Version("6.2"); diff --git a/default_build_settings.cmake b/default_build_settings.cmake index 5ba7494d1e1..07a70c38d71 100644 --- a/default_build_settings.cmake +++ b/default_build_settings.cmake @@ -43,7 +43,7 @@ function(setup_vespa_default_build_settings_darwin) else() set(DEFAULT_VESPA_LLVM_VERSION "10" PARENT_SCOPE) endif() - set(DEFAULT_CMAKE_PREFIX_PATH "${VESPA_DEPS}" "/usr/local/opt/bison" "/usr/local/opt/flex" "/usr/local/opt/openssl@1.1" "/usr/local/opt/openblas" PARENT_SCOPE) + set(DEFAULT_CMAKE_PREFIX_PATH "${VESPA_DEPS}" "/usr/local/opt/bison" "/usr/local/opt/flex" "/usr/local/opt/openssl@1.1" "/usr/local/opt/openblas" "/usr/local/opt/icu4c" PARENT_SCOPE) set(DEFAULT_EXTRA_LINK_DIRECTORY "${VESPA_DEPS}/lib" "/usr/local/opt/bison/lib" "/usr/local/opt/flex/lib" "/usr/local/opt/icu4c/lib" "/usr/local/opt/openssl@1.1/lib" "/usr/local/opt/openblas/lib") if(DEFINED DEFAULT_LLVM_LINK_DIRECTORY) list(APPEND DEFAULT_EXTRA_LINK_DIRECTORY "${DEFAULT_LLVM_LINK_DIRECTORY}") @@ -96,6 +96,11 @@ function(setup_vespa_default_build_settings_ubuntu_20_04) set(DEFAULT_VESPA_LLVM_VERSION "10" PARENT_SCOPE) endfunction() +function(setup_vespa_default_build_settings_debian_10) + message("-- Setting up default build settings for debian 10") + set(DEFAULT_VESPA_LLVM_VERSION "7" PARENT_SCOPE) +endfunction() + function(vespa_use_default_vespa_unprivileged) if(NOT DEFINED VESPA_UNPRIVILEGED) message("-- Setting VESPA_UNPRIVILEGED to yes") @@ -185,6 +190,8 @@ function(vespa_use_default_build_settings) setup_vespa_default_build_settings_ubuntu_19_10() elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "ubuntu 20.04") setup_vespa_default_build_settings_ubuntu_20_04() + elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "debian 10") + setup_vespa_default_build_settings_debian_10() else() message(FATAL_ERROR "-- Unkonwn vespa build platform ${VESPA_OS_DISTRO_COMBINED}") endif() @@ -278,8 +285,8 @@ function(vespa_use_default_cxx_compiler) unset(DEFAULT_CMAKE_CXX_COMPILER) if(NOT DEFINED VESPA_COMPILER_VARIANT OR VESPA_COMPILER_VARIANT STREQUAL "gcc") if(APPLE) - set(DEFAULT_CMAKE_C_COMPILER "/usr/local/bin/gcc-9") - set(DEFAULT_CMAKE_CXX_COMPILER "/usr/local/bin/g++-9") + set(DEFAULT_CMAKE_C_COMPILER "/usr/local/bin/gcc-10") + set(DEFAULT_CMAKE_CXX_COMPILER "/usr/local/bin/g++-10") endif() elseif(VESPA_COMPILER_VARIANT STREQUAL "clang") if(APPLE) diff --git a/functions.cmake b/functions.cmake index 864fc010faa..95828430768 100644 --- a/functions.cmake +++ b/functions.cmake @@ -710,3 +710,23 @@ endfunction() function(vespa_install_empty_tmp_dir TARGET) install(DIRECTORY DESTINATION ${TARGET} DIRECTORY_PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_WRITE GROUP_EXECUTE WORLD_READ WORLD_WRITE WORLD_EXECUTE SETGID) endfunction() + +function(vespa_suppress_warnings_for_protobuf_sources) + cmake_parse_arguments( + ARG + "" + "" + "SOURCES" + ${ARGN} + ) + # protoc-generated files emit compiler warnings that we normally treat as errors. + # Instead of rolling our own compiler plugin we'll pragmatically disable the noise. + if (NOT "${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang" AND NOT "${CMAKE_CXX_COMPILER_ID}" STREQUAL "AppleClang") + if(Protobuf_VERSION VERSION_LESS "3.7.0") + set(VESPA_DISABLE_UNUSED_WARNING "-Wno-unused-parameter") + else() + unset(VESPA_DISABLE_UNUSED_WARNING) + endif() + set_source_files_properties(${ARG_SOURCES} PROPERTIES COMPILE_FLAGS "-Wno-array-bounds -Wno-suggest-override -Wno-inline ${VESPA_DISABLE_UNUSED_WARNING}") + endif() +endfunction() diff --git a/hosted-tenant-base/pom.xml b/hosted-tenant-base/pom.xml index 40fc041204d..7afafb96280 100644 --- a/hosted-tenant-base/pom.xml +++ b/hosted-tenant-base/pom.xml @@ -34,7 +34,7 @@ <properties> <vespaversion>${project.version}</vespaversion> <target_jdk_version>11</target_jdk_version> - <maven-compiler-plugin.version>3.8.0</maven-compiler-plugin.version> + <maven-compiler-plugin.version>3.8.1</maven-compiler-plugin.version> <maven-surefire-plugin.version>2.22.0</maven-surefire-plugin.version> <junit.version>5.6.2</junit.version> <!-- NOTE: this must be in sync with junit version specified in 'tenant-cd-api' --> <test.categories>!integration</test.categories> diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/test/TestTimer.java b/jdisc_core/src/main/java/com/yahoo/jdisc/test/TestTimer.java index 4d274a33a1f..980e8e008b6 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/test/TestTimer.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/test/TestTimer.java @@ -4,6 +4,8 @@ package com.yahoo.jdisc.test; import com.yahoo.jdisc.Timer; import java.time.Duration; +import java.time.Instant; +import java.util.Objects; /** * A {@link Timer} to be used in tests when the advancement of time needs to be controlled. @@ -11,21 +13,34 @@ import java.time.Duration; * @author hakonhall */ public class TestTimer implements Timer { - private Duration durationSinceEpoch = Duration.ZERO; + private Instant instant; + + public TestTimer() { + this(Instant.EPOCH); + } + + public TestTimer(Instant instant) { + this.instant = Objects.requireNonNull(instant); + } public void setMillis(long millisSinceEpoch) { - durationSinceEpoch = Duration.ofMillis(millisSinceEpoch); + instant = Instant.ofEpochMilli(millisSinceEpoch); } public void advanceMillis(long millis) { advance(Duration.ofMillis(millis)); } public void advanceSeconds(long seconds) { advance(Duration.ofSeconds(seconds)); } public void advanceMinutes(long minutes) { advance(Duration.ofMinutes(minutes)); } public void advance(Duration duration) { - durationSinceEpoch = durationSinceEpoch.plus(duration); + instant = instant.plus(duration); + } + + @Override + public Instant currentTime() { + return instant; } @Override public long currentTimeMillis() { - return durationSinceEpoch.toMillis(); + return instant.toEpochMilli(); } } diff --git a/logd/src/logd/CMakeLists.txt b/logd/src/logd/CMakeLists.txt index 92f70ef9102..cc73b51f0b1 100644 --- a/logd/src/logd/CMakeLists.txt +++ b/logd/src/logd/CMakeLists.txt @@ -8,10 +8,7 @@ protobuf_generate_cpp(logd_PROTOBUF_SRCS logd_PROTOBUF_HDRS ../../../logserver/s vespa_add_source_target(protobufgen_logd DEPENDS ${logd_PROTOBUF_SRCS} ${logd_PROTOBUF_HDRS}) -# protoc-generated files emit compiler warnings that we normally treat as errors. -if (NOT "${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang" AND NOT "${CMAKE_CXX_COMPILER_ID}" STREQUAL "AppleClang") - set_source_files_properties(${logd_PROTOBUF_SRCS} PROPERTIES COMPILE_FLAGS "-Wno-array-bounds -Wno-suggest-override") -endif() +vespa_suppress_warnings_for_protobuf_sources(SOURCES ${logd_PROTOBUF_SRCS}) vespa_add_library(logd STATIC SOURCES diff --git a/model-integration/src/test/models/onnx/simple/simple.onnx b/model-integration/src/test/models/onnx/simple/simple.onnx index 41b458451d0..88ed0ef23f0 100644 --- a/model-integration/src/test/models/onnx/simple/simple.onnx +++ b/model-integration/src/test/models/onnx/simple/simple.onnx @@ -1,10 +1,10 @@ - simple.py:ã + simple.py:ã 0 query_tensor attribute_tensormatmul"MatMul " matmul - bias_tensoroutput"addsimple_scoringZ + bias_tensoroutput"Addsimple_scoringZ query_tensor @@ -20,4 +20,4 @@ output -B
\ No newline at end of file +B
\ No newline at end of file diff --git a/model-integration/src/test/models/onnx/simple/simple.py b/model-integration/src/test/models/onnx/simple/simple.py index 878cf1c928e..512a2d75125 100755 --- a/model-integration/src/test/models/onnx/simple/simple.py +++ b/model-integration/src/test/models/onnx/simple/simple.py @@ -14,7 +14,7 @@ nodes = [ ['matmul'], ), helper.make_node( - 'add', + 'Add', ['matmul', 'bias_tensor'], ['output'], ), diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java index 27731159e9f..a02043aea78 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java @@ -127,8 +127,8 @@ public class Autoscaler { public static boolean unstable(List<Node> nodes) { return nodes.stream().anyMatch(node -> node.status().wantToRetire() || - node.allocation().get().membership().retired() || - node.allocation().get().isRemovable()); + node.allocation().get().membership().retired() || + node.allocation().get().isRemovable()); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainer.java index 20258e7947b..ff9015863e9 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainer.java @@ -87,6 +87,9 @@ public class SpareCapacityMaintainer extends NodeRepositoryMaintainer { // Report with the assumption this will eventually succeed to avoid alerting before we're stuck spareHostCapacity++; } + else { + success = false; + } } metric.set("spareHostCapacity", spareHostCapacity, null); } diff --git a/parent/pom.xml b/parent/pom.xml index c2fab3da5d9..a9998af3962 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -82,12 +82,6 @@ <groupId>org.apache.felix</groupId> <artifactId>maven-bundle-plugin</artifactId> <version>${maven-bundle-plugin.version}</version> - <configuration> - <!-- TODO: remove when bundle-plugin understands java 10, https://issues.apache.org/jira/browse/FELIX-5879 --> - <instructions> - <_noee>true</_noee> - </instructions> - </configuration> </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId> @@ -770,8 +764,9 @@ <commons.math3.version>3.6.1</commons.math3.version> <junit.version>5.6.2</junit.version> <maven-assembly-plugin.version>3.1.1</maven-assembly-plugin.version> - <maven-bundle-plugin.version>3.5.0</maven-bundle-plugin.version> - <maven-compiler-plugin.version>3.8.0</maven-compiler-plugin.version> + <!-- TODO: in order to upgrade above 4.1.0, we probably need to convert fat-model-deps to a jar artifact. --> + <maven-bundle-plugin.version>4.1.0</maven-bundle-plugin.version> + <maven-compiler-plugin.version>3.8.1</maven-compiler-plugin.version> <maven-dependency-plugin.version>3.1.1</maven-dependency-plugin.version> <maven-deploy-plugin.version>2.8.1</maven-deploy-plugin.version> <maven-enforcer-plugin.version>3.0.0-M2</maven-enforcer-plugin.version> diff --git a/processing/abi-spec.json b/processing/abi-spec.json index 8f77672faec..a354787b491 100644 --- a/processing/abi-spec.json +++ b/processing/abi-spec.json @@ -1,4 +1,17 @@ { + "com.yahoo.processing.IllegalInputException": { + "superClass": "java.lang.IllegalArgumentException", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>(java.lang.String)", + "public void <init>(java.lang.Throwable)", + "public void <init>(java.lang.String, java.lang.Throwable)" + ], + "fields": [] + }, "com.yahoo.processing.Processor": { "superClass": "com.yahoo.component.chain.ChainedComponent", "interfaces": [], diff --git a/processing/src/main/java/com/yahoo/processing/IllegalInputException.java b/processing/src/main/java/com/yahoo/processing/IllegalInputException.java new file mode 100644 index 00000000000..3f1605860ed --- /dev/null +++ b/processing/src/main/java/com/yahoo/processing/IllegalInputException.java @@ -0,0 +1,25 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.processing; + +/** + * Thrown on illegal input received from the requesting client. + * Use this instead of the superclass, IllegalArgumentException + * to signal illegal input to the client without causing logging and stack traces, + * + * @author bratseth + */ +public class IllegalInputException extends IllegalArgumentException { + + public IllegalInputException(String message) { + super(message); + } + + public IllegalInputException(Throwable cause) { + super(cause); + } + + public IllegalInputException(String message, Throwable cause) { + super(message, cause); + } + +} diff --git a/processing/src/main/java/com/yahoo/processing/request/CloneHelper.java b/processing/src/main/java/com/yahoo/processing/request/CloneHelper.java index 93b7f88f5a7..837ff3db295 100644 --- a/processing/src/main/java/com/yahoo/processing/request/CloneHelper.java +++ b/processing/src/main/java/com/yahoo/processing/request/CloneHelper.java @@ -84,11 +84,11 @@ public class CloneHelper { if (object instanceof FreezableClass) return ((FreezableClass)object).clone(); else if (object instanceof PublicCloneable) - return ((PublicCloneable)object).clone(); + return ((PublicCloneable<?>)object).clone(); else if (object instanceof LinkedList) - return ((LinkedList) object).clone(); + return ((LinkedList<?>) object).clone(); else if (object instanceof ArrayList) - return ((ArrayList) object).clone(); + return ((ArrayList<?>) object).clone(); try { Method cloneMethod = cloneMethodCache.get(object); diff --git a/processing/src/main/java/com/yahoo/processing/response/DefaultIncomingData.java b/processing/src/main/java/com/yahoo/processing/response/DefaultIncomingData.java index 60e60796afe..c436f92f78b 100644 --- a/processing/src/main/java/com/yahoo/processing/response/DefaultIncomingData.java +++ b/processing/src/main/java/com/yahoo/processing/response/DefaultIncomingData.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.processing.response; -import com.google.common.util.concurrent.ExecutionList; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.SettableFuture; import com.yahoo.collections.Tuple2; @@ -26,14 +25,10 @@ public class DefaultIncomingData<DATATYPE extends Data> implements IncomingData< private List<Tuple2<Runnable,Executor>> newDataListeners = null; - /** - * If this is completed no more data can be added - */ + /** Whether this is completed, such that no more data can be added */ private boolean complete = false; - /** - * Creates an instance which must be assigned an owner after creation - */ + /** Creates an instance which must be assigned an owner after creation */ public DefaultIncomingData() { this(null); } @@ -43,9 +38,7 @@ public class DefaultIncomingData<DATATYPE extends Data> implements IncomingData< completionFuture = SettableFuture.create(); } - /** - * Assigns the owner of this. Throws an exception if the owner is already set. - */ + /** Assigns the owner of this. Throws an exception if the owner is already set. */ public final void assignOwner(DataList<DATATYPE> owner) { if (this.owner != null) throw new NullPointerException("Owner of " + this + " was already assigned"); this.owner = owner; @@ -61,42 +54,32 @@ public class DefaultIncomingData<DATATYPE extends Data> implements IncomingData< return completionFuture; } - /** - * Returns whether the data in this is complete - */ + /** Returns whether the data in this is complete */ @Override public synchronized boolean isComplete() { return complete; } - /** - * Add new data and mark this as completed - */ + /** Adds new data and marks this as completed */ @Override public synchronized void addLast(DATATYPE data) { addLast(Collections.singletonList(data)); } - /** - * Add new data without completing this - */ + /** Adds new data without completing this */ @Override public synchronized void add(DATATYPE data) { add(Collections.singletonList(data)); } - /** - * Add new data and mark this as completed - */ + /** Adds new data and marks this as completed */ @Override public synchronized void addLast(List<DATATYPE> data) { add(data); markComplete(); } - /** - * Add new data without completing this - */ + /** Adds new data without completing this */ @Override public synchronized void add(List<DATATYPE> data) { if (complete) throw new IllegalStateException("Attempted to add data to completed " + this); @@ -105,9 +88,7 @@ public class DefaultIncomingData<DATATYPE extends Data> implements IncomingData< notifyDataListeners(); } - /** - * Mark this as completed and notify any listeners - */ + /** Marks this as completed and notify any listeners */ @Override public synchronized void markComplete() { complete = true; @@ -115,7 +96,7 @@ public class DefaultIncomingData<DATATYPE extends Data> implements IncomingData< } /** - * Get and remove all the data currently available in this. + * Gets and removes all the data currently available in this. * The returned list is a modifiable fresh instance owned by the caller. */ public synchronized List<DATATYPE> drain() { diff --git a/searchcore/src/tests/proton/matching/query_test.cpp b/searchcore/src/tests/proton/matching/query_test.cpp index 6fbd43eabbe..7cc11b0dc0c 100644 --- a/searchcore/src/tests/proton/matching/query_test.cpp +++ b/searchcore/src/tests/proton/matching/query_test.cpp @@ -698,7 +698,7 @@ void Test::requireThatQueryGluesEverythingTogether() { ASSERT_TRUE(search.get()); } -void checkQueryAddsLocation(Test &test, const string &loc_string) { +void checkQueryAddsLocation(Test &test, const string &loc_in, const string &loc_out) { fef_test::IndexEnvironment index_environment; FieldInfo field_info(FieldType::INDEX, CollectionType::SINGLE, field, 0); index_environment.getFields().push_back(field_info); @@ -712,11 +712,11 @@ void checkQueryAddsLocation(Test &test, const string &loc_string) { Query query; query.buildTree(stack_dump, - loc_field + ":" + loc_string, + loc_field + ":" + loc_in, ViewResolver(), index_environment); vector<const ITermData *> term_data; query.extractTerms(term_data); - test.EXPECT_EQUAL(1u, term_data.size()); + test.EXPECT_EQUAL(2u, term_data.size()); FakeRequestContext requestContext; FakeSearchContext context; @@ -729,9 +729,9 @@ void checkQueryAddsLocation(Test &test, const string &loc_string) { query.fetchPostings(); SearchIterator::UP search = query.createSearch(*md); test.ASSERT_TRUE(search.get()); - if (!test.EXPECT_NOT_EQUAL(string::npos, search->asString().find(loc_string))) { - fprintf(stderr, "search (missing loc_string '%s'): %s", - loc_string.c_str(), search->asString().c_str()); + if (!test.EXPECT_NOT_EQUAL(string::npos, search->asString().find(loc_out))) { + fprintf(stderr, "search (missing loc_out '%s'): %s", + loc_out.c_str(), search->asString().c_str()); } } @@ -790,11 +790,15 @@ void Test::requireThatLocationIsAddedTheCorrectPlace() { } void Test::requireThatQueryAddsLocation() { - checkQueryAddsLocation(*this, "(2,10,10,3,0,1,0,0)"); + checkQueryAddsLocation(*this, "(2,10,10,3,0,1,0,0)", "{p:{x:10,y:10},r:3,b:{x:[7,13],y:[7,13]}}"); + checkQueryAddsLocation(*this, "{p:{x:10,y:10},r:3}", "{p:{x:10,y:10},r:3,b:{x:[7,13],y:[7,13]}}"); + checkQueryAddsLocation(*this, "{b:{x:[6,11],y:[8,15]},p:{x:10,y:10},r:3}", "{p:{x:10,y:10},r:3,b:{x:[7,11],y:[8,13]}}"); + checkQueryAddsLocation(*this, "{a:12345,b:{x:[8,10],y:[8,10]},p:{x:10,y:10},r:3}", "{p:{x:10,y:10},r:3,a:12345,b:{x:[8,10],y:[8,10]}}"); } void Test::requireThatQueryAddsLocationCutoff() { - checkQueryAddsLocation(*this, "[2,10,10,20,20]"); + checkQueryAddsLocation(*this, "[2,10,11,23,24]", "{b:{x:[10,23],y:[11,24]}}"); + checkQueryAddsLocation(*this, "{b:{y:[11,24],x:[10,23]}}", "{b:{x:[10,23],y:[11,24]}}"); } void diff --git a/searchcore/src/tests/proton/matching/termdataextractor_test.cpp b/searchcore/src/tests/proton/matching/termdataextractor_test.cpp index 36c34e38a04..fb338d409e7 100644 --- a/searchcore/src/tests/proton/matching/termdataextractor_test.cpp +++ b/searchcore/src/tests/proton/matching/termdataextractor_test.cpp @@ -103,8 +103,8 @@ void Test::requireThatTermsAreAdded() { vector<const ITermData *> term_data; TermDataExtractor::extractTerms(*node, term_data); - EXPECT_EQUAL(7u, term_data.size()); - for (int i = 0; i < 7; ++i) { + EXPECT_EQUAL(8u, term_data.size()); + for (int i = 0; i < 8; ++i) { EXPECT_EQUAL(id[i], term_data[i]->getUniqueId()); EXPECT_EQUAL(1u, term_data[i]->numFields()); } @@ -118,8 +118,8 @@ void Test::requireThatAViewWithTwoFieldsGivesOneTermDataPerTerm() { vector<const ITermData *> term_data; TermDataExtractor::extractTerms(*node, term_data); - EXPECT_EQUAL(7u, term_data.size()); - for (int i = 0; i < 7; ++i) { + EXPECT_EQUAL(8u, term_data.size()); + for (int i = 0; i < 8; ++i) { EXPECT_EQUAL(id[i], term_data[i]->getUniqueId()); EXPECT_EQUAL(2u, term_data[i]->numFields()); } diff --git a/searchcore/src/vespa/searchcore/proton/matching/query.cpp b/searchcore/src/vespa/searchcore/proton/matching/query.cpp index 65959e6e6de..994e26081e7 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/query.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/query.cpp @@ -83,7 +83,7 @@ GeoLocationSpec parse_location_string(string str) { return empty; } GeoLocationParser parser; - if (parser.parseOldFormatWithField(str)) { + if (parser.parseWithField(str)) { auto attr_name = PositionDataType::getZCurveFieldName(parser.getFieldName()); return GeoLocationSpec{attr_name, parser.getGeoLocation()}; } else { diff --git a/searchcore/src/vespa/searchcore/proton/matching/termdataextractor.cpp b/searchcore/src/vespa/searchcore/proton/matching/termdataextractor.cpp index c0007ae2bda..673355f9354 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/termdataextractor.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/termdataextractor.cpp @@ -29,8 +29,6 @@ public: } } - void visit(ProtonLocationTerm &) override {} - virtual void visit(ProtonNodeTypes::AndNot &n) override { assert(n.getChildren().size() > 0); n.getChildren()[0]->accept(*this); diff --git a/searchcore/src/vespa/searchcore/proton/server/documentretriever.cpp b/searchcore/src/vespa/searchcore/proton/server/documentretriever.cpp index 031dd4f35e7..952c359b716 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentretriever.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentretriever.cpp @@ -261,6 +261,7 @@ DocumentRetriever::getPartialDocument(search::DocumentIdT lid, const document::D case FieldSet::Type::DOCID: break; } + doc->setRepo(getDocumentTypeRepo()); } return doc; } diff --git a/searchlib/src/tests/common/location/geo_location_test.cpp b/searchlib/src/tests/common/location/geo_location_test.cpp index 31b844d0fc8..d142733041e 100644 --- a/searchlib/src/tests/common/location/geo_location_test.cpp +++ b/searchlib/src/tests/common/location/geo_location_test.cpp @@ -4,6 +4,7 @@ #include <vespa/searchlib/common/geo_location.h> #include <vespa/searchlib/common/geo_location_spec.h> #include <vespa/searchlib/common/geo_location_parser.h> +#include <vespa/searchlib/query/tree/location.h> #include <vespa/vespalib/gtest/gtest.h> using search::common::GeoLocation; @@ -16,16 +17,22 @@ using Aspect = search::common::GeoLocation::Aspect; constexpr int32_t plus_inf = std::numeric_limits<int32_t>::max(); constexpr int32_t minus_inf = std::numeric_limits<int32_t>::min(); -constexpr uint32_t u32_inf = std::numeric_limits<uint32_t>::max(); -bool is_parseable(const char *str) { +bool is_parseable(const char *str, bool with_field = false) { GeoLocationParser parser; - return parser.parseOldFormat(str); + if (with_field) { + return parser.parseWithField(str); + } + return parser.parseNoField(str); } -GeoLocation parse(const char *str) { +GeoLocation parse(const std::string &str, bool with_field = false) { GeoLocationParser parser; - EXPECT_TRUE(parser.parseOldFormat(str)); + if (with_field) { + EXPECT_TRUE(parser.parseWithField(str)); + } else { + EXPECT_TRUE(parser.parseNoField(str)); + } return parser.getGeoLocation(); } @@ -40,6 +47,15 @@ TEST(GeoLocationParserTest, malformed_bounding_boxes_are_not_parseable) { EXPECT_FALSE(is_parseable("[10,20,30,40]")); } +TEST(GeoLocationParserTest, new_bounding_box_formats) { + EXPECT_TRUE(is_parseable("{b:{x:[10,30],y:[20,40]}}")); + EXPECT_TRUE(is_parseable("{b:{}}")); + EXPECT_TRUE(is_parseable("{b:[]}")); + EXPECT_TRUE(is_parseable("{b:10,b:20}")); + EXPECT_TRUE(is_parseable("{b:[10, 20, 30, 40]}")); + EXPECT_FALSE(is_parseable("{b:{x:[10,30],y:[20,40]}")); +} + TEST(GeoLocationParserTest, malformed_circles_are_not_parseable) { EXPECT_TRUE(is_parseable("(2,10,20,5,0,0,0)")); EXPECT_FALSE(is_parseable("(2,10,20,5,0,0,0)(2,10,20,5,0,0,0)")); @@ -52,94 +68,147 @@ TEST(GeoLocationParserTest, malformed_circles_are_not_parseable) { EXPECT_FALSE(is_parseable("(10,20,5)")); } +TEST(GeoLocationParserTest, new_circle_formats) { + EXPECT_TRUE(is_parseable("{p:{x:10,y:20}}")); + EXPECT_TRUE(is_parseable("{p:{x:10,y:20},r:5}")); + EXPECT_TRUE(is_parseable("{p:{x:10, y:10}, r:5}")); + EXPECT_TRUE(is_parseable("{'p':{y:20,x:10},'r':5}")); + EXPECT_TRUE(is_parseable("{\n \"p\": { \"x\": 10, \"y\": 20},\n \"r\": 5\n}")); + // json demands colon: + EXPECT_FALSE(is_parseable("{p:{x:10,y:10},r=5}")); + // missing y -> 0 default: + EXPECT_TRUE(is_parseable("{p:{x:10},r:5}")); + // unused extra fields are ignored: + EXPECT_TRUE(is_parseable("{p:{x:10,y:10,z:10},r:5,c:1,d:17}")); +} + TEST(GeoLocationParserTest, bounding_boxes_can_be_parsed) { - auto loc = parse("[2,10,20,30,40]"); - EXPECT_EQ(false, loc.has_point); - EXPECT_EQ(true, loc.bounding_box.active()); - EXPECT_EQ(0u, loc.x_aspect.multiplier); - EXPECT_EQ(0, loc.point.x); - EXPECT_EQ(0, loc.point.y); - EXPECT_EQ(std::numeric_limits<uint32_t>::max(), loc.radius); - EXPECT_EQ(10, loc.bounding_box.x.low); - EXPECT_EQ(20, loc.bounding_box.y.low); - EXPECT_EQ(30, loc.bounding_box.x.high); - EXPECT_EQ(40, loc.bounding_box.y.high); + for (const auto & loc : { + parse("[2,10,20,30,40]"), + parse("{b:{x:[10,30],y:[20,40]}}") + }) { + EXPECT_EQ(false, loc.has_point); + EXPECT_EQ(true, loc.bounding_box.active()); + EXPECT_EQ(0u, loc.x_aspect.multiplier); + EXPECT_EQ(0, loc.point.x); + EXPECT_EQ(0, loc.point.y); + EXPECT_EQ(std::numeric_limits<uint32_t>::max(), loc.radius); + EXPECT_EQ(10, loc.bounding_box.x.low); + EXPECT_EQ(20, loc.bounding_box.y.low); + EXPECT_EQ(30, loc.bounding_box.x.high); + EXPECT_EQ(40, loc.bounding_box.y.high); + } } TEST(GeoLocationParserTest, circles_can_be_parsed) { - auto loc = parse("(2,10,20,5,0,0,0)"); - EXPECT_EQ(true, loc.has_point); - EXPECT_EQ(true, loc.bounding_box.active()); - EXPECT_EQ(0u, loc.x_aspect.multiplier); - EXPECT_EQ(10, loc.point.x); - EXPECT_EQ(20, loc.point.y); - EXPECT_EQ(5u, loc.radius); - EXPECT_EQ(5, loc.bounding_box.x.low); - EXPECT_EQ(15, loc.bounding_box.y.low); - EXPECT_EQ(15, loc.bounding_box.x.high); - EXPECT_EQ(25, loc.bounding_box.y.high); + for (const auto & loc : { + parse("(2,10,20,5,0,0,0)"), + parse("{p:{x:10,y:20},r:5}") + }) { + EXPECT_EQ(true, loc.has_point); + EXPECT_EQ(true, loc.bounding_box.active()); + EXPECT_EQ(0u, loc.x_aspect.multiplier); + EXPECT_EQ(10, loc.point.x); + EXPECT_EQ(20, loc.point.y); + EXPECT_EQ(5u, loc.radius); + EXPECT_EQ(5, loc.bounding_box.x.low); + EXPECT_EQ(15, loc.bounding_box.y.low); + EXPECT_EQ(15, loc.bounding_box.x.high); + EXPECT_EQ(25, loc.bounding_box.y.high); + } } TEST(GeoLocationParserTest, circles_can_have_aspect_ratio) { - auto loc = parse("(2,10,20,5,0,0,0,2147483648)"); - EXPECT_EQ(true, loc.has_point); - EXPECT_EQ(true, loc.bounding_box.active()); - EXPECT_EQ(2147483648u, loc.x_aspect.multiplier); - EXPECT_EQ(10, loc.point.x); - EXPECT_EQ(20, loc.point.y); - EXPECT_EQ(5u, loc.radius); - EXPECT_EQ(-1, loc.bounding_box.x.low); - EXPECT_EQ(15, loc.bounding_box.y.low); - EXPECT_EQ(21, loc.bounding_box.x.high); - EXPECT_EQ(25, loc.bounding_box.y.high); + for (const auto & loc : { + parse("(2,10,20,5,0,0,0,2147483648)"), + parse("{p:{x:10,y:20},r:5,a:2147483648}") + }) { + EXPECT_EQ(true, loc.has_point); + EXPECT_EQ(true, loc.bounding_box.active()); + EXPECT_EQ(2147483648u, loc.x_aspect.multiplier); + EXPECT_EQ(10, loc.point.x); + EXPECT_EQ(20, loc.point.y); + EXPECT_EQ(5u, loc.radius); + EXPECT_EQ(-1, loc.bounding_box.x.low); + EXPECT_EQ(15, loc.bounding_box.y.low); + EXPECT_EQ(21, loc.bounding_box.x.high); + EXPECT_EQ(25, loc.bounding_box.y.high); + } + auto loc2 = parse("{p:{x:10,y:10},a:3123456789}"); + EXPECT_EQ(3123456789, loc2.x_aspect.multiplier); } TEST(GeoLocationParserTest, bounding_box_can_be_specified_after_circle) { - auto loc = parse("(2,10,20,5,0,0,0)[2,10,20,30,40]"); - EXPECT_EQ(true, loc.has_point); - EXPECT_EQ(true, loc.bounding_box.active()); - EXPECT_EQ(0u, loc.x_aspect.multiplier); - EXPECT_EQ(10, loc.point.x); - EXPECT_EQ(20, loc.point.y); - EXPECT_EQ(5u, loc.radius); - EXPECT_EQ(10, loc.bounding_box.x.low); - EXPECT_EQ(20, loc.bounding_box.y.low); - EXPECT_EQ(15, loc.bounding_box.x.high); - EXPECT_EQ(25, loc.bounding_box.y.high); + for (const auto & loc : { + parse("(2,10,20,5,0,0,0)[2,10,20,30,40]"), + parse("{p:{x:10,y:20},r:5,b:{x:[10,30],y:[20,40]}}") + }) { + EXPECT_EQ(true, loc.has_point); + EXPECT_EQ(true, loc.bounding_box.active()); + EXPECT_EQ(0u, loc.x_aspect.multiplier); + EXPECT_EQ(10, loc.point.x); + EXPECT_EQ(20, loc.point.y); + EXPECT_EQ(5u, loc.radius); + EXPECT_EQ(10, loc.bounding_box.x.low); + EXPECT_EQ(20, loc.bounding_box.y.low); + EXPECT_EQ(15, loc.bounding_box.x.high); + EXPECT_EQ(25, loc.bounding_box.y.high); + } } TEST(GeoLocationParserTest, circles_can_be_specified_after_bounding_box) { - auto loc = parse("[2,10,20,30,40](2,10,20,5,0,0,0)"); + for (const auto & loc : { + parse("[2,10,20,30,40](2,10,20,5,0,0,0)"), + parse("{b:{x:[10,30],y:[20,40]},p:{x:10,y:20},r:5}") + }) { + EXPECT_EQ(true, loc.has_point); + EXPECT_EQ(true, loc.bounding_box.active()); + EXPECT_EQ(0u, loc.x_aspect.multiplier); + EXPECT_EQ(10, loc.point.x); + EXPECT_EQ(20, loc.point.y); + EXPECT_EQ(5u, loc.radius); + EXPECT_EQ(10, loc.bounding_box.x.low); + EXPECT_EQ(20, loc.bounding_box.y.low); + EXPECT_EQ(15, loc.bounding_box.x.high); + EXPECT_EQ(25, loc.bounding_box.y.high); + } + const auto &loc = parse("{a:12345,b:{x:[8,10],y:[8,10]},p:{x:10,y:10},r:3}"); EXPECT_EQ(true, loc.has_point); - EXPECT_EQ(true, loc.bounding_box.active()); - EXPECT_EQ(0u, loc.x_aspect.multiplier); EXPECT_EQ(10, loc.point.x); - EXPECT_EQ(20, loc.point.y); - EXPECT_EQ(5u, loc.radius); - EXPECT_EQ(10, loc.bounding_box.x.low); - EXPECT_EQ(20, loc.bounding_box.y.low); - EXPECT_EQ(15, loc.bounding_box.x.high); - EXPECT_EQ(25, loc.bounding_box.y.high); + EXPECT_EQ(10, loc.point.y); + EXPECT_EQ(12345u, loc.x_aspect.multiplier); } TEST(GeoLocationParserTest, santa_search_gives_non_wrapped_bounding_box) { - auto loc = parse("(2,122163600,89998536,290112,4,2000,0,109704)"); - EXPECT_GE(loc.bounding_box.x.high, loc.bounding_box.x.low); - EXPECT_GE(loc.bounding_box.y.high, loc.bounding_box.y.low); + for (const auto & loc : { + parse("(2,122163600,89998536,290112,4,2000,0,109704)"), + parse("{p:{x:122163600,y:89998536},r:290112,a:109704}") + }) { + EXPECT_GE(loc.bounding_box.x.high, loc.bounding_box.x.low); + EXPECT_GE(loc.bounding_box.y.high, loc.bounding_box.y.low); + } } TEST(GeoLocationParserTest, near_boundary_search_gives_non_wrapped_bounding_box) { - auto loc1 = parse("(2,2000000000,2000000000,3000000000,0,1,0)"); - EXPECT_GE(loc1.bounding_box.x.high, loc1.bounding_box.x.low); - EXPECT_GE(loc1.bounding_box.y.high, loc1.bounding_box.y.low); - EXPECT_EQ(std::numeric_limits<int32_t>::max(), loc1.bounding_box.y.high); - EXPECT_EQ(std::numeric_limits<int32_t>::max(), loc1.bounding_box.y.high); - - auto loc2 = parse("(2,-2000000000,-2000000000,3000000000,0,1,0)"); - EXPECT_GE(loc2.bounding_box.x.high, loc2.bounding_box.x.low); - EXPECT_GE(loc2.bounding_box.y.high, loc2.bounding_box.y.low); - EXPECT_EQ(std::numeric_limits<int32_t>::min(), loc2.bounding_box.x.low); - EXPECT_EQ(std::numeric_limits<int32_t>::min(), loc2.bounding_box.y.low); + for (const auto & loc1 : { + parse("(2,2000000000,2000000000,3000000000,0,1,0)"), + parse("{p:{x:2000000000,y:2000000000},r:3000000000}") + }) { + EXPECT_GE(loc1.bounding_box.x.high, loc1.bounding_box.x.low); + EXPECT_GE(loc1.bounding_box.y.high, loc1.bounding_box.y.low); + EXPECT_EQ(std::numeric_limits<int32_t>::max(), loc1.bounding_box.y.high); + EXPECT_EQ(std::numeric_limits<int32_t>::max(), loc1.bounding_box.y.high); + } + + for (const auto & loc2 : { + parse("(2,-2000000000,-2000000000,3000000000,0,1,0)"), + parse("{p:{x:-2000000000,y:-2000000000},r:3000000000}") + }) { + EXPECT_GE(loc2.bounding_box.x.high, loc2.bounding_box.x.low); + EXPECT_GE(loc2.bounding_box.y.high, loc2.bounding_box.y.low); + EXPECT_EQ(std::numeric_limits<int32_t>::min(), loc2.bounding_box.x.low); + EXPECT_EQ(std::numeric_limits<int32_t>::min(), loc2.bounding_box.y.low); + } } void check_box(const GeoLocation &location, Box expected) @@ -391,4 +460,67 @@ TEST(GeoLocationTest, box_point_radius_and_aspect) { EXPECT_EQ(location.bounding_box.y.high, 700); } +TEST(GeoLocationParserTest, can_parse_what_query_tree_produces) { + search::query::Point point_1{-17, 42}; + uint32_t distance = 12345; + uint32_t aspect_ratio = 67890; + search::query::Rectangle rectangle_1(-1, -2, 3, 4); + + search::query::Location loc_1(point_1); + std::string str_1 = loc_1.getJsonFormatString(); + auto result_1 = parse(str_1); + + EXPECT_EQ(true, result_1.has_point); + EXPECT_EQ(false, result_1.has_radius()); + EXPECT_EQ(false, result_1.x_aspect.active()); + EXPECT_EQ(false, result_1.bounding_box.active()); + EXPECT_EQ(-17, result_1.point.x); + EXPECT_EQ(42, result_1.point.y); + + search::query::Location loc_1b(point_1, distance, aspect_ratio); + std::string str_1b = loc_1b.getJsonFormatString(); + auto result_1b = parse(str_1b); + + EXPECT_EQ(true, result_1b.has_point); + EXPECT_EQ(true, result_1b.has_radius()); + EXPECT_EQ(true, result_1b.x_aspect.active()); + EXPECT_EQ(true, result_1b.bounding_box.active()); + EXPECT_EQ(-17, result_1b.point.x); + EXPECT_EQ(42, result_1b.point.y); + EXPECT_EQ(distance, result_1b.radius); + EXPECT_EQ(aspect_ratio, result_1b.x_aspect.multiplier); + EXPECT_EQ(42-distance, result_1b.bounding_box.y.low); + EXPECT_EQ(42+distance, result_1b.bounding_box.y.high); + + search::query::Location loc_2(rectangle_1); + std::string str_2 = loc_2.getJsonFormatString(); + auto result_2 = parse(str_2); + + EXPECT_EQ(false, result_2.has_point); + EXPECT_EQ(false, result_2.has_radius()); + EXPECT_EQ(false, result_2.x_aspect.active()); + EXPECT_EQ(true, result_2.bounding_box.active()); + EXPECT_EQ(-1, result_2.bounding_box.x.low); + EXPECT_EQ(-2, result_2.bounding_box.y.low); + EXPECT_EQ(3, result_2.bounding_box.x.high); + EXPECT_EQ(4, result_2.bounding_box.y.high); + + search::query::Location loc_3(rectangle_1, point_1, distance, aspect_ratio); + std::string str_3 = loc_3.getJsonFormatString(); + auto result_3 = parse(str_3); + + EXPECT_EQ(true, result_3.has_point); + EXPECT_EQ(true, result_3.has_radius()); + EXPECT_EQ(true, result_3.x_aspect.active()); + EXPECT_EQ(true, result_3.bounding_box.active()); + EXPECT_EQ(-17, result_3.point.x); + EXPECT_EQ(42, result_3.point.y); + EXPECT_EQ(distance, result_3.radius); + EXPECT_EQ(aspect_ratio, result_3.x_aspect.multiplier); + EXPECT_EQ(-1, result_3.bounding_box.x.low); + EXPECT_EQ(-2, result_3.bounding_box.y.low); + EXPECT_EQ(3, result_3.bounding_box.x.high); + EXPECT_EQ(4, result_3.bounding_box.y.high); +} + GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/tests/tensor/distance_functions/distance_functions_test.cpp b/searchlib/src/tests/tensor/distance_functions/distance_functions_test.cpp index 082d04f104b..9d240bc688a 100644 --- a/searchlib/src/tests/tensor/distance_functions/distance_functions_test.cpp +++ b/searchlib/src/tests/tensor/distance_functions/distance_functions_test.cpp @@ -25,7 +25,7 @@ void verify_geo_miles(const DistanceFunction *dist_fun, double abstract_distance = dist_fun->calc(t1, t2); double raw_score = dist_fun->to_rawscore(abstract_distance); double m = ((1.0/raw_score)-1.0); - double d_miles = m / 1609.344; + double d_miles = m / 1.609344; EXPECT_GE(d_miles, exp_miles*0.99); EXPECT_LE(d_miles, exp_miles*1.01); } @@ -159,7 +159,7 @@ TEST(GeoDegreesTest, gives_expected_score) double g63_a = geodeg->calc(t(g6_trd), t(g3_osl)); double g63_r = geodeg->to_rawscore(g63_a); - double g63_km = ((1.0/g63_r)-1.0) *.001; + double g63_km = ((1.0/g63_r)-1.0); EXPECT_GT(g63_km, 350); EXPECT_LT(g63_km, 375); diff --git a/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp b/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp index 032caaacba8..b9e4bf565ef 100644 --- a/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp @@ -263,8 +263,18 @@ public: const common::Location &location() const { return _location; } SearchIterator::UP - createLeafSearch(const TermFieldMatchDataArray &, bool strict) const override + createLeafSearch(const TermFieldMatchDataArray &tfmda, bool strict) const override { + if (tfmda.size() == 1) { + // search in exactly one field + fef::TermFieldMatchData &tfmd = *tfmda[0]; + return search::common::create_location_iterator(tfmd, + _attribute.getNumDocs(), + strict, + _location); + } else { + LOG(debug, "wrong size tfmda: %zu (fallback to old location iterator)\n", tfmda.size()); + } return FastS_AllocLocationIterator(_attribute.getNumDocs(), strict, _location); } }; @@ -273,7 +283,8 @@ public: Blueprint::UP make_location_blueprint(const FieldSpec &field, const IAttributeVector &attribute, const Location &loc) { - LOG(debug, "make_location_blueprint(p[%d,%d], r[%u], aspect[%u], bb[[%d,%d],[%d,%d]])", + LOG(debug, "make_location_blueprint(fieldId[%u], p[%d,%d], r[%u], aspect[%u], bb[[%d,%d],[%d,%d]])", + field.getFieldId(), loc.point.x, loc.point.y, loc.radius, loc.x_aspect.multiplier, loc.bounding_box.x.low, loc.bounding_box.x.high, diff --git a/searchlib/src/vespa/searchlib/common/geo_location_parser.cpp b/searchlib/src/vespa/searchlib/common/geo_location_parser.cpp index 05c53348699..7d793f9eb55 100644 --- a/searchlib/src/vespa/searchlib/common/geo_location_parser.cpp +++ b/searchlib/src/vespa/searchlib/common/geo_location_parser.cpp @@ -3,6 +3,11 @@ #include "geo_location_parser.h" #include <limits> #include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/vespalib/data/slime/slime.h> +#include <vespa/vespalib/data/slime/json_format.h> + +#include <vespa/log/log.h> +LOG_SETUP(".searchlib.common.geo_location_parser"); namespace { @@ -39,7 +44,7 @@ GeoLocationParser::GeoLocationParser() _max_x(std::numeric_limits<int32_t>::max()), _min_y(std::numeric_limits<int32_t>::min()), _max_y(std::numeric_limits<int32_t>::max()), - _parseError(NULL) + _parseError(nullptr) {} bool @@ -58,19 +63,6 @@ GeoLocationParser::correctDimensionalitySkip(const char * &p) { } bool -GeoLocationParser::parseOldFormatWithField(const std::string &str) -{ - auto sep = str.find(':'); - if (sep == std::string::npos) { - _parseError = "Location string lacks field specification."; - return false; - } - _field_name = str.substr(0, sep); - std::string only_loc = str.substr(sep + 1); - return parseOldFormat(only_loc); -} - -bool GeoLocationParser::parseOldFormat(const std::string &locStr) { bool foundBoundingBox = false; @@ -179,9 +171,87 @@ GeoLocationParser::parseOldFormat(const std::string &locStr) return _valid; } +bool +GeoLocationParser::parseWithField(const std::string &str) +{ + auto sep = str.find(':'); + if (sep == std::string::npos) { + _parseError = "Location string lacks field specification"; + return false; + } + _field_name = str.substr(0, sep); + std::string only_loc = str.substr(sep + 1); + return parseNoField(only_loc); +} + +bool +GeoLocationParser::parseNoField(const std::string &str) +{ + if (str.empty()) { + _parseError = "Location string is empty"; + return false; + } + if (str[0] == '(' || str[0] == '[') { + return parseOldFormat(str); + } + if (str[0] != '{') { + _parseError = "Location string should start with '{'"; + return false; + } + return parseJsonFormat(str); +} + +bool +GeoLocationParser::parseJsonFormat(const std::string &str) +{ + vespalib::Slime slime; + size_t decoded = vespalib::slime::JsonFormat::decode(str, slime); + if (decoded == 0) { + LOG(warning, "bad location JSON: %s\n>> %s <<", + slime.get()["error_message"].asString().make_string().c_str(), + str.c_str()); + _parseError = "Failed decoding JSON format location"; + return false; + } + // fprintf(stderr, "parsed location JSON %s -> %s\n", str.c_str(), slime.toString().c_str()); + const auto &root = slime.get(); + const auto &point = root["p"]; + const auto &radius = root["r"]; + const auto &aspect = root["a"]; + const auto &bbox = root["b"]; + + if (point.valid()) { + _x = point["x"].asLong(); + _y = point["y"].asLong(); + _has_point = true; + } + if (radius.valid()) { + _radius = radius.asLong(); + } + if (aspect.valid()) { + _x_aspect = aspect.asLong(); + } + if (bbox.valid()) { + _min_x = bbox["x"][0].asLong(); + _max_x = bbox["x"][1].asLong(); + _min_y = bbox["y"][0].asLong(); + _max_y = bbox["y"][1].asLong(); + _has_bounding_box = true; + } + if (_has_point || _has_bounding_box) { + _valid = true; + } else { + _parseError = "Neither point nor bounding box found"; + } + return _valid; +} + GeoLocation GeoLocationParser::getGeoLocation() const { + if (! _valid) { + return GeoLocation(); + } GeoLocation::Aspect aspect(_x_aspect); if (_has_bounding_box) { GeoLocation::Range x_range{_min_x, _max_x}; diff --git a/searchlib/src/vespa/searchlib/common/geo_location_parser.h b/searchlib/src/vespa/searchlib/common/geo_location_parser.h index 8936a620d21..e1a68163fbb 100644 --- a/searchlib/src/vespa/searchlib/common/geo_location_parser.h +++ b/searchlib/src/vespa/searchlib/common/geo_location_parser.h @@ -17,8 +17,8 @@ class GeoLocationParser public: GeoLocationParser(); - bool parseOldFormat(const std::string &locStr); - bool parseOldFormatWithField(const std::string &str); + bool parseNoField(const std::string &locStr); + bool parseWithField(const std::string &locStr); std::string getFieldName() const { return _field_name; } GeoLocation getGeoLocation() const; @@ -42,6 +42,8 @@ private: const char *_parseError; bool correctDimensionalitySkip(const char * &p); + bool parseJsonFormat(const std::string &locStr); + bool parseOldFormat(const std::string &locStr); }; } // namespace diff --git a/searchlib/src/vespa/searchlib/common/locationiterators.cpp b/searchlib/src/vespa/searchlib/common/locationiterators.cpp index d90ed3b41f3..413930522c6 100644 --- a/searchlib/src/vespa/searchlib/common/locationiterators.cpp +++ b/searchlib/src/vespa/searchlib/common/locationiterators.cpp @@ -7,6 +7,108 @@ #include <vespa/log/log.h> LOG_SETUP(".searchlib.common.locationiterators"); +namespace search::common { + +class LocationIterator : public search::queryeval::SearchIterator +{ +private: + static constexpr double pi = 3.14159265358979323846; + // microdegrees -> degrees -> radians -> km (using Earth mean radius) + static constexpr double udeg_to_km = 1.0e-6 * (pi / 180.0) * 6371.0088; + search::fef::TermFieldMatchData & _tfmd; + const unsigned int _numDocs; + const bool _strict; + const Location & _location; + uint32_t _num_values; + std::vector<search::AttributeVector::largeint_t> _pos; + + void doSeek(uint32_t docId) override; + void doUnpack(uint32_t docId) override; +public: + LocationIterator(search::fef::TermFieldMatchData &tfmd, + unsigned int numDocs, + bool strict, + const Location & location); + ~LocationIterator() override; +}; + +LocationIterator::LocationIterator(search::fef::TermFieldMatchData &tfmd, + unsigned int numDocs, + bool strict, + const Location & location) + : SearchIterator(), + _tfmd(tfmd), + _numDocs(numDocs), + _strict(strict), + _location(location), + _num_values(0), + _pos() +{ + _pos.resize(1); //Need at least 1 entry as the singlevalue attributes does not honour given size. + LOG(debug, "created LocationIterator(numDocs=%u)\n", numDocs); +}; + + +LocationIterator::~LocationIterator() = default; + +void +LocationIterator::doSeek(uint32_t docId) +{ + while (__builtin_expect(docId < getEndId(), true)) { + if (__builtin_expect(docId >= _numDocs, false)) { + break; + } + _num_values = _location.getVec()->get(docId, &_pos[0], _pos.size()); + while (_num_values > _pos.size()) { + _pos.resize(_num_values); + _num_values = _location.getVec()->get(docId, &_pos[0], _pos.size()); + } + for (uint32_t i = 0; i < _num_values; i++) { + int64_t docxy(_pos[i]); + if (_location.inside_limit(docxy)) { + setDocId(docId); + return; + } + } + if (!_strict) { + return; + } + ++docId; + } + setAtEnd(); +} + +void +LocationIterator::doUnpack(uint32_t docId) +{ + uint64_t sqabsdist = std::numeric_limits<uint64_t>::max(); + int32_t docx = 0; + int32_t docy = 0; + // use _num_values from _pos fetched in doSeek() + for (uint32_t i = 0; i < _num_values; i++) { + int64_t docxy(_pos[i]); + vespalib::geo::ZCurve::decode(docxy, &docx, &docy); + uint64_t sqdist = _location.sq_distance_to({docx, docy}); + if (sqdist < sqabsdist) { + sqabsdist = sqdist; + } + } + double dist = std::sqrt(double(sqabsdist)); + double score = 1.0 / (1.0 + (udeg_to_km * dist)); + LOG(debug, "unpack LI(%u) score %f\n", docId, score); + LOG(debug, "distance: %f micro-degrees ~= %f km", dist, udeg_to_km * dist); + _tfmd.setRawScore(docId, score); +} + +std::unique_ptr<search::queryeval::SearchIterator> +create_location_iterator(search::fef::TermFieldMatchData &tfmd, unsigned int numDocs, + bool strict, const Location & location) +{ + return std::make_unique<LocationIterator>(tfmd, numDocs, strict, location); +} + +} // namespace + using namespace search::common; class FastS_2DZLocationIterator : public search::queryeval::SearchIterator diff --git a/searchlib/src/vespa/searchlib/common/locationiterators.h b/searchlib/src/vespa/searchlib/common/locationiterators.h index e345bcae4fe..d963ac2e479 100644 --- a/searchlib/src/vespa/searchlib/common/locationiterators.h +++ b/searchlib/src/vespa/searchlib/common/locationiterators.h @@ -4,9 +4,19 @@ #include <vespa/searchlib/queryeval/searchiterator.h> #include <vespa/searchlib/common/location.h> +#include <vespa/searchlib/fef/termfieldmatchdata.h> + +namespace search::common { + +std::unique_ptr<search::queryeval::SearchIterator> +create_location_iterator(search::fef::TermFieldMatchData &tfmd, + unsigned int numDocs, + bool strict, + const Location & location); + +} // namespace std::unique_ptr<search::queryeval::SearchIterator> FastS_AllocLocationIterator(unsigned int numDocs, bool strict, const search::common::Location & location); - diff --git a/searchlib/src/vespa/searchlib/engine/CMakeLists.txt b/searchlib/src/vespa/searchlib/engine/CMakeLists.txt index 082af18d32b..570118de0af 100644 --- a/searchlib/src/vespa/searchlib/engine/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/engine/CMakeLists.txt @@ -5,11 +5,7 @@ protobuf_generate_cpp(searchlib_engine_PROTOBUF_SRCS searchlib_engine_PROTOBUF_H vespa_add_source_target(protobufgen_searchlib_engine DEPENDS ${searchlib_engine_PROTOBUF_SRCS} ${searchlib_engine_PROTOBUF_HDRS}) -# protoc-generated files emit compiler warnings that we normally treat as errors. -if (NOT "${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang" AND NOT "${CMAKE_CXX_COMPILER_ID}" STREQUAL "AppleClang") - set_source_files_properties(${searchlib_engine_PROTOBUF_SRCS} - PROPERTIES COMPILE_FLAGS "-Wno-array-bounds -Wno-suggest-override") -endif() +vespa_suppress_warnings_for_protobuf_sources(SOURCES ${searchlib_engine_PROTOBUF_SRCS}) vespa_add_library(searchlib_engine OBJECT SOURCES diff --git a/searchlib/src/vespa/searchlib/features/distancefeature.cpp b/searchlib/src/vespa/searchlib/features/distancefeature.cpp index 78a94c4abe0..d2431ccc06b 100644 --- a/searchlib/src/vespa/searchlib/features/distancefeature.cpp +++ b/searchlib/src/vespa/searchlib/features/distancefeature.cpp @@ -83,8 +83,8 @@ feature_t DistanceExecutor::calculateDistance(uint32_t docId) { _best_index = -1.0; - _best_x = 0.0; - _best_y = 0.0; + _best_x = -180.0 * 1.0e6; + _best_y = 90.0 * 1.0e6; if ((! _locations.empty()) && (_pos != nullptr)) { LOG(debug, "calculate 2D Z-distance from %zu locations", _locations.size()); return calculate2DZDistance(docId); @@ -135,8 +135,8 @@ DistanceExecutor::execute(uint32_t docId) { outputs().set_number(0, calculateDistance(docId)); outputs().set_number(1, _best_index); - outputs().set_number(2, _best_y * 0.000001); // latitude - outputs().set_number(3, _best_x * 0.000001); // longitude + outputs().set_number(2, _best_y * 1.0e-6); // latitude + outputs().set_number(3, _best_x * 1.0e-6); // longitude } const feature_t DistanceExecutor::DEFAULT_DISTANCE(6400000000.0); diff --git a/searchlib/src/vespa/searchlib/fef/iqueryenvironment.h b/searchlib/src/vespa/searchlib/fef/iqueryenvironment.h index 7c6a84916f4..d51b29cbfe3 100644 --- a/searchlib/src/vespa/searchlib/fef/iqueryenvironment.h +++ b/searchlib/src/vespa/searchlib/fef/iqueryenvironment.h @@ -6,7 +6,7 @@ #include "objectstore.h" #include <vespa/searchcommon/attribute/iattributecontext.h> -namespace search::common { class GeoLocationSpec; } +namespace search::common { struct GeoLocationSpec; } namespace search::fef { diff --git a/searchlib/src/vespa/searchlib/fef/termfieldmatchdata.h b/searchlib/src/vespa/searchlib/fef/termfieldmatchdata.h index efa2e3f5bfa..633d85f98df 100644 --- a/searchlib/src/vespa/searchlib/fef/termfieldmatchdata.h +++ b/searchlib/src/vespa/searchlib/fef/termfieldmatchdata.h @@ -260,7 +260,7 @@ public: * This indicates if this instance is actually used for ranking or not. * @return true if it is not needed. */ - bool isNotNeeded() const { return ((_flags & (UNPACK_NORMAL_FEATURES_FLAG | UNPACK_INTERLEAVED_FEATURES_FLAG)) == 0u); } + bool isNotNeeded() const { return ((_flags & (UNPACK_NORMAL_FEATURES_FLAG | UNPACK_INTERLEAVED_FEATURES_FLAG)) == 0u); } bool needs_normal_features() const { return ((_flags & UNPACK_NORMAL_FEATURES_FLAG) != 0u); } diff --git a/searchlib/src/vespa/searchlib/query/tree/location.cpp b/searchlib/src/vespa/searchlib/query/tree/location.cpp index 6e678f9e682..9b45ba18b97 100644 --- a/searchlib/src/vespa/searchlib/query/tree/location.cpp +++ b/searchlib/src/vespa/searchlib/query/tree/location.cpp @@ -33,12 +33,13 @@ Location::Location(const Rectangle &rect) bool Location::operator==(const Location &other) const { - auto me = getOldFormatString(); - auto it = other.getOldFormatString(); + auto me = getJsonFormatString(); + auto it = other.getJsonFormatString(); if (me == it) { return true; } else { // dump 'me' and 'it' here if unit tests fail + // fprintf(stderr, "me='%s', it='%s'\n", me.c_str(), it.c_str()); return false; } } @@ -69,8 +70,37 @@ Location::getOldFormatString() const return buf.str(); } -vespalib::asciistream &operator<<(vespalib::asciistream &out, const Location &loc) { - return out << loc.getOldFormatString(); +std::string +Location::getJsonFormatString() const +{ + // Only produce what search::common::GeoLocationParser can parse + vespalib::asciistream buf; + buf << "{"; + if (has_point) { + buf << "p:{x:" << point.x << ",y:" << point.y << "}"; + if (has_radius()) { + buf << "," << "r:" << radius; + } + if (x_aspect.active()) { + buf << "," << "a:" << x_aspect.multiplier; + } + } + if (bounding_box.active()) { + if (has_point) { + buf << ","; + } + buf << "b:{x:[" << bounding_box.x.low + << "," << bounding_box.x.high + << "],y:[" << bounding_box.y.low + << "," << bounding_box.y.high + << "]}" ; + } + buf << "}"; + return buf.str(); } +vespalib::asciistream &operator<<(vespalib::asciistream &out, const Location &loc) { + return out << loc.getJsonFormatString(); } + +} // namespace diff --git a/searchlib/src/vespa/searchlib/query/tree/location.h b/searchlib/src/vespa/searchlib/query/tree/location.h index 6b8090f45e1..143282e2958 100644 --- a/searchlib/src/vespa/searchlib/query/tree/location.h +++ b/searchlib/src/vespa/searchlib/query/tree/location.h @@ -22,6 +22,7 @@ public: bool operator==(const Location &other) const; std::string getOldFormatString() const; + std::string getJsonFormatString() const; }; vespalib::asciistream &operator<<(vespalib::asciistream &out, const Location &loc); diff --git a/searchlib/src/vespa/searchlib/query/tree/stackdumpquerycreator.h b/searchlib/src/vespa/searchlib/query/tree/stackdumpquerycreator.h index a7e00d41555..66702fcd85c 100644 --- a/searchlib/src/vespa/searchlib/query/tree/stackdumpquerycreator.h +++ b/searchlib/src/vespa/searchlib/query/tree/stackdumpquerycreator.h @@ -144,7 +144,9 @@ private: t = &builder.addSuffixTerm(term, view, id, weight); } else if (type == ParseItem::ITEM_GEO_LOCATION_TERM) { search::common::GeoLocationParser parser; - parser.parseOldFormat(term); + if (! parser.parseNoField(term)) { + LOG(warning, "invalid geo location term '%s'", term.data()); + } Location loc(parser.getGeoLocation()); t = &builder.addLocationTerm(loc, view, id, weight); } else if (type == ParseItem::ITEM_NUMTERM) { diff --git a/searchlib/src/vespa/searchlib/tensor/distance_functions.h b/searchlib/src/vespa/searchlib/tensor/distance_functions.h index 7e75920619f..f88b239885f 100644 --- a/searchlib/src/vespa/searchlib/tensor/distance_functions.h +++ b/searchlib/src/vespa/searchlib/tensor/distance_functions.h @@ -165,8 +165,8 @@ public: } double to_rawscore(double distance) const override { double hav_diff = sqrt(distance); - // distance in meters: - double d = 2 * asin(hav_diff) * 6371008.8; // Earth mean radius + // distance in kilometers: + double d = 2 * asin(hav_diff) * 6371.0088; // Earth mean radius return 1.0 / (1.0 + d); } double calc_with_limit(const vespalib::tensor::TypedCells& lhs, diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.cpp index 1f9b553b56a..822017e0bdf 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.cpp @@ -76,7 +76,7 @@ GetDocsumsState::parse_locations() assert(_parsedLocations.empty()); // only allowed to call this once if (! _args.getLocation().empty()) { GeoLocationParser parser; - if (parser.parseOldFormatWithField(_args.getLocation())) { + if (parser.parseWithField(_args.getLocation())) { auto view = parser.getFieldName(); auto attr_name = PositionDataType::getZCurveFieldName(view); GeoLocationSpec spec{attr_name, parser.getGeoLocation()}; @@ -94,7 +94,7 @@ GetDocsumsState::parse_locations() vespalib::string view = iterator.getIndexName(); vespalib::string term = iterator.getTerm(); GeoLocationParser parser; - if (parser.parseOldFormat(term)) { + if (parser.parseNoField(term)) { auto attr_name = PositionDataType::getZCurveFieldName(view); GeoLocationSpec spec{attr_name, parser.getGeoLocation()}; _parsedLocations.push_back(spec); diff --git a/storage/src/vespa/storage/distributor/distributor.h b/storage/src/vespa/storage/distributor/distributor.h index 84e195fdff2..fcc08030764 100644 --- a/storage/src/vespa/storage/distributor/distributor.h +++ b/storage/src/vespa/storage/distributor/distributor.h @@ -25,11 +25,11 @@ #include <unordered_map> namespace storage { + struct DoneInitializeHandler; + class HostInfo; +} -struct DoneInitializeHandler; -class HostInfo; - -namespace distributor { +namespace storage::distributor { class DistributorBucketSpaceRepo; class SimpleMaintenanceScanner; @@ -342,5 +342,4 @@ private: bool _must_send_updated_host_info; }; -} // distributor -} // storage +} diff --git a/storage/src/vespa/storage/distributor/operations/external/visitoroperation.cpp b/storage/src/vespa/storage/distributor/operations/external/visitoroperation.cpp index 70dd215cc1d..5a03e05d563 100644 --- a/storage/src/vespa/storage/distributor/operations/external/visitoroperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/external/visitoroperation.cpp @@ -41,6 +41,8 @@ VisitorOperation::BucketInfo::toString() const return ost.str(); } +VisitorOperation::SuperBucketInfo::~SuperBucketInfo() = default; + VisitorOperation::VisitorOperation( DistributorComponent& owner, DistributorBucketSpace &bucketSpace, diff --git a/storage/src/vespa/storage/distributor/operations/external/visitoroperation.h b/storage/src/vespa/storage/distributor/operations/external/visitoroperation.h index fdfe60731f5..42b0bd56b9e 100644 --- a/storage/src/vespa/storage/distributor/operations/external/visitoroperation.h +++ b/storage/src/vespa/storage/distributor/operations/external/visitoroperation.h @@ -74,6 +74,7 @@ private: subBucketsCompletelyExpanded(false) { } + ~SuperBucketInfo(); }; diff --git a/storage/src/vespa/storage/persistence/fieldvisitor.h b/storage/src/vespa/storage/persistence/fieldvisitor.h index 99558fe6e9c..688874742a6 100644 --- a/storage/src/vespa/storage/persistence/fieldvisitor.h +++ b/storage/src/vespa/storage/persistence/fieldvisitor.h @@ -15,7 +15,7 @@ namespace storage { class FieldVisitor : public document::select::Visitor { private: - document::DocumentType _docType; + const document::DocumentType & _docType; document::Field::Set::Builder _fields; public: diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp index f536def28a0..4df2b5e591b 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp @@ -14,9 +14,6 @@ #include <vespa/storageapi/message/stat.h> #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/vespalib/util/exceptions.h> -#ifndef XXH_INLINE_ALL -# define XXH_INLINE_ALL // Let XXH64 be inlined for fixed hash size (bucket ID) -#endif #include <xxhash.h> #include <vespa/log/log.h> diff --git a/storage/src/vespa/storage/persistence/testandsethelper.cpp b/storage/src/vespa/storage/persistence/testandsethelper.cpp index 1dbcbcc3fc6..57586249817 100644 --- a/storage/src/vespa/storage/persistence/testandsethelper.cpp +++ b/storage/src/vespa/storage/persistence/testandsethelper.cpp @@ -70,7 +70,7 @@ TestAndSetHelper::retrieveAndMatch(spi::Context & context) { auto docPtr = result.getDocumentPtr(); if (_docSelectionUp->contains(*docPtr) != document::select::Result::True) { return api::ReturnCode(api::ReturnCode::TEST_AND_SET_CONDITION_FAILED, - vespalib::make_string("Condition did not match document partition=%d, nodeIndex=%d bucket=%lx %s", + vespalib::make_string("Condition did not match document partition=%d, nodeIndex=%d bucket=%" PRIx64 " %s", _thread._env._partition, _thread._env._nodeIndex, _cmd.getBucketId().getRawId(), _cmd.hasBeenRemapped() ? "remapped" : "")); } @@ -82,7 +82,7 @@ TestAndSetHelper::retrieveAndMatch(spi::Context & context) { } return api::ReturnCode(api::ReturnCode::TEST_AND_SET_CONDITION_FAILED, - vespalib::make_string("Document does not exist partition=%d, nodeIndex=%d bucket=%lx %s", + vespalib::make_string("Document does not exist partition=%d, nodeIndex=%d bucket=%" PRIx64 " %s", _thread._env._partition, _thread._env._nodeIndex, _cmd.getBucketId().getRawId(), _cmd.hasBeenRemapped() ? "remapped" : "")); } diff --git a/storage/src/vespa/storage/storageserver/storagenodecontext.cpp b/storage/src/vespa/storage/storageserver/storagenodecontext.cpp index 75e1f12773f..49390d4e579 100644 --- a/storage/src/vespa/storage/storageserver/storagenodecontext.cpp +++ b/storage/src/vespa/storage/storageserver/storagenodecontext.cpp @@ -13,4 +13,6 @@ StorageNodeContext::StorageNodeContext(ComponentRegister::UP compReg, framework: _componentRegister->setThreadPool(_threadPool); } +StorageNodeContext::~StorageNodeContext() = default; + } // storage diff --git a/storage/src/vespa/storage/storageserver/storagenodecontext.h b/storage/src/vespa/storage/storageserver/storagenodecontext.h index eabca618bfb..163c02ef5af 100644 --- a/storage/src/vespa/storage/storageserver/storagenodecontext.h +++ b/storage/src/vespa/storage/storageserver/storagenodecontext.h @@ -40,6 +40,8 @@ struct StorageNodeContext { */ FastOS_ThreadPool& getThreadPool() { return _threadPool.getThreadPool(); } + ~StorageNodeContext(); + protected: // Initialization has been split in two as subclass needs to initialize // component register before sending it on. diff --git a/storageapi/src/vespa/storageapi/mbusprot/CMakeLists.txt b/storageapi/src/vespa/storageapi/mbusprot/CMakeLists.txt index 1b1e224f034..b749844775d 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/CMakeLists.txt +++ b/storageapi/src/vespa/storageapi/mbusprot/CMakeLists.txt @@ -9,11 +9,8 @@ PROTOBUF_GENERATE_CPP(storageapi_PROTOBUF_SRCS storageapi_PROTOBUF_HDRS vespa_add_source_target(protobufgen_storageapi_mbusprot DEPENDS ${storageapi_PROTOBUF_SRCS} ${storageapi_PROTOBUF_HDRS}) -# protoc-generated files emit compiler warnings that we normally treat as errors. -# Instead of rolling our own compiler plugin we'll pragmatically disable the noise. -if (NOT "${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang" AND NOT "${CMAKE_CXX_COMPILER_ID}" STREQUAL "AppleClang") - set_source_files_properties(${storageapi_PROTOBUF_SRCS} PROPERTIES COMPILE_FLAGS "-Wno-array-bounds -Wno-suggest-override -Wno-inline") -endif() +vespa_suppress_warnings_for_protobuf_sources(SOURCES ${storageapi_PROTOBUF_SRCS}) + # protoc explicitly annotates methods with inline, which triggers -Werror=inline when # the header file grows over a certain size. set_source_files_properties(protocolserialization7.cpp PROPERTIES COMPILE_FLAGS "-Wno-inline") diff --git a/storageapi/src/vespa/storageapi/message/visitor.cpp b/storageapi/src/vespa/storageapi/message/visitor.cpp index faf58361276..d87f65a72cf 100644 --- a/storageapi/src/vespa/storageapi/message/visitor.cpp +++ b/storageapi/src/vespa/storageapi/message/visitor.cpp @@ -4,6 +4,7 @@ #include <vespa/document/fieldset/fieldsets.h> #include <vespa/vespalib/util/array.hpp> #include <climits> +#include <ostream> namespace storage::api { diff --git a/streamingvisitors/src/vespa/searchvisitor/queryenvironment.cpp b/streamingvisitors/src/vespa/searchvisitor/queryenvironment.cpp index 06bdb9f69bb..383f6c3d44b 100644 --- a/streamingvisitors/src/vespa/searchvisitor/queryenvironment.cpp +++ b/streamingvisitors/src/vespa/searchvisitor/queryenvironment.cpp @@ -26,7 +26,7 @@ parseLocation(const string & location_str) return fefLocations; } GeoLocationParser locationParser; - if (!locationParser.parseOldFormatWithField(location_str)) { + if (!locationParser.parseWithField(location_str)) { LOG(warning, "Location parse error (location: '%s'): %s. Location ignored.", location_str.c_str(), locationParser.getParseError()); return fefLocations; diff --git a/tenant-base/pom.xml b/tenant-base/pom.xml index addaedec7df..ed5c32563c7 100644 --- a/tenant-base/pom.xml +++ b/tenant-base/pom.xml @@ -35,7 +35,7 @@ <vespaversion>${project.version}</vespaversion> <test-framework.version>${project.version}</test-framework.version> <target_jdk_version>11</target_jdk_version> - <maven-compiler-plugin.version>3.8.0</maven-compiler-plugin.version> + <maven-compiler-plugin.version>3.8.1</maven-compiler-plugin.version> <maven-surefire-plugin.version>2.22.0</maven-surefire-plugin.version> <junit.version>5.6.2</junit.version> <!-- Keep in sync with hosted-tenant-base and tenant-cd until all direct use is removed --> <endpoint>https://api.vespa-external.aws.oath.cloud:4443</endpoint> diff --git a/vespa-hadoop/pom.xml b/vespa-hadoop/pom.xml index 7e1a8d832b2..794eb038137 100644 --- a/vespa-hadoop/pom.xml +++ b/vespa-hadoop/pom.xml @@ -198,7 +198,7 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-compiler-plugin</artifactId> - <version>3.8.0</version> + <version>3.8.1</version> <configuration> <jdkToolchain> <version>${java.version}</version> diff --git a/vespalib/src/vespa/vespalib/testkit/test_master.hpp b/vespalib/src/vespa/vespalib/testkit/test_master.hpp index fc163c8d14e..8a5f7eb337b 100644 --- a/vespalib/src/vespa/vespalib/testkit/test_master.hpp +++ b/vespalib/src/vespa/vespalib/testkit/test_master.hpp @@ -4,6 +4,15 @@ namespace vespalib { +#if !defined(__clang__) && defined(__GNUC__) && __GNUC__ < 9 +// cf. https://cplusplus.github.io/LWG/issue2221 +template<class charT, class traits> +std::basic_ostream<charT, traits>& operator<<(std::basic_ostream<charT, traits>& os, nullptr_t) +{ + return os << (void*) nullptr; +} +#endif + template<class A, class B, class OP> bool TestMaster::compare(const char *file, uint32_t line, |