diff options
132 files changed, 1761 insertions, 1412 deletions
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java index 9ae9a158631..f9338f9cb35 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java @@ -7,7 +7,6 @@ import com.yahoo.container.handler.ThreadpoolConfig; import com.yahoo.search.config.QrStartConfig; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.component.Handler; -import com.yahoo.vespa.model.container.component.SystemBindingPattern; /** * @author hmusum @@ -40,7 +39,7 @@ public class LogserverContainerCluster extends ContainerCluster<LogserverContain private void addLogHandler() { Handler<?> logHandler = Handler.fromClassName(ContainerCluster.LOG_HANDLER_CLASS); - logHandler.addServerBindings(SystemBindingPattern.fromHttpPath("/logs")); + logHandler.addServerBindings("http://*/logs"); addComponent(logHandler); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java index 5b3e4e1479e..08f4e2fa12f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java @@ -14,7 +14,6 @@ import com.yahoo.vespa.model.container.Container; import com.yahoo.vespa.model.container.component.AccessLogComponent; import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.container.component.Handler; -import com.yahoo.vespa.model.container.component.SystemBindingPattern; import com.yahoo.vespa.model.container.xml.PlatformBundles; import java.util.Set; @@ -37,10 +36,10 @@ public class ClusterControllerContainer extends Container implements super(parent, "" + index, index); addHandler("clustercontroller-status", "com.yahoo.vespa.clustercontroller.apps.clustercontroller.StatusHandler", - "/clustercontroller-status/*"); + "clustercontroller-status/*"); addHandler("clustercontroller-state-restapi-v2", "com.yahoo.vespa.clustercontroller.apps.clustercontroller.StateRestApiV2Handler", - "/cluster/v2/*"); + "cluster/v2/*"); if (runStandaloneZooKeeper) { addComponent("clustercontroller-zkrunner", "com.yahoo.vespa.zookeeper.VespaZooKeeperServerImpl", @@ -78,8 +77,8 @@ public class ClusterControllerContainer extends Container implements return ContainerServiceType.CLUSTERCONTROLLER_CONTAINER; } - private void addHandler(Handler h, String path) { - h.addServerBindings(SystemBindingPattern.fromHttpPath(path)); + private void addHandler(Handler h, String binding) { + h.addServerBindings("http://*/" + binding); super.addHandler(h); } @@ -97,8 +96,9 @@ public class ClusterControllerContainer extends Container implements addComponent(new Component<>(createComponentModel(id, className, bundle))); } - private void addHandler(String id, String className, String path) { - addHandler(new Handler(createComponentModel(id, className, CLUSTERCONTROLLER_BUNDLE)), path); + private void addHandler(String id, String className, String binding) { + addHandler(new Handler(createComponentModel(id, className, CLUSTERCONTROLLER_BUNDLE)), + binding); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java index b5936887b50..4dc9811a024 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java @@ -7,12 +7,12 @@ import ai.vespa.metricsproxy.core.MetricsConsumers; import ai.vespa.metricsproxy.core.MetricsManager; import ai.vespa.metricsproxy.core.MonitoringConfig; import ai.vespa.metricsproxy.core.VespaMetrics; +import ai.vespa.metricsproxy.http.metrics.MetricsV1Handler; import ai.vespa.metricsproxy.http.application.ApplicationMetricsHandler; import ai.vespa.metricsproxy.http.application.ApplicationMetricsRetriever; import ai.vespa.metricsproxy.http.application.MetricsNodesConfig; -import ai.vespa.metricsproxy.http.metrics.MetricsV1Handler; -import ai.vespa.metricsproxy.http.prometheus.PrometheusHandler; import ai.vespa.metricsproxy.http.yamas.YamasHandler; +import ai.vespa.metricsproxy.http.prometheus.PrometheusHandler; import ai.vespa.metricsproxy.metric.ExternalMetrics; import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensions; import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensionsConfig; @@ -38,7 +38,6 @@ import com.yahoo.vespa.model.admin.monitoring.MetricsConsumer; import com.yahoo.vespa.model.admin.monitoring.Monitoring; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.component.Handler; -import com.yahoo.vespa.model.container.component.SystemBindingPattern; import com.yahoo.vespa.model.container.xml.PlatformBundles; import java.nio.file.Path; @@ -130,9 +129,8 @@ public class MetricsProxyContainerCluster extends ContainerCluster<MetricsProxyC static Handler<AbstractConfigProducer<?>> createMetricsHandler(Class<? extends ThreadedHttpRequestHandler> clazz, String bindingPath) { Handler<AbstractConfigProducer<?>> metricsHandler = new Handler<>( new ComponentModel(clazz.getName(), null, METRICS_PROXY_BUNDLE_NAME, null)); - metricsHandler.addServerBindings( - SystemBindingPattern.fromHttpPath(bindingPath), - SystemBindingPattern.fromHttpPath(bindingPath + "/*")); + metricsHandler.addServerBindings("http://*" + bindingPath, + "http://*" + bindingPath + "/*"); return metricsHandler; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/UriBindingsValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/UriBindingsValidator.java deleted file mode 100644 index 249827b11d9..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/UriBindingsValidator.java +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.application.validation; - -import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.vespa.model.VespaModel; -import com.yahoo.vespa.model.container.ApplicationContainerCluster; -import com.yahoo.vespa.model.container.component.BindingPattern; -import com.yahoo.vespa.model.container.component.Handler; -import com.yahoo.vespa.model.container.component.SystemBindingPattern; -import com.yahoo.vespa.model.container.http.FilterBinding; -import com.yahoo.vespa.model.container.http.Http; - -import java.util.logging.Level; - -import static com.yahoo.config.model.ConfigModelContext.ApplicationType.HOSTED_INFRASTRUCTURE; - -/** - * Validates URI bindings for filters and handlers - * - * @author bjorncs - */ -class UriBindingsValidator extends Validator { - - @Override - public void validate(VespaModel model, DeployState deployState) { - for (ApplicationContainerCluster cluster : model.getContainerClusters().values()) { - for (Handler<?> handler : cluster.getHandlers()) { - for (BindingPattern binding : handler.getServerBindings()) { - validateUserBinding(binding, model, deployState); - } - } - Http http = cluster.getHttp(); - if (http != null) { - for (FilterBinding binding : cluster.getHttp().getBindings()) { - validateUserBinding(binding.binding(), model, deployState); - } - } - } - } - - private static void validateUserBinding(BindingPattern binding, VespaModel model, DeployState deployState) { - validateScheme(binding, deployState); - if (isHostedApplication(model, deployState)) { - validateHostedApplicationUserBinding(binding); - } - } - - private static void validateScheme(BindingPattern binding, DeployState deployState) { - if (binding.scheme().equals("https")) { - String message = createErrorMessage( - binding, "'https' bindings are deprecated, use 'http' instead to bind to both http and https traffic."); - deployState.getDeployLogger().log(Level.WARNING, message); - } - } - - private static void validateHostedApplicationUserBinding(BindingPattern binding) { - // only perform these validation for used-generated bindings - // bindings produced by the hosted config model amender will violate some of the rules below - if (binding instanceof SystemBindingPattern) return; - - if (binding.port().isPresent()) { - throw new IllegalArgumentException(createErrorMessage(binding, "binding with port is not allowed")); - } - if (!binding.host().equals(BindingPattern.WILDCARD_PATTERN)) { - throw new IllegalArgumentException(createErrorMessage(binding, "only binding with wildcard ('*') for hostname is allowed")); - } - if (!binding.scheme().equals("http") && !binding.scheme().equals("https")) { - throw new IllegalArgumentException(createErrorMessage(binding, "only 'http' is allowed as scheme")); - } - } - - private static boolean isHostedApplication(VespaModel model, DeployState deployState) { - return deployState.isHosted() && model.getAdmin().getApplicationType() != HOSTED_INFRASTRUCTURE; - } - - private static String createErrorMessage(BindingPattern binding, String message) { - return String.format("For binding '%s': %s", binding.patternString(), message); - } - -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java index 3a4dee300da..22dd0289390 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java @@ -61,7 +61,6 @@ public class Validation { new AccessControlFilterValidator().validate(model, deployState); new CloudWatchValidator().validate(model, deployState); new AwsAccessControlValidator().validate(model, deployState); - new UriBindingsValidator().validate(model, deployState); List<ConfigChangeAction> result = Collections.emptyList(); if (deployState.getProperties().isFirstTimeDeployment()) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomClientProviderBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomClientProviderBuilder.java index 0fdd1af56f3..11fab0ada29 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomClientProviderBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomClientProviderBuilder.java @@ -2,12 +2,11 @@ package com.yahoo.vespa.model.builder.xml.dom; import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.text.XML; +import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.container.component.Handler; -import com.yahoo.vespa.model.container.component.UserBindingPattern; import org.w3c.dom.Element; /** @@ -25,10 +24,10 @@ public class DomClientProviderBuilder extends DomHandlerBuilder { Handler<? super Component<?, ?>> client = createHandler(clientElement); for (Element binding : XML.getChildren(clientElement, "binding")) - client.addClientBindings(UserBindingPattern.fromPattern(XML.getValue(binding))); + client.addClientBindings(XML.getValue(binding)); for (Element serverBinding : XML.getChildren(clientElement, "serverBinding")) - client.addServerBindings(UserBindingPattern.fromPattern(XML.getValue(serverBinding))); + client.addServerBindings(XML.getValue(serverBinding)); DomComponentBuilder.addChildren(deployState, parent, clientElement, client); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java index 145535fe06f..ac6d089cf24 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java @@ -8,10 +8,8 @@ import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.text.XML; import com.yahoo.vespa.model.container.ApplicationContainerCluster; -import com.yahoo.vespa.model.container.component.BindingPattern; import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.container.component.Handler; -import com.yahoo.vespa.model.container.component.UserBindingPattern; import com.yahoo.vespa.model.container.xml.BundleInstantiationSpecificationBuilder; import org.w3c.dom.Element; @@ -29,14 +27,11 @@ import static java.util.logging.Level.INFO; */ public class DomHandlerBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Handler> { - private static final Set<BindingPattern> reservedBindings = - Set.of( - METRICS_V2_HANDLER_BINDING_1, - METRICS_V2_HANDLER_BINDING_2, - STATE_HANDLER_BINDING_1, - STATE_HANDLER_BINDING_2, - VIP_HANDLER_BINDING); - + private static final Set<String> reservedBindings = Set.of(METRICS_V2_HANDLER_BINDING_1, + METRICS_V2_HANDLER_BINDING_2, + STATE_HANDLER_BINDING_1, + STATE_HANDLER_BINDING_2, + VIP_HANDLER_BINDING); private final ApplicationContainerCluster cluster; public DomHandlerBuilder(ApplicationContainerCluster cluster) { @@ -48,10 +43,10 @@ public class DomHandlerBuilder extends VespaDomBuilder.DomConfigProducerBuilder< Handler<? super Component<?, ?>> handler = createHandler(handlerElement); for (Element binding : XML.getChildren(handlerElement, "binding")) - addServerBinding(handler, UserBindingPattern.fromPattern(XML.getValue(binding)), deployState.getDeployLogger()); + addServerBinding(handler, XML.getValue(binding), deployState.getDeployLogger()); for (Element clientBinding : XML.getChildren(handlerElement, "clientBinding")) - handler.addClientBindings(UserBindingPattern.fromPattern(XML.getValue(clientBinding))); + handler.addClientBindings(XML.getValue(clientBinding)); DomComponentBuilder.addChildren(deployState, parent, handlerElement, handler); @@ -63,30 +58,27 @@ public class DomHandlerBuilder extends VespaDomBuilder.DomConfigProducerBuilder< return new Handler<>(new ComponentModel(bundleSpec)); } - private void addServerBinding(Handler<? super Component<?, ?>> handler, BindingPattern binding, DeployLogger log) { + private void addServerBinding(Handler<? super Component<?, ?>> handler, String binding, DeployLogger log) { throwIfBindingIsReserved(binding, handler); handler.addServerBindings(binding); removeExistingServerBinding(binding, handler, log); } - private void throwIfBindingIsReserved(BindingPattern binding, Handler<?> newHandler) { + private void throwIfBindingIsReserved(String binding, Handler<?> newHandler) { for (var reserved : reservedBindings) { - if (binding.hasSamePattern(reserved)) { - throw new IllegalArgumentException("Binding '" + binding.patternString() + "' is a reserved Vespa binding and " + + if (binding.equals(reserved)) { + throw new IllegalArgumentException("Binding '" + binding + "' is a reserved Vespa binding and " + "cannot be used by handler: " + newHandler.getComponentId()); } } } - private void removeExistingServerBinding(BindingPattern binding, Handler<?> newHandler, DeployLogger log) { + private void removeExistingServerBinding(String binding, Handler<?> newHandler, DeployLogger log) { for (var handler : cluster.getHandlers()) { - for (BindingPattern serverBinding : handler.getServerBindings()) { - if (serverBinding.hasSamePattern(binding)) { - handler.removeServerBinding(serverBinding); - log.log(INFO, "Binding '" + binding.patternString() + "' was already in use by handler '" + - handler.getComponentId() + "', but will now be taken over by handler: " + newHandler.getComponentId()); - - } + if (handler.getServerBindings().contains(binding)) { + handler.removeServerBinding(binding); + log.log(INFO, "Binding '" + binding + "' was already in use by handler '" + + handler.getComponentId() + "', but will now be taken over by handler: " + newHandler.getComponentId()); } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java b/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java index 159a87be27d..58f03bffb30 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java @@ -6,8 +6,6 @@ import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.component.Handler; -import com.yahoo.vespa.model.container.component.SystemBindingPattern; -import com.yahoo.vespa.model.container.component.UserBindingPattern; import java.util.Collection; import java.util.Collections; @@ -26,7 +24,7 @@ public class ContainerDocumentApi { } private void setupHandlers(ContainerCluster cluster) { - cluster.addComponent(newVespaClientHandler("com.yahoo.document.restapi.resource.RestApi", "/document/v1/*")); + cluster.addComponent(newVespaClientHandler("com.yahoo.document.restapi.resource.RestApi", "document/v1/*")); cluster.addComponent(newVespaClientHandler("com.yahoo.vespa.http.server.FeedHandler", ContainerCluster.RESERVED_URI_PREFIX + "/feedapi")); } @@ -34,18 +32,9 @@ public class ContainerDocumentApi { Handler<AbstractConfigProducer<?>> handler = new Handler<>(new ComponentModel( BundleInstantiationSpecification.getFromStrings(componentId, null, vespaClientBundleSpecification), "")); - if (options.bindings.isEmpty()) { - handler.addServerBindings( - SystemBindingPattern.fromHttpPath(bindingSuffix), - SystemBindingPattern.fromHttpPath(bindingSuffix + '/')); - } else { - for (String rootBinding : options.bindings) { - String pathWithoutLeadingSlash = bindingSuffix.substring(1); - handler.addServerBindings( - UserBindingPattern.fromPattern(rootBinding + pathWithoutLeadingSlash), - UserBindingPattern.fromPattern(rootBinding + pathWithoutLeadingSlash + '/')); - } - + for (String rootBinding : options.bindings) { + handler.addServerBindings(rootBinding + bindingSuffix, + rootBinding + bindingSuffix + '/'); } return handler; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index 1427fa492dc..b0ac02d0fe8 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -22,12 +22,10 @@ import com.yahoo.search.config.QrStartConfig; import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.vespa.config.search.core.RankingConstantsConfig; import com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainer; -import com.yahoo.vespa.model.container.component.BindingPattern; import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.container.component.ConfigProducerGroup; import com.yahoo.vespa.model.container.component.Handler; import com.yahoo.vespa.model.container.component.Servlet; -import com.yahoo.vespa.model.container.component.SystemBindingPattern; import com.yahoo.vespa.model.container.jersey.Jersey2Servlet; import com.yahoo.vespa.model.container.jersey.RestApi; import com.yahoo.vespa.model.container.xml.PlatformBundles; @@ -57,12 +55,12 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat MetricsProxyApiConfig.Producer { public static final String METRICS_V2_HANDLER_CLASS = MetricsV2Handler.class.getName(); - public static final BindingPattern METRICS_V2_HANDLER_BINDING_1 = SystemBindingPattern.fromHttpPath(MetricsV2Handler.V2_PATH); - public static final BindingPattern METRICS_V2_HANDLER_BINDING_2 = SystemBindingPattern.fromHttpPath(MetricsV2Handler.V2_PATH + "/*"); + public static final String METRICS_V2_HANDLER_BINDING_1 = "http://*" + MetricsV2Handler.V2_PATH; + public static final String METRICS_V2_HANDLER_BINDING_2 = METRICS_V2_HANDLER_BINDING_1 + "/*"; public static final String PROMETHEUS_V1_HANDLER_CLASS = PrometheusV1Handler.class.getName(); - private static final BindingPattern PROMETHEUS_V1_HANDLER_BINDING_1 = SystemBindingPattern.fromHttpPath(PrometheusV1Handler.V1_PATH); - private static final BindingPattern PROMETHEUS_V1_HANDLER_BINDING_2 = SystemBindingPattern.fromHttpPath(PrometheusV1Handler.V1_PATH + "/*"); + private static final String PROMETHEUS_V1_HANDLER_BINDING_1 = "http://*" + PrometheusV1Handler.V1_PATH; + private static final String PROMETHEUS_V1_HANDLER_BINDING_2 = PROMETHEUS_V1_HANDLER_BINDING_1 + "/*"; public static final int heapSizePercentageOfTotalNodeMemory = 60; public static final int heapSizePercentageOfTotalNodeMemoryWhenCombinedCluster = 17; @@ -127,7 +125,7 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat addMetricsHandler(PROMETHEUS_V1_HANDLER_CLASS, PROMETHEUS_V1_HANDLER_BINDING_1, PROMETHEUS_V1_HANDLER_BINDING_2); } - private void addMetricsHandler(String handlerClass, BindingPattern rootBinding, BindingPattern innerBinding) { + private void addMetricsHandler(String handlerClass, String rootBinding, String innerBinding) { Handler<AbstractConfigProducer<?>> handler = new Handler<>( new ComponentModel(handlerClass, null, null, null)); handler.addServerBindings(rootBinding, innerBinding); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index 8bb456ab7e7..240157fb7aa 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -39,7 +39,6 @@ import com.yahoo.vespa.model.Service; import com.yahoo.vespa.model.admin.monitoring.Monitoring; import com.yahoo.vespa.model.clients.ContainerDocumentApi; import com.yahoo.vespa.model.container.component.AccessLogComponent; -import com.yahoo.vespa.model.container.component.BindingPattern; import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.container.component.ComponentGroup; import com.yahoo.vespa.model.container.component.ComponentsConfigGenerator; @@ -48,7 +47,6 @@ import com.yahoo.vespa.model.container.component.FileStatusHandlerComponent; import com.yahoo.vespa.model.container.component.Handler; import com.yahoo.vespa.model.container.component.SimpleComponent; import com.yahoo.vespa.model.container.component.StatisticsComponent; -import com.yahoo.vespa.model.container.component.SystemBindingPattern; import com.yahoo.vespa.model.container.component.chain.ProcessingHandler; import com.yahoo.vespa.model.container.docproc.ContainerDocproc; import com.yahoo.vespa.model.container.docproc.DocprocChains; @@ -109,7 +107,7 @@ public abstract class ContainerCluster<CONTAINER extends Container> * normal compatibility concerns only applies to libraries using the URIs in * question, not contents served from the URIs themselves. */ - public static final String RESERVED_URI_PREFIX = "/reserved-for-internal-use"; + public static final String RESERVED_URI_PREFIX = "reserved-for-internal-use"; public static final String APPLICATION_STATUS_HANDLER_CLASS = "com.yahoo.container.handler.observability.ApplicationStatusHandler"; public static final String BINDINGS_OVERVIEW_HANDLER_CLASS = BindingsOverviewHandler.class.getName(); @@ -119,13 +117,13 @@ public abstract class ContainerCluster<CONTAINER extends Container> public static final String G1GC = "-XX:+UseG1GC -XX:MaxTenuringThreshold=15"; public static final String STATE_HANDLER_CLASS = "com.yahoo.container.jdisc.state.StateHandler"; - public static final BindingPattern STATE_HANDLER_BINDING_1 = SystemBindingPattern.fromHttpPath(StateHandler.STATE_API_ROOT); - public static final BindingPattern STATE_HANDLER_BINDING_2 = SystemBindingPattern.fromHttpPath(StateHandler.STATE_API_ROOT + "/*"); + public static final String STATE_HANDLER_BINDING_1 = "http://*" + StateHandler.STATE_API_ROOT; + public static final String STATE_HANDLER_BINDING_2 = STATE_HANDLER_BINDING_1 + "/*"; public static final String ROOT_HANDLER_PATH = "/"; - public static final BindingPattern ROOT_HANDLER_BINDING = SystemBindingPattern.fromHttpPath(ROOT_HANDLER_PATH); + public static final String ROOT_HANDLER_BINDING = "http://*" + ROOT_HANDLER_PATH; - public static final BindingPattern VIP_HANDLER_BINDING = SystemBindingPattern.fromHttpPath("/status.html"); + public static final String VIP_HANDLER_BINDING = "http://*/status.html"; private final String name; @@ -236,7 +234,7 @@ public abstract class ContainerCluster<CONTAINER extends Container> Handler<AbstractConfigProducer<?>> statusHandler = new Handler<>( new ComponentModel(BundleInstantiationSpecification.getInternalHandlerSpecificationFromStrings( APPLICATION_STATUS_HANDLER_CLASS, null), null)); - statusHandler.addServerBindings(SystemBindingPattern.fromHttpPath("/ApplicationStatus")); + statusHandler.addServerBindings("http://*/ApplicationStatus"); addComponent(statusHandler); } @@ -311,7 +309,7 @@ public abstract class ContainerCluster<CONTAINER extends Container> containers.forEach(this::addContainer); } - public void setProcessingChains(ProcessingChains processingChains, BindingPattern... serverBindings) { + public void setProcessingChains(ProcessingChains processingChains, String... serverBindings) { if (this.processingChains != null) throw new IllegalStateException("ProcessingChains should only be set once."); @@ -322,7 +320,7 @@ public abstract class ContainerCluster<CONTAINER extends Container> processingChains, "com.yahoo.processing.handler.ProcessingHandler"); - for (BindingPattern binding: serverBindings) + for (String binding: serverBindings) processingHandler.addServerBindings(binding); addComponent(processingHandler); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java index 72f1921e6a2..6b4f8d486ec 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java @@ -7,7 +7,6 @@ import com.yahoo.searchdefinition.derived.RankProfileList; import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.vespa.config.search.core.RankingConstantsConfig; import com.yahoo.vespa.model.container.component.Handler; -import com.yahoo.vespa.model.container.component.SystemBindingPattern; import java.util.List; import java.util.Objects; @@ -22,7 +21,7 @@ public class ContainerModelEvaluation implements RankProfilesConfig.Producer, Ra private final static String BUNDLE_NAME = "model-evaluation"; private final static String EVALUATOR_NAME = ModelsEvaluator.class.getName(); private final static String REST_HANDLER_NAME = "ai.vespa.models.handler.ModelsEvaluationHandler"; - private final static String REST_BINDING_PATH = "/model-evaluation/v1"; + private final static String REST_BINDING = "model-evaluation/v1"; /** Global rank profiles, aka models */ private final RankProfileList rankProfileList; @@ -49,9 +48,8 @@ public class ContainerModelEvaluation implements RankProfilesConfig.Producer, Ra public static Handler<?> getHandler() { Handler<?> handler = new Handler<>(new ComponentModel(REST_HANDLER_NAME, null, BUNDLE_NAME)); - handler.addServerBindings( - SystemBindingPattern.fromHttpPath(REST_BINDING_PATH), - SystemBindingPattern.fromHttpPath(REST_BINDING_PATH + "/*")); + handler.addServerBindings("http://*/" + REST_BINDING, + "http://*/" + REST_BINDING + "/*"); return handler; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/BindingPattern.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/BindingPattern.java deleted file mode 100644 index 1d5736ba7e2..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/BindingPattern.java +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.container.component; - -import java.util.Comparator; -import java.util.Objects; -import java.util.Optional; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * URI binding pattern used by filter and handler bindings. - * - * @author bjorncs - */ -public abstract class BindingPattern implements Comparable<BindingPattern> { - - private static final Pattern BINDING_PATTERN = - Pattern.compile("([^:]+)://([^:/]+)(:((\\*)|([0-9]+)))?(/.*)", Pattern.UNICODE_CASE | Pattern.CANON_EQ); - - public static final String WILDCARD_PATTERN = "*"; - - private final String scheme; - private final String host; - private final String port; - private final String path; - - protected BindingPattern( - String scheme, - String host, - String port, - String path) { - this.scheme = Objects.requireNonNull(scheme, "Scheme in binding must be specified"); - this.host = Objects.requireNonNull(host, "Host must be specified"); - this.port = port; - this.path = validatePath(path); - } - - protected BindingPattern(String binding) { - Matcher matcher = BINDING_PATTERN.matcher(binding); - if (!matcher.matches()) throw new IllegalArgumentException("Invalid binding: " + binding); - this.scheme = matcher.group(1); - this.host = matcher.group(2); - this.port = matcher.group(4); - this.path = matcher.group(7); - } - - private static String validatePath(String path) { - Objects.requireNonNull(path, "Path must be specified"); - if (!path.startsWith("/")) throw new IllegalArgumentException("Path must have '/' as prefix: " + path); - return path; - } - - public String scheme() { return scheme; } - public String host() { return host; } - public Optional<String> port() { return Optional.ofNullable(port); } - public String path() { return path; } - - public String patternString() { - StringBuilder builder = new StringBuilder(scheme).append("://").append(host); - if (port != null) { - builder.append(':').append(port); - } - return builder.append(path).toString(); - } - - /** Compares the underlying pattern string for equality */ - public boolean hasSamePattern(BindingPattern other) { return this.patternString().equals(other.patternString()); } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - BindingPattern that = (BindingPattern) o; - return Objects.equals(scheme, that.scheme) && - Objects.equals(host, that.host) && - Objects.equals(port, that.port) && - Objects.equals(path, that.path); - } - - @Override public int hashCode() { return Objects.hash(scheme, host, port, path); } - - @Override - public int compareTo(BindingPattern o) { - return Comparator.comparing(BindingPattern::scheme) - .thenComparing(BindingPattern::host) - .thenComparing(pattern -> pattern.port().orElse(null)) - .thenComparing(BindingPattern::path) - .compare(this, o); - } -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/DiscBindingsConfigGenerator.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/DiscBindingsConfigGenerator.java index 02face328d9..d7e393ee474 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/DiscBindingsConfigGenerator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/DiscBindingsConfigGenerator.java @@ -1,16 +1,13 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container.component; -import java.util.Collection; -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.Map; +import java.util.*; import static com.yahoo.container.jdisc.JdiscBindingsConfig.Handlers; -import static java.util.stream.Collectors.toList; /** * @author gjoranv + * @since 5.1.8 */ public class DiscBindingsConfigGenerator { @@ -29,11 +26,7 @@ public class DiscBindingsConfigGenerator { return Collections.singletonMap(handler.model.getComponentId().stringValue(), new Handlers.Builder() - .serverBindings(toStrings(handler.getServerBindings())) - .clientBindings(toStrings(handler.getClientBindings()))); - } - - private static Collection<String> toStrings(Collection<BindingPattern> bindings) { - return bindings.stream().map(BindingPattern::patternString).collect(toList()); + .serverBindings(handler.getServerBindings()) + .clientBindings(handler.getClientBindings())); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/FileStatusHandlerComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/FileStatusHandlerComponent.java index 839594502c6..3d9a1b2e665 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/FileStatusHandlerComponent.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/FileStatusHandlerComponent.java @@ -15,7 +15,7 @@ public class FileStatusHandlerComponent extends Handler implements VipStatusConf private final String fileName; - public FileStatusHandlerComponent(String id, String fileName, BindingPattern... bindings) { + public FileStatusHandlerComponent(String id, String fileName, String... bindings) { super(new ComponentModel(id, CLASS, null, null)); this.fileName = fileName; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java index efee5c6a9a0..82484e07773 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java @@ -1,8 +1,9 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container.component; -import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.osgi.provider.model.ComponentModel; +import com.yahoo.config.model.producer.AbstractConfigProducer; import java.util.ArrayList; import java.util.Arrays; @@ -22,8 +23,8 @@ import java.util.Set; */ public class Handler<CHILD extends AbstractConfigProducer<?>> extends Component<CHILD, ComponentModel> { - private final Set<BindingPattern> serverBindings = new LinkedHashSet<>(); - private final List<BindingPattern> clientBindings = new ArrayList<>(); + private Set<String> serverBindings = new LinkedHashSet<>(); + private List<String> clientBindings = new ArrayList<>(); public Handler(ComponentModel model) { super(model); @@ -33,23 +34,27 @@ public class Handler<CHILD extends AbstractConfigProducer<?>> extends Component< return new Handler<>(new ComponentModel(className, null, null, null)); } - public void addServerBindings(BindingPattern... bindings) { + public static Handler<AbstractConfigProducer<?>> getVespaHandlerFromClassName(String className) { + return new Handler<>(new ComponentModel(BundleInstantiationSpecification.getInternalHandlerSpecificationFromStrings(className, null), null)); + } + + public void addServerBindings(String... bindings) { serverBindings.addAll(Arrays.asList(bindings)); } - public void removeServerBinding(BindingPattern binding) { + public void removeServerBinding(String binding) { serverBindings.remove(binding); } - public void addClientBindings(BindingPattern... bindings) { + public void addClientBindings(String... bindings) { clientBindings.addAll(Arrays.asList(bindings)); } - public final Set<BindingPattern> getServerBindings() { + public final Set<String> getServerBindings() { return Collections.unmodifiableSet(serverBindings); } - public final List<BindingPattern> getClientBindings() { + public final List<String> getClientBindings() { return Collections.unmodifiableList(clientBindings); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/SystemBindingPattern.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/SystemBindingPattern.java deleted file mode 100644 index 3ae531539ef..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/SystemBindingPattern.java +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.container.component; - -/** - * A {@link BindingPattern} which is implicitly constructed by the model, e.g for built-in handlers and filter chains. - * - * @author bjorncs - */ -public class SystemBindingPattern extends BindingPattern { - - private SystemBindingPattern(String scheme, String host, String port, String path) { super(scheme, host, port, path); } - private SystemBindingPattern(String binding) { super(binding); } - - public static SystemBindingPattern fromHttpPath(String path) { return new SystemBindingPattern("http", "*", null, path);} - public static SystemBindingPattern fromPattern(String binding) { return new SystemBindingPattern(binding);} - public static SystemBindingPattern fromHttpPortAndPath(String port, String path) { return new SystemBindingPattern("http", "*", port, path); } - - @Override - public String toString() { - return "SystemBindingPattern{" + - "scheme='" + scheme() + '\'' + - ", host='" + host() + '\'' + - ", port='" + port().orElse(null) + '\'' + - ", path='" + path() + '\'' + - '}'; - } -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/UserBindingPattern.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/UserBindingPattern.java deleted file mode 100644 index 43f57fa0343..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/UserBindingPattern.java +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.container.component; - -/** - * A {@link BindingPattern} which is constructed directly from a user provided 'binding' element from services.xml. - * - * @author bjorncs - */ -public class UserBindingPattern extends BindingPattern { - - private UserBindingPattern(String scheme, String host, String port, String path) { super(scheme, host, port, path); } - private UserBindingPattern(String binding) { super(binding); } - - public static UserBindingPattern fromHttpPath(String path) { return new UserBindingPattern("http", "*", null, path); } - public static UserBindingPattern fromPattern(String binding) { return new UserBindingPattern(binding); } - - @Override - public String toString() { - return "UserBindingPattern{" + - "scheme='" + scheme() + '\'' + - ", host='" + host() + '\'' + - ", port='" + port().orElse(null) + '\'' + - ", path='" + path() + '\'' + - '}'; - } -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/ContainerDocproc.java b/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/ContainerDocproc.java index 82061a0425f..d4b4dcea78e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/ContainerDocproc.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/ContainerDocproc.java @@ -9,7 +9,6 @@ import com.yahoo.container.jdisc.config.SessionConfig; import com.yahoo.docproc.jdisc.messagebus.MbusRequestContext; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.component.ContainerSubsystem; -import com.yahoo.vespa.model.container.component.SystemBindingPattern; import java.util.HashMap; import java.util.Map; @@ -45,7 +44,7 @@ public class ContainerDocproc extends ContainerSubsystem<DocprocChains> private void addSource( final ContainerCluster cluster, final String name, final SessionConfig.Type.Enum type) { final MbusClient mbusClient = new MbusClient(name, type); - mbusClient.addClientBindings(SystemBindingPattern.fromPattern("mbus://*/" + mbusClient.getSessionName())); + mbusClient.addClientBindings("mbus://*/" + mbusClient.getSessionName()); cluster.addComponent(mbusClient); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/DocprocChains.java b/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/DocprocChains.java index 68dc2518c23..5d08a0a6998 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/DocprocChains.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/DocprocChains.java @@ -7,7 +7,6 @@ import com.yahoo.container.jdisc.config.SessionConfig; import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.component.Component; -import com.yahoo.vespa.model.container.component.SystemBindingPattern; import com.yahoo.vespa.model.container.component.chain.Chains; import com.yahoo.vespa.model.container.component.chain.ProcessingHandler; @@ -39,12 +38,12 @@ public class DocprocChains extends Chains<DocprocChain> { } private void addServerAndClientForChain(ApplicationContainerCluster cluster, DocprocChain docprocChain) { - docprocHandler.addServerBindings(SystemBindingPattern.fromPattern("mbus://*/" + docprocChain.getSessionName())); + docprocHandler.addServerBindings("mbus://*/" + docprocChain.getSessionName()); cluster.addMbusServer(ComponentId.fromString(docprocChain.getSessionName())); MbusClient client = new MbusClient(docprocChain.getSessionName(), SessionConfig.Type.INTERMEDIATE); - client.addClientBindings(SystemBindingPattern.fromPattern("mbus://*/" + client.getSessionName())); + client.addClientBindings("mbus://*/" + client.getSessionName()); addComponent(client); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java index e96951dc83a..9676b8b1e4a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java @@ -3,20 +3,21 @@ package com.yahoo.vespa.model.container.http; import com.yahoo.component.ComponentId; import com.yahoo.component.ComponentSpecification; +import com.yahoo.config.application.api.DeployLogger; import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.ContainerCluster; -import com.yahoo.vespa.model.container.component.BindingPattern; import com.yahoo.vespa.model.container.component.FileStatusHandlerComponent; import com.yahoo.vespa.model.container.component.Handler; -import com.yahoo.vespa.model.container.component.SystemBindingPattern; -import com.yahoo.vespa.model.container.component.chain.Chain; +import com.yahoo.vespa.model.container.component.Servlet; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; -import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; /** * Helper class for http access control. @@ -24,15 +25,11 @@ import java.util.Set; * @author gjoranv * @author bjorncs */ -public class AccessControl { +public final class AccessControl { public static final ComponentId ACCESS_CONTROL_CHAIN_ID = ComponentId.fromString("access-control-chain"); - public static final ComponentId ACCESS_CONTROL_EXCLUDED_CHAIN_ID = ComponentId.fromString("access-control-excluded-chain"); - private static final int HOSTED_CONTAINER_PORT = 4443; - - // Handlers that are excluded from access control - private static final List<String> EXCLUDED_HANDLERS = List.of( + public static final List<String> UNPROTECTED_HANDLERS = List.of( FileStatusHandlerComponent.CLASS, ContainerCluster.APPLICATION_STATUS_HANDLER_CLASS, ContainerCluster.BINDINGS_OVERVIEW_HANDLER_CLASS, @@ -42,15 +39,18 @@ public class AccessControl { ApplicationContainerCluster.PROMETHEUS_V1_HANDLER_CLASS ); - public static class Builder { - private final String domain; + public static final class Builder { + private String domain; private boolean readEnabled = false; private boolean writeEnabled = true; - private final Set<BindingPattern> excludeBindings = new LinkedHashSet<>(); + private final Set<String> excludeBindings = new LinkedHashSet<>(); private Collection<Handler<?>> handlers = Collections.emptyList(); + private Collection<Servlet> servlets = Collections.emptyList(); + private final DeployLogger logger; - public Builder(String domain) { + public Builder(String domain, DeployLogger logger) { this.domain = domain; + this.logger = logger; } public Builder readEnabled(boolean readEnabled) { @@ -58,111 +58,102 @@ public class AccessControl { return this; } - public Builder writeEnabled(boolean writeEnabled) { - this.writeEnabled = writeEnabled; + public Builder writeEnabled(boolean writeEnalbed) { + this.writeEnabled = writeEnalbed; return this; } - public Builder excludeBinding(BindingPattern binding) { + public Builder excludeBinding(String binding) { this.excludeBindings.add(binding); return this; } public Builder setHandlers(ApplicationContainerCluster cluster) { this.handlers = cluster.getHandlers(); + this.servlets = cluster.getAllServlets(); return this; } public AccessControl build() { - return new AccessControl(domain, writeEnabled, readEnabled, excludeBindings, handlers); + return new AccessControl(domain, writeEnabled, readEnabled, + excludeBindings, servlets, handlers, logger); } } public final String domain; public final boolean readEnabled; public final boolean writeEnabled; - private final Set<BindingPattern> excludedBindings; + private final Set<String> excludedBindings; private final Collection<Handler<?>> handlers; + private final Collection<Servlet> servlets; + private final DeployLogger logger; private AccessControl(String domain, boolean writeEnabled, boolean readEnabled, - Set<BindingPattern> excludedBindings, - Collection<Handler<?>> handlers) { + Set<String> excludedBindings, + Collection<Servlet> servlets, + Collection<Handler<?>> handlers, + DeployLogger logger) { this.domain = domain; this.readEnabled = readEnabled; this.writeEnabled = writeEnabled; this.excludedBindings = Collections.unmodifiableSet(excludedBindings); this.handlers = handlers; + this.servlets = servlets; + this.logger = logger; } - public void configureHttpFilterChains(Http http) { - http.setAccessControl(this); - addAccessControlFilterChain(http); - addAccessControlExcludedChain(http); - removeDuplicateBindingsFromAccessControlChain(http); + public List<Binding> getBindings() { + return Stream.concat(getHandlerBindings(), getServletBindings()) + .collect(Collectors.toCollection(ArrayList::new)); } public static boolean hasHandlerThatNeedsProtection(ApplicationContainerCluster cluster) { - return cluster.getHandlers().stream() - .anyMatch(handler -> ! isExcludedHandler(handler) && hasNonMbusBinding(handler)); + return cluster.getHandlers().stream().anyMatch(AccessControl::handlerNeedsProtection); } - private void addAccessControlFilterChain(Http http) { - http.getFilterChains().add(createChain(ACCESS_CONTROL_CHAIN_ID)); - http.getBindings().addAll(List.of(createAccessControlBinding("/"), createAccessControlBinding("/*"))); + private Stream<Binding> getHandlerBindings() { + return handlers.stream() + .filter(this::shouldHandlerBeProtected) + .flatMap(handler -> handler.getServerBindings().stream()) + .map(binding -> accessControlBinding(binding, logger)); } - private void addAccessControlExcludedChain(Http http) { - http.getFilterChains().add(createChain(ACCESS_CONTROL_EXCLUDED_CHAIN_ID)); - for (BindingPattern excludedBinding : excludedBindings) { - http.getBindings().add(createAccessControlExcludedBinding(excludedBinding)); - } - for (Handler<?> handler : handlers) { - if (isExcludedHandler(handler)) { - for (BindingPattern binding : handler.getServerBindings()) { - http.getBindings().add(createAccessControlExcludedBinding(binding)); - } - } - } + private Stream<Binding> getServletBindings() { + return servlets.stream() + .filter(this::shouldServletBeProtected) + .flatMap(AccessControl::servletBindings) + .map(binding -> accessControlBinding(binding, logger)); } - // Remove bindings from access control chain that have binding pattern as a different filter chain - private void removeDuplicateBindingsFromAccessControlChain(Http http) { - Set<FilterBinding> duplicateBindings = new HashSet<>(); - for (FilterBinding binding : http.getBindings()) { - if (binding.chainId().toId().equals(ACCESS_CONTROL_CHAIN_ID)) { - for (FilterBinding otherBinding : http.getBindings()) { - if (!binding.chainId().equals(otherBinding.chainId()) - && binding.binding().equals(otherBinding.binding())) { - duplicateBindings.add(binding); - } - } - } - } - duplicateBindings.forEach(http.getBindings()::remove); + private boolean shouldHandlerBeProtected(Handler<?> handler) { + return ! isBuiltinGetOnly(handler) + && handler.getServerBindings().stream().noneMatch(excludedBindings::contains); } - private static FilterBinding createAccessControlBinding(String path) { - return FilterBinding.create( - new ComponentSpecification(ACCESS_CONTROL_CHAIN_ID.stringValue()), - SystemBindingPattern.fromHttpPortAndPath(Integer.toString(HOSTED_CONTAINER_PORT), path)); + private static boolean isBuiltinGetOnly(Handler<?> handler) { + return UNPROTECTED_HANDLERS.contains(handler.getClassId().getName()); } - private static FilterBinding createAccessControlExcludedBinding(BindingPattern excludedBinding) { - BindingPattern rewrittenBinding = SystemBindingPattern.fromHttpPortAndPath( - Integer.toString(HOSTED_CONTAINER_PORT), excludedBinding.path()); // only keep path from excluded binding - return FilterBinding.create( - new ComponentSpecification(ACCESS_CONTROL_EXCLUDED_CHAIN_ID.stringValue()), - rewrittenBinding); + private boolean shouldServletBeProtected(Servlet servlet) { + return servletBindings(servlet).noneMatch(excludedBindings::contains); } - private static Chain<Filter> createChain(ComponentId id) { return new Chain<>(FilterChains.emptyChainSpec(id)); } + private static Binding accessControlBinding(String binding, DeployLogger logger) { + return Binding.create(new ComponentSpecification(ACCESS_CONTROL_CHAIN_ID.stringValue()), binding, logger); + } + + private static Stream<String> servletBindings(Servlet servlet) { + return Stream.of("http://*/").map(protocol -> protocol + servlet.bindingPath); + } - private static boolean isExcludedHandler(Handler<?> handler) { return EXCLUDED_HANDLERS.contains(handler.getClassId().getName()); } + private static boolean handlerNeedsProtection(Handler<?> handler) { + return ! isBuiltinGetOnly(handler) && hasNonMbusBinding(handler); + } private static boolean hasNonMbusBinding(Handler<?> handler) { - return handler.getServerBindings().stream().anyMatch(binding -> ! binding.scheme().equals("mbus")); + return handler.getServerBindings().stream().anyMatch(binding -> ! binding.startsWith("mbus")); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/Binding.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/Binding.java new file mode 100644 index 00000000000..28f4949f210 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/Binding.java @@ -0,0 +1,39 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.http; + +import com.yahoo.component.ComponentSpecification; +import com.yahoo.config.application.api.DeployLogger; + +import java.util.logging.Level; + +/** + * @author bjorncs + */ +public class Binding { + + private final ComponentSpecification filterId; + private final String binding; + + private Binding(ComponentSpecification filterId, String binding) { + this.filterId = filterId; + this.binding = binding; + } + + public static Binding create(ComponentSpecification filterId, String binding, DeployLogger logger) { + if (binding.startsWith("https://")) { + logger.log(Level.WARNING, String.format("For binding '%s' on '%s': 'https' bindings are deprecated, " + + "use 'http' instead to bind to both http and https traffic.", + binding, filterId)); + } + return new Binding(filterId, binding); + } + + public ComponentSpecification filterId() { + return filterId; + } + + public String binding() { + return binding; + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/FilterBinding.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/FilterBinding.java deleted file mode 100644 index 1ca54769683..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/FilterBinding.java +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.container.http; - -import com.yahoo.component.ComponentSpecification; -import com.yahoo.vespa.model.container.component.BindingPattern; - -import java.util.Objects; - -/** - * @author bjorncs - */ -public class FilterBinding { - - private final ComponentSpecification chainId; - private final BindingPattern binding; - - private FilterBinding(ComponentSpecification chainId, BindingPattern binding) { - this.chainId = chainId; - this.binding = binding; - } - - public static FilterBinding create(ComponentSpecification chainId, BindingPattern binding) { - return new FilterBinding(chainId, binding); - } - - public ComponentSpecification chainId() { - return chainId; - } - - public BindingPattern binding() { - return binding; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - FilterBinding that = (FilterBinding) o; - return Objects.equals(chainId, that.chainId) && - Objects.equals(binding, that.binding); - } - - @Override - public int hashCode() { - return Objects.hash(chainId, binding); - } -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/Http.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/Http.java index f58f5faa382..0fcf7b2d06c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/Http.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/Http.java @@ -21,7 +21,7 @@ import java.util.concurrent.CopyOnWriteArrayList; public class Http extends AbstractConfigProducer<AbstractConfigProducer<?>> implements ServerConfig.Producer { private final FilterChains filterChains; - private final List<FilterBinding> bindings = new CopyOnWriteArrayList<>(); + private final List<Binding> bindings = new CopyOnWriteArrayList<>(); private volatile JettyHttpServer httpServer; private volatile AccessControl accessControl; @@ -64,7 +64,7 @@ public class Http extends AbstractConfigProducer<AbstractConfigProducer<?>> impl setHttpServer(null); } - public List<FilterBinding> getBindings() { + public List<Binding> getBindings() { return bindings; } @@ -74,16 +74,16 @@ public class Http extends AbstractConfigProducer<AbstractConfigProducer<?>> impl @Override public void getConfig(ServerConfig.Builder builder) { - for (FilterBinding binding : bindings) { + for (Binding binding : bindings) { builder.filter(new ServerConfig.Filter.Builder() - .id(binding.chainId().stringValue()) - .binding(binding.binding().patternString())); + .id(binding.filterId().stringValue()) + .binding(binding.binding())); } } @Override public void validate() { - if (((Collection<FilterBinding>) bindings).isEmpty()) return; + if (((Collection<Binding>) bindings).isEmpty()) return; if (filterChains == null) throw new IllegalArgumentException("Null FilterChains are not allowed when there are filter bindings"); @@ -91,9 +91,9 @@ public class Http extends AbstractConfigProducer<AbstractConfigProducer<?>> impl ComponentRegistry<ChainedComponent<?>> filters = filterChains.componentsRegistry(); ComponentRegistry<Chain<Filter>> chains = filterChains.allChains(); - for (FilterBinding binding: bindings) { - if (filters.getComponent(binding.chainId()) == null && chains.getComponent(binding.chainId()) == null) - throw new RuntimeException("Can't find filter " + binding.chainId() + " for binding " + binding.binding()); + for (Binding binding: bindings) { + if (filters.getComponent(binding.filterId()) == null && chains.getComponent(binding.filterId()) == null) + throw new RuntimeException("Can't find filter " + binding.filterId() + " for binding " + binding.binding()); } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java index c86d8b206d5..bfde9b9add1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java @@ -13,9 +13,9 @@ import com.yahoo.vespa.model.builder.xml.dom.ModelElement; import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.Container; -import com.yahoo.vespa.model.container.component.UserBindingPattern; +import com.yahoo.vespa.model.container.component.chain.Chain; import com.yahoo.vespa.model.container.http.AccessControl; -import com.yahoo.vespa.model.container.http.FilterBinding; +import com.yahoo.vespa.model.container.http.Binding; import com.yahoo.vespa.model.container.http.FilterChains; import com.yahoo.vespa.model.container.http.Http; import org.w3c.dom.Element; @@ -25,6 +25,8 @@ import java.util.List; import java.util.Optional; import java.util.logging.Level; +import static com.yahoo.vespa.model.container.http.AccessControl.ACCESS_CONTROL_CHAIN_ID; + /** * @author Tony Vaagenes * @author gjoranv @@ -34,17 +36,19 @@ public class HttpBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Http> @Override protected Http doBuild(DeployState deployState, AbstractConfigProducer ancestor, Element spec) { FilterChains filterChains; - List<FilterBinding> bindings = new ArrayList<>(); + List<Binding> bindings = new ArrayList<>(); AccessControl accessControl = null; Element filteringElem = XML.getChild(spec, "filtering"); if (filteringElem != null) { filterChains = new FilterChainsBuilder().build(deployState, ancestor, filteringElem); - bindings = readFilterBindings(filteringElem); + bindings = readFilterBindings(filteringElem, deployState.getDeployLogger()); Element accessControlElem = XML.getChild(filteringElem, "access-control"); if (accessControlElem != null) { accessControl = buildAccessControl(deployState, ancestor, accessControlElem); + bindings.addAll(accessControl.getBindings()); + filterChains.add(new Chain<>(FilterChains.emptyChainSpec(ACCESS_CONTROL_CHAIN_ID))); } } else { filterChains = new FilterChainsBuilder().newChainsInstance(ancestor); @@ -52,16 +56,14 @@ public class HttpBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Http> Http http = new Http(filterChains); http.getBindings().addAll(bindings); + http.setAccessControl(accessControl); http.setHttpServer(new JettyHttpServerBuilder().build(deployState, ancestor, spec)); - if (accessControl != null) { - accessControl.configureHttpFilterChains(http); - } return http; } private AccessControl buildAccessControl(DeployState deployState, AbstractConfigProducer ancestor, Element accessControlElem) { AthenzDomain domain = getAccessControlDomain(deployState, accessControlElem); - AccessControl.Builder builder = new AccessControl.Builder(domain.value()); + AccessControl.Builder builder = new AccessControl.Builder(domain.value(), deployState.getDeployLogger()); getContainerCluster(ancestor).ifPresent(builder::setHandlers); @@ -73,7 +75,7 @@ public class HttpBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Http> Element excludeElem = XML.getChild(accessControlElem, "exclude"); if (excludeElem != null) { XML.getChildren(excludeElem, "binding").stream() - .map(xml -> UserBindingPattern.fromPattern(XML.getValue(xml))) + .map(XML::getValue) .forEach(builder::excludeBinding); } return builder.build(); @@ -111,8 +113,8 @@ public class HttpBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Http> return Optional.of((ApplicationContainerCluster) currentProducer); } - private List<FilterBinding> readFilterBindings(Element filteringSpec) { - List<FilterBinding> result = new ArrayList<>(); + private List<Binding> readFilterBindings(Element filteringSpec, DeployLogger logger) { + List<Binding> result = new ArrayList<>(); for (Element child: XML.getChildren(filteringSpec)) { String tagName = child.getTagName(); @@ -121,7 +123,7 @@ public class HttpBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Http> for (Element bindingSpec: XML.getChildren(child, "binding")) { String binding = XML.getValue(bindingSpec); - result.add(FilterBinding.create(chainId, UserBindingPattern.fromPattern(binding))); + result.add(Binding.create(chainId, binding, logger)); } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/processing/ProcessingChains.java b/config-model/src/main/java/com/yahoo/vespa/model/container/processing/ProcessingChains.java index f6b24bf9635..4fd79a4f335 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/processing/ProcessingChains.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/processing/ProcessingChains.java @@ -2,8 +2,6 @@ package com.yahoo.vespa.model.container.processing; import com.yahoo.config.model.producer.AbstractConfigProducer; -import com.yahoo.vespa.model.container.component.BindingPattern; -import com.yahoo.vespa.model.container.component.SystemBindingPattern; import com.yahoo.vespa.model.container.component.chain.Chains; /** @@ -13,7 +11,7 @@ import com.yahoo.vespa.model.container.component.chain.Chains; */ public class ProcessingChains extends Chains<ProcessingChain> { - public static final BindingPattern[] defaultBindings = new BindingPattern[]{SystemBindingPattern.fromHttpPath("/processing/*")}; + public static final String[] defaultBindings = new String[] {"http://*/processing/*"}; public ProcessingChains(AbstractConfigProducer parent, String subId) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/GUIHandler.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/GUIHandler.java index f01bbcd3951..1e717f89819 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/GUIHandler.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/GUIHandler.java @@ -14,7 +14,7 @@ public class GUIHandler extends Handler<AbstractConfigProducer<?>> { public static final String BUNDLE = "container-search-gui"; public static final String CLASS = "com.yahoo.search.query.gui.GUIHandler"; - public static final String BINDING_PATH = "/querybuilder/*"; + public static final String BINDING = "*/querybuilder/*"; public GUIHandler() { super(new ComponentModel(bundleSpec(CLASS, BUNDLE))); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 51583588201..41e092c7ea5 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -29,6 +29,7 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; import com.yahoo.search.rendering.RendererRegistry; import com.yahoo.searchdefinition.derived.RankProfileList; @@ -56,11 +57,10 @@ import com.yahoo.vespa.model.container.ContainerModel; import com.yahoo.vespa.model.container.ContainerModelEvaluation; import com.yahoo.vespa.model.container.IdentityProvider; import com.yahoo.vespa.model.container.SecretStore; -import com.yahoo.vespa.model.container.component.BindingPattern; +import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.container.component.FileStatusHandlerComponent; import com.yahoo.vespa.model.container.component.Handler; -import com.yahoo.vespa.model.container.component.SystemBindingPattern; -import com.yahoo.vespa.model.container.component.UserBindingPattern; +import com.yahoo.vespa.model.container.component.chain.Chain; import com.yahoo.vespa.model.container.component.chain.ProcessingHandler; import com.yahoo.vespa.model.container.docproc.ContainerDocproc; import com.yahoo.vespa.model.container.docproc.DocprocChains; @@ -93,6 +93,7 @@ import java.util.function.Consumer; import java.util.regex.Pattern; import java.util.stream.Collectors; +import static com.yahoo.vespa.model.container.http.AccessControl.ACCESS_CONTROL_CHAIN_ID; import static java.util.logging.Level.WARNING; /** @@ -112,7 +113,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { private static final String ENVIRONMENT_VARIABLES_ELEMENT = "environment-variables"; static final String SEARCH_HANDLER_CLASS = com.yahoo.search.handler.SearchHandler.class.getName(); - static final BindingPattern SEARCH_HANDLER_BINDING = SystemBindingPattern.fromHttpPath("/search/*"); + static final String SEARCH_HANDLER_BINDING = "http://*/search/*"; public enum Networking { disable, enable } @@ -277,10 +278,8 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { String name = "status.html"; Optional<String> statusFile = Optional.ofNullable(System.getenv(HOSTED_VESPA_STATUS_FILE_SETTING)); cluster.addComponent( - new FileStatusHandlerComponent( - name + "-status-handler", - statusFile.orElse(HOSTED_VESPA_STATUS_FILE), - SystemBindingPattern.fromHttpPath("/" + name))); + new FileStatusHandlerComponent(name + "-status-handler", statusFile.orElse(HOSTED_VESPA_STATUS_FILE), + "http://*/" + name)); } else { cluster.addVipHandler(); } @@ -369,12 +368,15 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { if (http.getAccessControl().isPresent()) return; // access control added explicitly AthenzDomain tenantDomain = deployState.getProperties().athenzDomain().orElse(null); if (tenantDomain == null) return; // tenant domain not present, cannot add access control. this should eventually be a failure. - new AccessControl.Builder(tenantDomain.value()) - .setHandlers(cluster) - .readEnabled(false) - .writeEnabled(false) - .build() - .configureHttpFilterChains(http); + AccessControl accessControl = + new AccessControl.Builder(tenantDomain.value(), deployState.getDeployLogger()) + .setHandlers(cluster) + .readEnabled(false) + .writeEnabled(false) + .build(); + http.getFilterChains().add(new Chain<>(FilterChains.emptyChainSpec(ACCESS_CONTROL_CHAIN_ID))); + http.setAccessControl(accessControl); + http.getBindings().addAll(accessControl.getBindings()); } private Http buildHttp(DeployState deployState, ApplicationContainerCluster cluster, Element httpElement) { @@ -793,8 +795,8 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { ProcessingHandler<SearchChains> searchHandler = new ProcessingHandler<>(cluster.getSearch().getChains(), "com.yahoo.search.handler.SearchHandler"); - BindingPattern[] defaultBindings = {SEARCH_HANDLER_BINDING}; - for (BindingPattern binding: serverBindings(searchElement, defaultBindings)) { + String[] defaultBindings = {SEARCH_HANDLER_BINDING}; + for (String binding: serverBindings(searchElement, defaultBindings)) { searchHandler.addServerBindings(binding); } @@ -803,12 +805,12 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { private void addGUIHandler(ApplicationContainerCluster cluster) { Handler<?> guiHandler = new GUIHandler(); - guiHandler.addServerBindings(SystemBindingPattern.fromHttpPath(GUIHandler.BINDING_PATH)); + guiHandler.addServerBindings("http://"+GUIHandler.BINDING); cluster.addComponent(guiHandler); } - private BindingPattern[] serverBindings(Element searchElement, BindingPattern... defaultBindings) { + private String[] serverBindings(Element searchElement, String... defaultBindings) { List<Element> bindings = XML.getChildren(searchElement, "binding"); if (bindings.isEmpty()) return defaultBindings; @@ -816,16 +818,16 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { return toBindingList(bindings); } - private BindingPattern[] toBindingList(List<Element> bindingElements) { - List<BindingPattern> result = new ArrayList<>(); + private String[] toBindingList(List<Element> bindingElements) { + List<String> result = new ArrayList<>(); for (Element element: bindingElements) { String text = element.getTextContent().trim(); if (!text.isEmpty()) - result.add(UserBindingPattern.fromPattern(text)); + result.add(text); } - return result.toArray(BindingPattern[]::new); + return result.toArray(new String[result.size()]); } private ContainerDocumentApi buildDocumentApi(ApplicationContainerCluster cluster, Element spec) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocumentApiOptionsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocumentApiOptionsBuilder.java index 61464799812..ae74dbdb4a7 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocumentApiOptionsBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocumentApiOptionsBuilder.java @@ -6,17 +6,19 @@ import com.yahoo.vespa.model.clients.ContainerDocumentApi; import org.w3c.dom.Element; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.List; import java.util.logging.Logger; /** * @author Einar M R Rosenvinge + * @since 5.1.11 */ public class DocumentApiOptionsBuilder { private static final Logger log = Logger.getLogger(DocumentApiOptionsBuilder.class.getName()); - + private static final String[] DEFAULT_BINDINGS = {"http://*/"}; public static ContainerDocumentApi.Options build(Element spec) { return new ContainerDocumentApi.Options(getBindings(spec)); @@ -25,7 +27,8 @@ public class DocumentApiOptionsBuilder { private static List<String> getBindings(Element spec) { Collection<Element> bindingElems = XML.getChildren(spec, "binding"); if (bindingElems.isEmpty()) - return List.of(); + return Arrays.asList(DEFAULT_BINDINGS); + List<String> bindings = new ArrayList<>(); for (Element e :bindingElems) { String binding = getBinding(e); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/UriBindingsValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/UriBindingsValidatorTest.java deleted file mode 100644 index cce88bc02f9..00000000000 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/UriBindingsValidatorTest.java +++ /dev/null @@ -1,104 +0,0 @@ -package com.yahoo.vespa.model.application.validation;// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -import com.yahoo.config.application.api.ApplicationPackage; -import com.yahoo.config.model.NullConfigModelRegistry; -import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.config.model.deploy.TestProperties; -import com.yahoo.config.model.test.MockApplicationPackage; -import com.yahoo.vespa.model.VespaModel; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.xml.sax.SAXException; - -import java.io.IOException; - -/** - * @author bjorncs - */ -public class UriBindingsValidatorTest { - - @Rule - public ExpectedException exceptionRule = ExpectedException.none(); - - @Test - public void fails_on_user_handler_binding_with_port() throws IOException, SAXException { - exceptionRule.expect(IllegalArgumentException.class); - exceptionRule.expectMessage("For binding 'http://*:4443/my-handler': binding with port is not allowed"); - runUriBindingValidator(true, createServicesXmlWithHandler("http://*:4443/my-handler")); - } - - @Test - public void fails_on_user_handler_binding_with_hostname() throws IOException, SAXException { - exceptionRule.expect(IllegalArgumentException.class); - exceptionRule.expectMessage("For binding 'http://myhostname/my-handler': only binding with wildcard ('*') for hostname is allowed"); - runUriBindingValidator(true, createServicesXmlWithHandler("http://myhostname/my-handler")); - } - - @Test - public void fails_on_user_handler_binding_with_non_http_scheme() throws IOException, SAXException { - exceptionRule.expect(IllegalArgumentException.class); - exceptionRule.expectMessage("For binding 'ftp://*/my-handler': only 'http' is allowed as scheme"); - runUriBindingValidator(true, createServicesXmlWithHandler("ftp://*/my-handler")); - } - - @Test - public void fails_on_invalid_filter_binding() throws IOException, SAXException { - exceptionRule.expect(IllegalArgumentException.class); - exceptionRule.expectMessage("For binding 'https://*:4443/my-request-filer-chain': binding with port is not allowed"); - runUriBindingValidator(true, createServicesXmlWithRequestFilterChain("https://*:4443/my-request-filer-chain")); - } - - @Test - public void allows_valid_user_binding() throws IOException, SAXException { - runUriBindingValidator(true, createServicesXmlWithHandler("http://*/my-handler")); - } - - @Test - public void only_restricts_user_bindings_on_hosted() throws IOException, SAXException { - runUriBindingValidator(false, createServicesXmlWithRequestFilterChain("https://*:4443/my-request-filer-chain")); - } - - private void runUriBindingValidator(boolean isHosted, String servicesXml) throws IOException, SAXException { - ApplicationPackage app = new MockApplicationPackage.Builder() - .withServices(servicesXml) - .build(); - DeployState deployState = new DeployState.Builder() - .applicationPackage(app) - .properties(new TestProperties().setHostedVespa(isHosted)) - .build(); - VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); - new UriBindingsValidator().validate(model, deployState); - } - - private static String createServicesXmlWithHandler(String handlerBinding) { - return String.join( - "\n", - "<services version='1.0'>", - " <container id='default' version='1.0'>", - " <handler id='custom.Handler'>", - " <binding>" + handlerBinding + "</binding>", - " </handler>", - " </container>", - "</services>"); - } - - private static String createServicesXmlWithRequestFilterChain(String filterBinding) { - return String.join( - "\n", - "<services version='1.0'>", - " <container version='1.0'>", - " <http>", - " <server port='8080' id='main' />", - " <filtering>", - " <request-chain id='myChain'>", - " <filter id='myFilter'/>", - " <binding>" + filterBinding + "</binding>", - " </request-chain>", - " </filtering>", - " </http>", - " </container>", - "</services>"); - } - -}
\ No newline at end of file diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/component/BindingPatternTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/component/BindingPatternTest.java deleted file mode 100644 index 91a2b65c0e0..00000000000 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/component/BindingPatternTest.java +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.container.component; - -import org.junit.Test; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; - -/** - * @author bjorncs - */ -public class BindingPatternTest { - - @Test - public void parses_valid_bindings_correctly() { - assertBindingParses("http://host:1234/path"); - assertBindingParses("http://host/path"); - assertBindingParses("http://host/"); - assertBindingParses("*://*:*/*"); - assertBindingParses("http://*/*"); - assertBindingParses("https://*/my/path"); - assertBindingParses("https://*/path/*"); - assertBindingParses("https://host:*/path/*"); - assertBindingParses("https://host:1234/*"); - } - - @Test - public void getters_returns_correct_components() { - { - BindingPattern pattern = SystemBindingPattern.fromPattern("http://host:1234/path/*"); - assertEquals("http", pattern.scheme()); - assertEquals("host", pattern.host()); - assertEquals("1234", pattern.port().get()); - assertEquals("/path/*", pattern.path()); - } - { - BindingPattern pattern = SystemBindingPattern.fromPattern("https://*/path/v1/"); - assertEquals("https", pattern.scheme()); - assertEquals("*", pattern.host()); - assertFalse(pattern.port().isPresent()); - assertEquals("/path/v1/", pattern.path()); - } - } - - private static void assertBindingParses(String binding) { - BindingPattern pattern = SystemBindingPattern.fromPattern(binding); - String stringRepresentation = pattern.patternString(); - assertEquals( - "Expected string representation of parsed binding to match original binding string", - binding, stringRepresentation); - } - -}
\ No newline at end of file diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/http/FilterBindingsTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/http/FilterBindingsTest.java index 5b0c13a4038..0f9de516a4b 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/http/FilterBindingsTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/http/FilterBindingsTest.java @@ -1,12 +1,10 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container.http; -import com.yahoo.config.model.builder.xml.test.DomBuilderTest; import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.builder.xml.test.DomBuilderTest; import com.yahoo.jdisc.http.ServerConfig; import com.yahoo.vespa.model.container.ContainerModel; -import com.yahoo.vespa.model.container.component.BindingPattern; -import com.yahoo.vespa.model.container.component.UserBindingPattern; import com.yahoo.vespa.model.container.component.chain.Chain; import com.yahoo.vespa.model.container.http.xml.HttpBuilder; import com.yahoo.vespa.model.container.xml.ContainerModelBuilder; @@ -23,7 +21,7 @@ import static org.junit.Assert.assertNotNull; */ public class FilterBindingsTest extends DomBuilderTest { - private static final BindingPattern MY_CHAIN_BINDING = UserBindingPattern.fromHttpPath("/my-chain-binding"); + private static final String MY_CHAIN_BINDING = "http://*/my-chain-binding"; private Http buildHttp(Element xml) { Http http = new HttpBuilder().build(root.getDeployState(), root, xml); @@ -44,14 +42,14 @@ public class FilterBindingsTest extends DomBuilderTest { "<http>", " <filtering>", " <request-chain id='my-request-chain'>", - " <binding>" + MY_CHAIN_BINDING.patternString() + "</binding>", + " <binding>" + MY_CHAIN_BINDING + "</binding>", " </request-chain>", " </filtering>", "</http>"); Http http = buildHttp(xml); - FilterBinding binding = first(http.getBindings()); - assertEquals("my-request-chain", binding.chainId().getName()); + Binding binding = first(http.getBindings()); + assertEquals("my-request-chain", binding.filterId().getName()); assertEquals(MY_CHAIN_BINDING, binding.binding()); Chain<Filter> myChain = http.getFilterChains().allChains().getComponent("my-request-chain"); @@ -64,14 +62,14 @@ public class FilterBindingsTest extends DomBuilderTest { "<http>", " <filtering>", " <response-chain id='my-response-chain'>", - " <binding>" + MY_CHAIN_BINDING.patternString() + "</binding>", + " <binding>" + MY_CHAIN_BINDING + "</binding>", " </response-chain>", " </filtering>", "</http>"); Http http = buildHttp(xml); - FilterBinding binding = first(http.getBindings()); - assertEquals("my-response-chain", binding.chainId().getName()); + Binding binding = first(http.getBindings()); + assertEquals("my-response-chain", binding.filterId().getName()); assertEquals(MY_CHAIN_BINDING, binding.binding()); Chain<Filter> myChain = http.getFilterChains().allChains().getComponent("my-response-chain"); @@ -85,7 +83,7 @@ public class FilterBindingsTest extends DomBuilderTest { " <http>", " <filtering>", " <request-chain id='my-request-chain'>", - " <binding>" + MY_CHAIN_BINDING.patternString() + "</binding>", + " <binding>" + MY_CHAIN_BINDING + "</binding>", " </request-chain>", " </filtering>", " <server id='server1' port='8000' />", @@ -98,13 +96,13 @@ public class FilterBindingsTest extends DomBuilderTest { final ServerConfig config = root.getConfig(ServerConfig.class, "container/http/jdisc-jetty/server1"); assertEquals(1, config.filter().size()); assertEquals("my-request-chain", config.filter(0).id()); - assertEquals(MY_CHAIN_BINDING.patternString(), config.filter(0).binding()); + assertEquals(MY_CHAIN_BINDING, config.filter(0).binding()); } { final ServerConfig config = root.getConfig(ServerConfig.class, "container/http/jdisc-jetty/server2"); assertEquals(1, config.filter().size()); assertEquals("my-request-chain", config.filter(0).id()); - assertEquals(MY_CHAIN_BINDING.patternString(), config.filter(0).binding()); + assertEquals(MY_CHAIN_BINDING, config.filter(0).binding()); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessControlTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessControlTest.java index 4c3a1084005..28e23ce3222 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessControlTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessControlTest.java @@ -1,182 +1,271 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container.xml; +import com.google.common.collect.ImmutableSet; +import com.yahoo.collections.CollectionUtil; import com.yahoo.component.ComponentId; import com.yahoo.config.model.builder.xml.test.DomBuilderTest; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.config.provision.AthenzDomain; +import com.yahoo.container.jdisc.state.StateHandler; import com.yahoo.vespa.model.container.ApplicationContainer; +import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.http.AccessControl; -import com.yahoo.vespa.model.container.http.FilterChains; import com.yahoo.vespa.model.container.http.Http; +import com.yahoo.vespa.model.container.http.Binding; +import com.yahoo.vespa.model.container.http.xml.HttpBuilder; +import com.yahoo.vespa.model.container.jersey.Jersey2Servlet; import org.junit.Test; +import org.w3c.dom.Element; -import java.util.ArrayList; -import java.util.List; +import java.util.Collection; +import java.util.HashSet; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; +import static com.yahoo.config.model.test.TestUtil.joinLines; import static com.yahoo.vespa.defaults.Defaults.getDefaults; -import static org.hamcrest.CoreMatchers.hasItem; import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.hasItems; -import static org.hamcrest.Matchers.not; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; /** * @author gjoranv - * @author bjorncs */ public class AccessControlTest extends ContainerModelBuilderTestBase { + private static final Set<String> REQUIRED_HANDLER_BINDINGS = ImmutableSet.of( + "/custom-handler/", + "/search/", + "/document/", + ContainerCluster.RESERVED_URI_PREFIX); + + private static final Set<String> FORBIDDEN_HANDLER_BINDINGS = ImmutableSet.of( + "/ApplicationStatus", + "/status.html", + "/statistics/", + StateHandler.STATE_API_ROOT, + ContainerCluster.ROOT_HANDLER_PATH); + @Test - public void access_control_filter_chains_are_set_up() { - Http http = createModelAndGetHttp( + public void access_control_filter_chain_is_set_up() { + Element clusterElem = DomBuilderTest.parse( " <http>", " <filtering>", - " <access-control domain='my-tenant-domain' />", + " <access-control domain='foo' />", " </filtering>", " </http>"); - FilterChains filterChains = http.getFilterChains(); - assertTrue(filterChains.hasChain(AccessControl.ACCESS_CONTROL_CHAIN_ID)); - assertTrue(filterChains.hasChain(AccessControl.ACCESS_CONTROL_EXCLUDED_CHAIN_ID)); + Http http = new HttpBuilder().build(root.getDeployState(), root, clusterElem); + root.freezeModelTopology(); + + assertTrue(http.getFilterChains().hasChain(AccessControl.ACCESS_CONTROL_CHAIN_ID)); } @Test public void properties_are_set_from_xml() { - Http http = createModelAndGetHttp( + Element clusterElem = DomBuilderTest.parse( " <http>", " <filtering>", - " <access-control domain='my-tenant-domain'/>", + " <access-control domain='my-domain'/>", " </filtering>", " </http>"); + Http http = new HttpBuilder().build(root.getDeployState(), root, clusterElem); + root.freezeModelTopology(); AccessControl accessControl = http.getAccessControl().get(); - assertEquals("Wrong domain.", "my-tenant-domain", accessControl.domain); + assertEquals("Wrong domain.", "my-domain", accessControl.domain); } @Test public void read_is_disabled_and_write_is_enabled_by_default() { - Http http = createModelAndGetHttp( + Element clusterElem = DomBuilderTest.parse( " <http>", " <filtering>", - " <access-control domain='my-tenant-domain'/>", + " <access-control domain='foo' />", " </filtering>", " </http>"); + Http http = new HttpBuilder().build(root.getDeployState(), root, clusterElem); + root.freezeModelTopology(); + assertFalse("Wrong default value for read.", http.getAccessControl().get().readEnabled); assertTrue("Wrong default value for write.", http.getAccessControl().get().writeEnabled); } @Test public void read_and_write_can_be_overridden() { - Http http = createModelAndGetHttp( + Element clusterElem = DomBuilderTest.parse( " <http>", " <filtering>", - " <access-control domain='my-tenant-domain' read='true' write='false'/>", + " <access-control domain='foo' read='true' write='false'/>", " </filtering>", " </http>"); + Http http = new HttpBuilder().build(root.getDeployState(), root, clusterElem); + root.freezeModelTopology(); + assertTrue("Given read value not honoured.", http.getAccessControl().get().readEnabled); assertFalse("Given write value not honoured.", http.getAccessControl().get().writeEnabled); } @Test - public void access_control_excluded_filter_chain_has_all_bindings_from_excluded_handlers() { - Http http = createModelAndGetHttp( + public void access_control_filter_chain_has_correct_handler_bindings() { + Element clusterElem = DomBuilderTest.parse( + "<container version='1.0'>", + " <search/>", + " <document-api/>", + " <handler id='custom.Handler'>", + " <binding>http://*/custom-handler/*</binding>", + " </handler>", " <http>", " <filtering>", - " <access-control/>", + " <access-control domain='foo' />", " </filtering>", - " </http>"); + " </http>", + "</container>"); + + Http http = getHttp(clusterElem); + + Set<String> foundRequiredBindings = REQUIRED_HANDLER_BINDINGS.stream() + .filter(requiredBinding -> containsBinding(http.getBindings(), requiredBinding)) + .collect(Collectors.toSet()); + Set<String> missingRequiredBindings = new HashSet<>(REQUIRED_HANDLER_BINDINGS); + missingRequiredBindings.removeAll(foundRequiredBindings); + assertTrue("Access control chain was not bound to: " + CollectionUtil.mkString(missingRequiredBindings, ", "), + missingRequiredBindings.isEmpty()); - Set<String> actualBindings = getFilterBindings(http, AccessControl.ACCESS_CONTROL_EXCLUDED_CHAIN_ID); - assertThat(actualBindings, containsInAnyOrder( - "http://*:4443/ApplicationStatus", - "http://*:4443/status.html", - "http://*:4443/state/v1", - "http://*:4443/state/v1/*", - "http://*:4443/prometheus/v1", - "http://*:4443/prometheus/v1/*", - "http://*:4443/metrics/v2", - "http://*:4443/metrics/v2/*", - "http://*:4443/")); + FORBIDDEN_HANDLER_BINDINGS.forEach(forbiddenPath -> { + String forbiddenBinding = String.format("http://*%s", forbiddenPath); + http.getBindings().forEach( + binding -> assertNotEquals("Access control chain was bound to: " + binding.binding(), binding.binding(), forbiddenBinding)); + }); } @Test - public void access_control_excluded_chain_does_not_contain_any_bindings_from_access_control_chain() { - Http http = createModelAndGetHttp( - " <http>", - " <filtering>", - " <access-control/>", - " </filtering>", - " </http>"); + public void handler_can_be_excluded_by_excluding_one_of_its_bindings() { + final String notExcludedBinding = "http://*/custom-handler/*"; + final String excludedBinding = "http://*/excluded/*"; + Element clusterElem = DomBuilderTest.parse( + "<container version='1.0'>", + httpWithExcludedBinding(excludedBinding), + " <handler id='custom.Handler'>", + " <binding>" + notExcludedBinding + "</binding>", + " <binding>" + excludedBinding + "</binding>", + " </handler>", + "</container>"); - Set<String> bindings = getFilterBindings(http, AccessControl.ACCESS_CONTROL_CHAIN_ID); - Set<String> excludedBindings = getFilterBindings(http, AccessControl.ACCESS_CONTROL_EXCLUDED_CHAIN_ID); + Http http = getHttp(clusterElem); + assertFalse("Excluded binding was not removed.", + containsBinding(http.getBindings(), excludedBinding)); + assertFalse("Not all bindings of an excluded handler were removed.", + containsBinding(http.getBindings(), notExcludedBinding)); - for (String binding : bindings) { - assertThat(excludedBindings, not(hasItem(binding))); - } } - @Test - public void access_control_excluded_filter_chain_has_user_provided_excluded_bindings() { - Http http = createModelAndGetHttp( + public void access_control_filter_chain_has_all_servlet_bindings() { + final String servletPath = "servlet/path"; + final String restApiPath = "api/v0"; + final Set<String> requiredBindings = ImmutableSet.of(servletPath, restApiPath); + Element clusterElem = DomBuilderTest.parse( + "<container version='1.0'>", + " <servlet id='foo' class='bar' bundle='baz'>", + " <path>" + servletPath + "</path>", + " </servlet>", + " <rest-api jersey2='true' path='" + restApiPath + "' />", " <http>", - " <handler id='custom.Handler'>", - " <binding>http://*/custom-handler/*</binding>", - " </handler>", " <filtering>", - " <access-control>", - " <exclude>", - " <binding>http://*/custom-handler/*</binding>", - " <binding>http://*/search/*</binding>", - " </exclude>", - " </access-control>", + " <access-control domain='foo' />", " </filtering>", - " </http>"); + " </http>", + "</container>"); + + Http http = getHttp(clusterElem); - Set<String> actualBindings = getFilterBindings(http, AccessControl.ACCESS_CONTROL_EXCLUDED_CHAIN_ID); - assertThat(actualBindings, hasItems("http://*:4443/custom-handler/*", "http://*:4443/search/*", "http://*:4443/status.html")); + Set<String> missingRequiredBindings = requiredBindings.stream() + .filter(requiredBinding -> ! containsBinding(http.getBindings(), requiredBinding)) + .collect(Collectors.toSet()); + + assertTrue("Access control chain was not bound to: " + CollectionUtil.mkString(missingRequiredBindings, ", "), + missingRequiredBindings.isEmpty()); } @Test - public void access_control_filter_chain_contains_catchall_bindings() { - Http http = createModelAndGetHttp( - " <http>", - " <filtering>", - " <access-control/>", - " </filtering>", - " </http>"); - Set<String> actualBindings = getFilterBindings(http, AccessControl.ACCESS_CONTROL_CHAIN_ID); - assertThat(actualBindings, containsInAnyOrder("http://*:4443/*")); + public void servlet_can_be_excluded_by_excluding_one_of_its_bindings() { + final String servletPath = "servlet/path"; + final String notExcludedBinding = "http://*:8081/" + servletPath; + final String excludedBinding = "http://*:8080/" + servletPath; + Element clusterElem = DomBuilderTest.parse( + "<container version='1.0'>", + httpWithExcludedBinding(excludedBinding), + " <servlet id='foo' class='bar' bundle='baz'>", + " <path>" + servletPath + "</path>", + " </servlet>", + "</container>"); + + Http http = getHttp(clusterElem); + assertFalse("Excluded binding was not removed.", + containsBinding(http.getBindings(), excludedBinding)); + assertFalse("Not all bindings of an excluded servlet were removed.", + containsBinding(http.getBindings(), notExcludedBinding)); + } @Test + public void rest_api_can_be_excluded_by_excluding_one_of_its_bindings() { + final String restApiPath = "api/v0"; + final String notExcludedBinding = "http://*:8081/" + restApiPath + Jersey2Servlet.BINDING_SUFFIX;; + final String excludedBinding = "http://*:8080/" + restApiPath + Jersey2Servlet.BINDING_SUFFIX;; + Element clusterElem = DomBuilderTest.parse( + "<container version='1.0'>", + httpWithExcludedBinding(excludedBinding), + " <rest-api jersey2='true' path='" + restApiPath + "' />", + "</container>"); + + Http http = getHttp(clusterElem); + assertFalse("Excluded binding was not removed.", + containsBinding(http.getBindings(), excludedBinding)); + assertFalse("Not all bindings of an excluded rest-api were removed.", + containsBinding(http.getBindings(), notExcludedBinding)); + + } + + + @Test public void access_control_is_implicitly_added_for_hosted_apps() { - Http http = createModelAndGetHttp("<container version='1.0'/>"); - Optional<AccessControl> maybeAccessControl = http.getAccessControl(); + Element clusterElem = DomBuilderTest.parse( + "<container version='1.0'>", + nodesXml, + "</container>" ); + AthenzDomain tenantDomain = AthenzDomain.from("my-tenant-domain"); + DeployState state = new DeployState.Builder().properties( + new TestProperties() + .setAthenzDomain(tenantDomain) + .setHostedVespa(true)) + .build(); + createModel(root, state, null, clusterElem); + Optional<AccessControl> maybeAccessControl = + ((ApplicationContainer) root.getProducer("container/container.0")).getHttp().getAccessControl(); assertThat(maybeAccessControl.isPresent(), is(true)); AccessControl accessControl = maybeAccessControl.get(); assertThat(accessControl.writeEnabled, is(false)); assertThat(accessControl.readEnabled, is(false)); - assertThat(accessControl.domain, equalTo("my-tenant-domain")); + assertThat(accessControl.domain, equalTo(tenantDomain.value())); } @Test public void access_control_is_implicitly_added_for_hosted_apps_with_existing_http_element() { - Http http = createModelAndGetHttp( + Element clusterElem = DomBuilderTest.parse( + "<container version='1.0'>", " <http>", " <server port='" + getDefaults().vespaWebServicePort() + "' id='main' />", " <filtering>", @@ -185,33 +274,49 @@ public class AccessControlTest extends ContainerModelBuilderTestBase { " <filter id='inner' />", " </request-chain>", " </filtering>", - " </http>"); - assertThat(http.getAccessControl().isPresent(), is(true)); - assertThat(http.getFilterChains().hasChain(AccessControl.ACCESS_CONTROL_CHAIN_ID), is(true)); - assertThat(http.getFilterChains().hasChain(ComponentId.fromString("myChain")), is(true)); - } - - private Http createModelAndGetHttp(String... httpElement) { - List<String> servicesXml = new ArrayList<>(); - servicesXml.add("<container version='1.0'>"); - servicesXml.addAll(List.of(httpElement)); - servicesXml.add("</container>"); - + " </http>", + nodesXml, + "</container>" ); AthenzDomain tenantDomain = AthenzDomain.from("my-tenant-domain"); DeployState state = new DeployState.Builder().properties( new TestProperties() .setAthenzDomain(tenantDomain) .setHostedVespa(true)) .build(); - createModel(root, state, null, DomBuilderTest.parse(servicesXml.toArray(String[]::new))); - return ((ApplicationContainer) root.getProducer("container/container.0")).getHttp(); + createModel(root, state, null, clusterElem); + Http http = ((ApplicationContainer) root.getProducer("container/container.0")).getHttp(); + assertThat(http.getAccessControl().isPresent(), is(true)); + assertThat(http.getFilterChains().hasChain(AccessControl.ACCESS_CONTROL_CHAIN_ID), is(true)); + assertThat(http.getFilterChains().hasChain(ComponentId.fromString("myChain")), is(true)); } - private static Set<String> getFilterBindings(Http http, ComponentId filerChain) { - return http.getBindings().stream() - .filter(binding -> binding.chainId().toId().equals(filerChain)) - .map(binding -> binding.binding().patternString()) - .collect(Collectors.toSet()); + + private String httpWithExcludedBinding(String excludedBinding) { + return joinLines( + " <http>", + " <filtering>", + " <access-control domain='foo'>", + " <exclude>", + " <binding>" + excludedBinding + "</binding>", + " </exclude>", + " </access-control>", + " </filtering>", + " </http>"); } + private Http getHttp(Element clusterElem) { + createModel(root, clusterElem); + ContainerCluster cluster = (ContainerCluster) root.getChildren().get("container"); + Http http = cluster.getHttp(); + assertNotNull(http); + return http; + } + + private boolean containsBinding(Collection<Binding> bindings, String binding) { + for (Binding b : bindings) { + if (b.binding().contains(binding)) + return true; + } + return false; + } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerDocumentApiBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerDocumentApiBuilderTest.java index 73a68429b6d..ac2e1b88c0b 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerDocumentApiBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerDocumentApiBuilderTest.java @@ -4,8 +4,6 @@ package com.yahoo.vespa.model.container.xml; import com.yahoo.config.model.builder.xml.test.DomBuilderTest; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.component.Handler; -import com.yahoo.vespa.model.container.component.SystemBindingPattern; -import com.yahoo.vespa.model.container.component.UserBindingPattern; import org.junit.Test; import org.w3c.dom.Element; @@ -42,21 +40,24 @@ public class ContainerDocumentApiBuilderTest extends ContainerModelBuilderTestBa "<container id='cluster1' version='1.0'>", " <document-api>", " <binding>http://*/document-api/</binding>", + " <binding>missing-trailing-slash</binding>", " </document-api>", nodesXml, "</container>"); createModel(root, elem); - verifyCustomBindings("com.yahoo.vespa.http.server.FeedHandler"); + verifyCustomBindings("com.yahoo.vespa.http.server.FeedHandler", ContainerCluster.RESERVED_URI_PREFIX + "/feedapi"); } - private void verifyCustomBindings(String id) { + private void verifyCustomBindings(String id, String bindingSuffix) { Handler<?> handler = getHandlers("cluster1").get(id); - assertThat(handler.getServerBindings(), hasItem(UserBindingPattern.fromHttpPath("/document-api/reserved-for-internal-use/feedapi"))); - assertThat(handler.getServerBindings(), hasItem(UserBindingPattern.fromHttpPath("/document-api/reserved-for-internal-use/feedapi/"))); + assertThat(handler.getServerBindings(), hasItem("http://*/document-api/" + bindingSuffix)); + assertThat(handler.getServerBindings(), hasItem("http://*/document-api/" + bindingSuffix + "/")); + assertThat(handler.getServerBindings(), hasItem("missing-trailing-slash/" + bindingSuffix)); + assertThat(handler.getServerBindings(), hasItem("missing-trailing-slash/" + bindingSuffix + "/")); - assertThat(handler.getServerBindings().size(), is(2)); + assertThat(handler.getServerBindings().size(), is(4)); } @Test @@ -75,12 +76,8 @@ public class ContainerDocumentApiBuilderTest extends ContainerModelBuilderTestBa assertThat(handlerMap.get("com.yahoo.container.jdisc.state.StateHandler"), not(nullValue())); assertThat(handlerMap.get("com.yahoo.vespa.http.server.FeedHandler"), not(nullValue())); - assertThat(handlerMap.get("com.yahoo.vespa.http.server.FeedHandler").getServerBindings() - .contains(SystemBindingPattern.fromHttpPath("/reserved-for-internal-use/feedapi")), - is(true)); - assertThat(handlerMap.get("com.yahoo.vespa.http.server.FeedHandler").getServerBindings() - .contains(SystemBindingPattern.fromHttpPath("/reserved-for-internal-use/feedapi")), - is(true)); + assertThat(handlerMap.get("com.yahoo.vespa.http.server.FeedHandler").getServerBindings().contains("http://*/" + ContainerCluster.RESERVED_URI_PREFIX + "/feedapi"), is(true)); + assertThat(handlerMap.get("com.yahoo.vespa.http.server.FeedHandler").getServerBindings().contains("http://*/" + ContainerCluster.RESERVED_URI_PREFIX + "/feedapi/"), is(true)); assertThat(handlerMap.get("com.yahoo.vespa.http.server.FeedHandler").getServerBindings().size(), equalTo(2)); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java index 6114449c948..fdd7ae57f0f 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java @@ -241,7 +241,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { Element clusterElem = DomBuilderTest.parse( "<container id='default' version='1.0'>" + " <handler id='userRootHandler'>" + - " <binding>" + ROOT_HANDLER_BINDING.patternString() + "</binding>" + + " <binding>" + ROOT_HANDLER_BINDING + "</binding>" + " </handler>" + "</container>"); createModel(root, clusterElem); @@ -260,7 +260,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { Element clusterElem = DomBuilderTest.parse( "<container id='default' version='1.0'>" + " <handler id='userHandler'>" + - " <binding>" + STATE_HANDLER_BINDING_1.patternString() + "</binding>" + + " <binding>" + STATE_HANDLER_BINDING_1 + "</binding>" + " </handler>" + "</container>"); try { @@ -277,9 +277,9 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { createClusterWithJDiscHandler(); String discBindingsConfig = root.getConfig(JdiscBindingsConfig.class, "default").toString(); assertThat(discBindingsConfig, containsString("{discHandler}")); - assertThat(discBindingsConfig, containsString(".serverBindings[0] \"http://*/binding0\"")); - assertThat(discBindingsConfig, containsString(".serverBindings[1] \"http://*/binding1\"")); - assertThat(discBindingsConfig, containsString(".clientBindings[0] \"http://*/clientBinding\"")); + assertThat(discBindingsConfig, containsString(".serverBindings[0] \"binding0\"")); + assertThat(discBindingsConfig, containsString(".serverBindings[1] \"binding1\"")); + assertThat(discBindingsConfig, containsString(".clientBindings[0] \"clientBinding\"")); } @Test @@ -292,9 +292,9 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { Element clusterElem = DomBuilderTest.parse( "<container id='default' version='1.0'>", " <handler id='discHandler'>", - " <binding>http://*/binding0</binding>", - " <binding>http://*/binding1</binding>", - " <clientBinding>http://*/clientBinding</clientBinding>", + " <binding>binding0</binding>", + " <binding>binding1</binding>", + " <clientBinding>clientBinding</clientBinding>", " </handler>", "</container>"); @@ -340,16 +340,16 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { Element clusterElem = DomBuilderTest.parse( "<container id='default' version='1.0'>", " <processing>", - " <binding>http://*/binding0</binding>", - " <binding>http://*/binding1</binding>", + " <binding>binding0</binding>", + " <binding>binding1</binding>", " </processing>", "</container>"); createModel(root, clusterElem); String discBindingsConfig = root.getConfig(JdiscBindingsConfig.class, "default").toString(); - assertThat(discBindingsConfig, containsString(".serverBindings[0] \"http://*/binding0\"")); - assertThat(discBindingsConfig, containsString(".serverBindings[1] \"http://*/binding1\"")); + assertThat(discBindingsConfig, containsString(".serverBindings[0] \"binding0\"")); + assertThat(discBindingsConfig, containsString(".serverBindings[1] \"binding1\"")); assertThat(discBindingsConfig, not(containsString("/processing/*"))); } @@ -358,9 +358,9 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { createModelWithClientProvider(); String discBindingsConfig = root.getConfig(JdiscBindingsConfig.class, "default").toString(); assertThat(discBindingsConfig, containsString("{discClient}")); - assertThat(discBindingsConfig, containsString(".clientBindings[0] \"http://*/binding0\"")); - assertThat(discBindingsConfig, containsString(".clientBindings[1] \"http://*/binding1\"")); - assertThat(discBindingsConfig, containsString(".serverBindings[0] \"http://*/serverBinding\"")); + assertThat(discBindingsConfig, containsString(".clientBindings[0] \"binding0\"")); + assertThat(discBindingsConfig, containsString(".clientBindings[1] \"binding1\"")); + assertThat(discBindingsConfig, containsString(".serverBindings[0] \"serverBinding\"")); } @Test @@ -373,9 +373,9 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { Element clusterElem = DomBuilderTest.parse( "<container id='default' version='1.0'>" + " <client id='discClient'>" + - " <binding>http://*/binding0</binding>" + - " <binding>http://*/binding1</binding>" + - " <serverBinding>http://*/serverBinding</serverBinding>" + + " <binding>binding0</binding>" + + " <binding>binding1</binding>" + + " <serverBinding>serverBinding</serverBinding>" + " </client>" + "</container>" ); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/SearchBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/SearchBuilderTest.java index c8564c5a273..b2f9c805be1 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/SearchBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/SearchBuilderTest.java @@ -20,8 +20,6 @@ import static com.yahoo.test.Matchers.hasItemWithMethod; import static com.yahoo.vespa.model.container.search.ContainerSearch.QUERY_PROFILE_REGISTRY_CLASS; import static com.yahoo.vespa.model.container.xml.ContainerModelBuilder.SEARCH_HANDLER_BINDING; import static com.yahoo.vespa.model.container.xml.ContainerModelBuilder.SEARCH_HANDLER_CLASS; -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.not; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -50,7 +48,7 @@ public class SearchBuilderTest extends ContainerModelBuilderTestBase { createModel(root, clusterElem); String discBindingsConfig = root.getConfig(JdiscBindingsConfig.class, "default").toString(); - assertThat(discBindingsConfig, containsString(GUIHandler.BINDING_PATH)); + assertTrue(discBindingsConfig.contains(GUIHandler.BINDING)); ApplicationContainerCluster cluster = (ApplicationContainerCluster)root.getChildren().get("default"); @@ -68,8 +66,8 @@ public class SearchBuilderTest extends ContainerModelBuilderTestBase { Element clusterElem = DomBuilderTest.parse( "<container id='default' version='1.0'>", " <search>", - " <binding>http://*/binding0</binding>", - " <binding>http://*/binding1</binding>", + " <binding>binding0</binding>", + " <binding>binding1</binding>", " </search>", nodesXml, "</container>"); @@ -77,9 +75,9 @@ public class SearchBuilderTest extends ContainerModelBuilderTestBase { createModel(root, clusterElem); String discBindingsConfig = root.getConfig(JdiscBindingsConfig.class, "default").toString(); - assertThat(discBindingsConfig, containsString(".serverBindings[0] \"http://*/binding0\"")); - assertThat(discBindingsConfig, containsString(".serverBindings[1] \"http://*/binding1\"")); - assertThat(discBindingsConfig, not(containsString("/search/*"))); + assertTrue(discBindingsConfig.contains(".serverBindings[0] \"binding0\"")); + assertTrue(discBindingsConfig.contains(".serverBindings[1] \"binding1\"")); + assertFalse(discBindingsConfig.contains("/search/*")); } @Test @@ -105,7 +103,7 @@ public class SearchBuilderTest extends ContainerModelBuilderTestBase { "<container id='default' version='1.0'>", " <search />", " <handler id='" + myHandler + "'>", - " <binding>" + SEARCH_HANDLER_BINDING.patternString() + "</binding>", + " <binding>" + SEARCH_HANDLER_BINDING + "</binding>", " </handler>", nodesXml, "</container>"); @@ -113,7 +111,7 @@ public class SearchBuilderTest extends ContainerModelBuilderTestBase { createModel(root, clusterElem); var discBindingsConfig = root.getConfig(JdiscBindingsConfig.class, "default"); - assertEquals(SEARCH_HANDLER_BINDING.patternString(), discBindingsConfig.handlers(myHandler).serverBindings(0)); + assertEquals(SEARCH_HANDLER_BINDING, discBindingsConfig.handlers(myHandler).serverBindings(0)); assertNull(discBindingsConfig.handlers(SEARCH_HANDLER_CLASS)); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index eb268546580..ebeb7e7e377 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -63,6 +63,7 @@ import com.yahoo.vespa.config.server.tenant.Tenant; import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.Lock; +import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.orchestrator.Orchestrator; import java.io.File; @@ -90,6 +91,7 @@ import java.util.stream.Collectors; import static com.yahoo.config.model.api.container.ContainerServiceType.CLUSTERCONTROLLER_CONTAINER; import static com.yahoo.config.model.api.container.ContainerServiceType.CONTAINER; import static com.yahoo.config.model.api.container.ContainerServiceType.LOGSERVER_CONTAINER; +import static com.yahoo.vespa.config.server.filedistribution.FileDistributionUtil.fileReferenceExistsOnDisk; import static com.yahoo.vespa.curator.Curator.CompletionWaiter; import static com.yahoo.vespa.config.server.filedistribution.FileDistributionUtil.getFileReferencesOnDisk; import static com.yahoo.vespa.config.server.tenant.TenantRepository.HOSTED_VESPA_TENANT; @@ -252,7 +254,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye boolean ignoreSessionStaleFailure, Instant now) { ApplicationId applicationId = prepareParams.getApplicationId(); long sessionId = createSession(applicationId, prepareParams.getTimeoutBudget(), applicationPackage); - Tenant tenant = tenantRepository.getTenant(applicationId.tenant()); + Tenant tenant = getTenant(applicationId); PrepareResult result = prepare(tenant, sessionId, prepareParams, now); activate(tenant, sessionId, prepareParams.getTimeoutBudget(), ignoreSessionStaleFailure); return result; @@ -379,10 +381,6 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye } } - private static boolean isValidSession(Session session) { - return session != null; - } - // As of now, config generation is based on session id, and config generation must be a monotonically // increasing number static void checkIfActiveIsNewerThanSessionToBeActivated(long sessionId, long currentActiveSessionId) { @@ -412,7 +410,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye * @throws RuntimeException if the delete transaction fails. This method is exception safe. */ public boolean delete(ApplicationId applicationId, Duration waitTime) { - Tenant tenant = tenantRepository.getTenant(applicationId.tenant()); + Tenant tenant = getTenant(applicationId); if (tenant == null) return false; TenantApplications tenantApplications = tenant.getApplicationRepo(); @@ -586,7 +584,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye } private boolean localSessionHasBeenDeleted(ApplicationId applicationId, long sessionId, Duration waitTime) { - SessionRepository sessionRepository = tenantRepository.getTenant(applicationId.tenant()).getSessionRepository(); + SessionRepository sessionRepository = getTenant(applicationId).getSessionRepository(); Instant end = Instant.now().plus(waitTime); do { if (sessionRepository.getRemoteSession(sessionId) == null) return true; @@ -596,6 +594,26 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye return false; } + public Optional<String> getApplicationPackageReference(ApplicationId applicationId) { + Optional<String> applicationPackage = Optional.empty(); + RemoteSession session = getActiveSession(applicationId); + if (session != null) { + FileReference applicationPackageReference = session.getApplicationPackageReference(); + File downloadDirectory = new File(Defaults.getDefaults().underVespaHome(configserverConfig().fileReferencesDir())); + if (applicationPackageReference != null && ! fileReferenceExistsOnDisk(downloadDirectory, applicationPackageReference)) + applicationPackage = Optional.of(applicationPackageReference.value()); + } + return applicationPackage; + } + + public List<Version> getAllVersions(ApplicationId applicationId) { + Optional<ApplicationSet> applicationSet = getCurrentActiveApplicationSet(getTenant(applicationId), applicationId); + if (applicationSet.isEmpty()) + return List.of(); + else + return applicationSet.get().getAllVersions(applicationId); + } + // ---------------- Convergence ---------------------------------------------------------------- public HttpResponse checkServiceForConfigConvergence(ApplicationId applicationId, String hostAndPort, URI uri, @@ -671,11 +689,11 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye * @return the active session, or null if there is no active session for the given application id. */ public RemoteSession getActiveSession(ApplicationId applicationId) { - return getActiveSession(tenantRepository.getTenant(applicationId.tenant()), applicationId); + return getActiveSession(getTenant(applicationId), applicationId); } public long getSessionIdForApplication(ApplicationId applicationId) { - Tenant tenant = tenantRepository.getTenant(applicationId.tenant()); + Tenant tenant = getTenant(applicationId); if (tenant == null) throw new NotFoundException("Tenant '" + applicationId.tenant() + "' not found"); return getSessionIdForApplication(tenant, applicationId); } @@ -704,7 +722,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye DeployLogger logger, boolean internalRedeploy, TimeoutBudget timeoutBudget) { - Tenant tenant = tenantRepository.getTenant(applicationId.tenant()); + Tenant tenant = getTenant(applicationId); SessionRepository sessionRepository = tenant.getSessionRepository(); RemoteSession fromSession = getExistingSession(tenant, applicationId); LocalSession session = sessionRepository.createSessionFromExisting(fromSession, logger, internalRedeploy, timeoutBudget); @@ -724,7 +742,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye } public long createSession(ApplicationId applicationId, TimeoutBudget timeoutBudget, File applicationDirectory) { - Tenant tenant = tenantRepository.getTenant(applicationId.tenant()); + Tenant tenant = getTenant(applicationId); tenant.getApplicationRepo().createApplication(applicationId); Optional<Long> activeSessionId = tenant.getApplicationRepo().activeSessionOf(applicationId); LocalSession session = tenant.getSessionRepository().createSession(applicationDirectory, @@ -742,9 +760,9 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye Set<ApplicationId> applicationIds = new HashSet<>(); sessionsPerTenant.values() .forEach(sessionList -> sessionList.stream() - .map(Session::getApplicationId) - .filter(Objects::nonNull) - .forEach(applicationIds::add)); + .map(Session::getOptionalApplicationId) + .filter(Optional::isPresent) + .forEach(appId -> applicationIds.add(appId.get()))); Map<ApplicationId, Long> activeSessions = new HashMap<>(); applicationIds.forEach(applicationId -> { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java index 0aab83d5a6a..cbea6d99dd2 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java @@ -105,7 +105,11 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable initializing(vipStatusMode); // Run maintainers that cleans up zookeeper and disk usage before bootstrapping - configServerMaintenance.ifPresent(ConfigServerMaintenance::runBeforeBootstrap); + try { + configServerMaintenance.ifPresent(ConfigServerMaintenance::runBeforeBootstrap); + } catch (Exception e) { + log.log(Level.INFO, "Running maintainers before bootstrap failed, continuing with bootstrap", e); + } switch (mode) { case BOOTSTRAP_IN_SEPARATE_THREAD: diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationMapper.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationMapper.java index 5ce9ebca69d..c0158b55422 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationMapper.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationMapper.java @@ -57,7 +57,7 @@ public final class ApplicationMapper { /** Returns whether this registry has an application for the given application id */ public boolean hasApplication(ApplicationId applicationId, Instant now) { - return hasApplicationForVersion(applicationId, Optional.<Version>empty(), now); + return hasApplicationForVersion(applicationId, Optional.empty(), now); } /** Returns whether this registry has an application for the given application id and vespa version */ diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java index d87a37829de..acc1b2ecfde 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java @@ -119,6 +119,15 @@ public class TenantApplications implements RequestHandler, HostValidator<Applica return curator.getChildren(applicationsPath).stream() .sorted() .map(ApplicationId::fromSerializedForm) + .filter(applicationId -> { + if ( ! applicationId.tenant().equals(tenant)) { + log.log(Level.WARNING, "There is an application ('" + applicationId + "') with wrong tenant (should be '" + + tenant + "') in " + applicationsPath); + return false; + } else { + return true; + } + }) .filter(id -> activeSessionOf(id).isPresent()) .collect(Collectors.toUnmodifiableList()); } @@ -420,4 +429,5 @@ public class TenantApplications implements RequestHandler, HostValidator<Applica } public TenantFileSystemDirs getTenantFileSystemDirs() { return tenantFileSystemDirs; } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java index 8c2e6027691..11ce659625d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java @@ -108,17 +108,18 @@ public class Deployment implements com.yahoo.config.provision.Deployment { @Override public void prepare() { if (prepared) return; - try (ActionTimer timer = applicationRepository.timerFor(session.getApplicationId(), "deployment.prepareMillis")) { + ApplicationId applicationId = session.getApplicationId(); + try (ActionTimer timer = applicationRepository.timerFor(applicationId, "deployment.prepareMillis")) { TimeoutBudget timeoutBudget = new TimeoutBudget(clock, timeout); - PrepareParams.Builder params = new PrepareParams.Builder().applicationId(session.getApplicationId()) + PrepareParams.Builder params = new PrepareParams.Builder().applicationId(applicationId) .timeoutBudget(timeoutBudget) .ignoreValidationErrors(!validate) .vespaVersion(version.toString()) .isBootstrap(isBootstrap); dockerImageRepository.ifPresent(params::dockerImageRepository); athenzDomain.ifPresent(params::athenzDomain); - Optional<ApplicationSet> activeApplicationSet = applicationRepository.getCurrentActiveApplicationSet(tenant, session.getApplicationId()); + Optional<ApplicationSet> activeApplicationSet = applicationRepository.getCurrentActiveApplicationSet(tenant, applicationId); tenant.getSessionRepository().prepareLocalSession(session, logger, params.build(), activeApplicationSet, tenant.getPath(), clock.instant()); this.prepared = true; @@ -131,11 +132,10 @@ public class Deployment implements com.yahoo.config.provision.Deployment { if ( ! prepared) prepare(); - try (ActionTimer timer = applicationRepository.timerFor(session.getApplicationId(), "deployment.activateMillis")) { + validateSessionStatus(session); + ApplicationId applicationId = session.getApplicationId(); + try (ActionTimer timer = applicationRepository.timerFor(applicationId, "deployment.activateMillis")) { TimeoutBudget timeoutBudget = new TimeoutBudget(clock, timeout); - validateSessionStatus(session); - ApplicationId applicationId = session.getApplicationId(); - if ( ! timeoutBudget.hasTimeLeft()) throw new RuntimeException("Timeout exceeded when trying to activate '" + applicationId + "'"); RemoteSession previousActiveSession; @@ -148,7 +148,7 @@ public class Deployment implements com.yahoo.config.provision.Deployment { throw e; } catch (Exception e) { - throw new InternalServerException("Error activating application", e); + throw new InternalServerException("Error when activating '" + applicationId + "'", e); } waiter.awaitCompletion(timeoutBudget.timeLeft()); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java index 8426d6b56cf..0a805cc6b21 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.config.server.http.v2; import com.google.inject.Inject; import com.yahoo.component.Version; -import com.yahoo.config.FileReference; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; @@ -18,18 +17,13 @@ import com.yahoo.jdisc.application.BindingMatch; import com.yahoo.jdisc.application.UriPattern; import com.yahoo.slime.Cursor; import com.yahoo.vespa.config.server.ApplicationRepository; -import com.yahoo.vespa.config.server.application.ApplicationSet; import com.yahoo.vespa.config.server.http.ContentHandler; import com.yahoo.vespa.config.server.http.ContentRequest; import com.yahoo.vespa.config.server.http.HttpErrorResponse; import com.yahoo.vespa.config.server.http.HttpHandler; import com.yahoo.vespa.config.server.http.JSONResponse; import com.yahoo.vespa.config.server.http.NotFoundException; -import com.yahoo.vespa.config.server.session.RemoteSession; -import com.yahoo.vespa.config.server.tenant.Tenant; -import com.yahoo.vespa.defaults.Defaults; -import java.io.File; import java.io.IOException; import java.time.Duration; import java.util.List; @@ -38,8 +32,6 @@ import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.Stream; -import static com.yahoo.vespa.config.server.filedistribution.FileDistributionUtil.fileReferenceExistsOnDisk; - /** * Operations on applications (delete, wait for config convergence, restart, application content etc.) * @@ -169,21 +161,10 @@ public class ApplicationHandler extends HttpHandler { } GetApplicationResponse getApplicationResponse(ApplicationId applicationId) { - Tenant tenant = applicationRepository.getTenant(applicationId); - Optional<ApplicationSet> applicationSet = applicationRepository.getCurrentActiveApplicationSet(tenant, applicationId); - String applicationPackage = ""; - RemoteSession session = applicationRepository.getActiveSession(applicationId); - if (session != null) { - FileReference applicationPackageReference = session.getApplicationPackageReference(); - File downloadDirectory = new File(Defaults.getDefaults().underVespaHome(applicationRepository.configserverConfig().fileReferencesDir())); - if (applicationPackageReference != null && ! fileReferenceExistsOnDisk(downloadDirectory, applicationPackageReference)) - applicationPackage = applicationPackageReference.value(); - } - return new GetApplicationResponse(Response.Status.OK, applicationRepository.getApplicationGeneration(applicationId), - applicationSet.get().getAllVersions(applicationId), - applicationPackage); + applicationRepository.getAllVersions(applicationId), + applicationRepository.getApplicationPackageReference(applicationId)); } @Override @@ -346,10 +327,10 @@ public class ApplicationHandler extends HttpHandler { } private static class GetApplicationResponse extends JSONResponse { - GetApplicationResponse(int status, long generation, List<Version> modelVersions, String applicationPackageReference) { + GetApplicationResponse(int status, long generation, List<Version> modelVersions, Optional<String> applicationPackageReference) { super(status); object.setLong("generation", generation); - object.setString("applicationPackageFileReference", applicationPackageReference); + object.setString("applicationPackageFileReference", applicationPackageReference.orElse("")); Cursor modelVersionArray = object.setArray("modelVersions"); modelVersions.forEach(version -> modelVersionArray.addString(version.toFullString())); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandler.java index 124902c988a..4e75234620f 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandler.java @@ -45,7 +45,7 @@ public class SessionActiveHandler extends SessionHandler { Utils.checkThatTenantExists(tenantRepository, tenantName); Tenant tenant = tenantRepository.getTenant(tenantName); TimeoutBudget timeoutBudget = getTimeoutBudget(request, DEFAULT_ACTIVATE_TIMEOUT); - final Long sessionId = getSessionIdV2(request); + long sessionId = getSessionIdV2(request); ApplicationId applicationId = applicationRepository.activate(tenant, sessionId, timeoutBudget, shouldIgnoreSessionStaleFailure(request)); ApplicationMetaData metaData = applicationRepository.getMetadataFromLocalSession(tenant, sessionId); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java index e9687000b3c..e41d5264c08 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java @@ -89,7 +89,7 @@ public class ApplicationPackageMaintainer extends ConfigServerMaintainer { } private void createLocalSessionIfMissing(ApplicationId applicationId, long sessionId) { - Tenant tenant = applicationRepository.tenantRepository().getTenant(applicationId.tenant()); + Tenant tenant = applicationRepository.getTenant(applicationId); SessionRepository sessionRepository = tenant.getSessionRepository(); if (sessionRepository.getLocalSession(sessionId) == null) sessionRepository.createLocalSessionUsingDistributedApplicationPackage(sessionId); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java index 642ac33ab09..36cac87a326 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java @@ -60,7 +60,7 @@ public class RemoteSession extends Session { // Read hosts allocated on the config server instance which created this Optional<AllocatedHosts> allocatedHosts = applicationPackage.getAllocatedHosts(); - return ApplicationSet.fromList(applicationLoader.buildModels(sessionZooKeeperClient.readApplicationId(), + return ApplicationSet.fromList(applicationLoader.buildModels(getApplicationId(), sessionZooKeeperClient.readDockerImageRepository(), sessionZooKeeperClient.readVespaVersion(), applicationPackage, diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java index 0fc85b5e51a..b3e35e955de 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java @@ -34,21 +34,22 @@ public abstract class Session implements Comparable<Session> { protected final Optional<ApplicationPackage> applicationPackage; protected Session(TenantName tenant, long sessionId, SessionZooKeeperClient sessionZooKeeperClient) { - this.tenant = tenant; - this.sessionId = sessionId; - this.sessionZooKeeperClient = sessionZooKeeperClient; - this.applicationPackage = Optional.empty(); + this(tenant, sessionId, sessionZooKeeperClient, Optional.empty()); } protected Session(TenantName tenant, long sessionId, SessionZooKeeperClient sessionZooKeeperClient, ApplicationPackage applicationPackage) { + this(tenant, sessionId, sessionZooKeeperClient, Optional.of(applicationPackage)); + } + + private Session(TenantName tenant, long sessionId, SessionZooKeeperClient sessionZooKeeperClient, + Optional<ApplicationPackage> applicationPackage) { this.tenant = tenant; this.sessionId = sessionId; this.sessionZooKeeperClient = sessionZooKeeperClient; - this.applicationPackage = Optional.of(applicationPackage); + this.applicationPackage = applicationPackage; } - public final long getSessionId() { return sessionId; } @@ -133,7 +134,20 @@ public abstract class Session implements Comparable<Session> { sessionZooKeeperClient.writeAthenzDomain(athenzDomain); } - public ApplicationId getApplicationId() { return sessionZooKeeperClient.readApplicationId(); } + /** Returns application id read from ZooKeeper. Will throw RuntimeException if not found */ + public ApplicationId getApplicationId() { + return sessionZooKeeperClient.readApplicationId() + .orElseThrow(() -> new RuntimeException("Unable to read application id for session " + sessionId)); + } + + /** Returns application id read from ZooKeeper. Will return Optional.empty() if not found */ + public Optional<ApplicationId> getOptionalApplicationId() { + try { + return Optional.of(getApplicationId()); + } catch (RuntimeException e) { + return Optional.empty(); + } + } public FileReference getApplicationPackageReference() {return sessionZooKeeperClient.readApplicationPackageReference(); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java index 6c4ef469be6..cce79412cfe 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java @@ -181,8 +181,9 @@ public class SessionRepository { deleteLocalSession(candidate); } else if (createTime.plus(Duration.ofDays(1)).isBefore(clock.instant())) { // Sessions with state ACTIVATE, but which are not actually active - ApplicationId applicationId = candidate.getApplicationId(); - Long activeSession = activeSessions.get(applicationId); + Optional<ApplicationId> applicationId = candidate.getOptionalApplicationId(); + if (applicationId.isEmpty()) continue; + Long activeSession = activeSessions.get(applicationId.get()); if (activeSession == null || activeSession != candidate.getSessionId()) { deleteLocalSession(candidate); log.log(Level.INFO, "Deleted inactive session " + candidate.getSessionId() + " created " + @@ -622,7 +623,8 @@ public class SessionRepository { log.log(Level.INFO, "File reference for session id " + sessionId + ": " + fileReference + " not found in " + fileDirectory); return Optional.empty(); } - ApplicationId applicationId = sessionZKClient.readApplicationId(); + ApplicationId applicationId = sessionZKClient.readApplicationId() + .orElseThrow(() -> new RuntimeException("Could not find application id for session " + sessionId)); log.log(Level.INFO, "Creating local session for tenant '" + tenantName + "' with session id " + sessionId); LocalSession localSession = createLocalSession(sessionDir, applicationId, sessionId); addLocalSession(localSession); @@ -696,7 +698,7 @@ public class SessionRepository { public Transaction createActivateTransaction(Session session) { Transaction transaction = createSetStatusTransaction(session, Session.Status.ACTIVATE); - transaction.add(applicationRepo.createPutTransaction(session.sessionZooKeeperClient.readApplicationId(), session.getSessionId()).operations()); + transaction.add(applicationRepo.createPutTransaction(session.getApplicationId(), session.getSessionId()).operations()); return transaction; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java index cf1e07788ff..bbf72067b00 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java @@ -11,7 +11,6 @@ import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.AthenzDomain; import com.yahoo.config.provision.DockerImage; -import com.yahoo.config.provision.NodeFlavors; import com.yahoo.path.Path; import com.yahoo.text.Utf8; import com.yahoo.transaction.Transaction; @@ -52,12 +51,6 @@ public class SessionZooKeeperClient { private final Path sessionStatusPath; private final String serverId; // hostname - // Only for testing - // TODO: Remove, use the constructor below - public SessionZooKeeperClient(Curator curator, Path sessionPath) { - this(curator, ConfigCurator.create(curator), sessionPath, "1"); - } - public SessionZooKeeperClient(Curator curator, ConfigCurator configCurator, Path sessionPath, @@ -147,9 +140,11 @@ public class SessionZooKeeperClient { configCurator.putData(applicationIdPath(), id.serializedForm()); } - public ApplicationId readApplicationId() { + public Optional<ApplicationId> readApplicationId() { String idString = configCurator.getData(applicationIdPath()); - return idString == null ? null : ApplicationId.fromSerializedForm(idString); + return (idString == null) + ? Optional.empty() + : Optional.of(ApplicationId.fromSerializedForm(idString)); } void writeApplicationPackageReference(FileReference applicationPackageReference) { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java index e42fb78f595..c2e394fc450 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java @@ -9,6 +9,7 @@ import com.yahoo.config.application.api.ApplicationMetaData; import com.yahoo.config.model.NullConfigModelRegistry; import com.yahoo.config.model.api.ApplicationRoles; import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; @@ -41,11 +42,13 @@ import com.yahoo.vespa.config.server.session.PrepareParams; import com.yahoo.vespa.config.server.session.RemoteSession; import com.yahoo.vespa.config.server.session.Session; import com.yahoo.vespa.config.server.session.SessionRepository; +import com.yahoo.vespa.config.server.session.SessionZooKeeperClient; import com.yahoo.vespa.config.server.session.SilentDeployLogger; import com.yahoo.vespa.config.server.tenant.ApplicationRolesStore; import com.yahoo.vespa.config.server.tenant.Tenant; import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; +import com.yahoo.vespa.config.util.ConfigUtils; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; import com.yahoo.vespa.flags.FlagSource; @@ -61,6 +64,7 @@ import org.junit.rules.TemporaryFolder; import java.io.File; import java.io.IOException; +import java.nio.file.Files; import java.time.Duration; import java.time.Instant; import java.util.ArrayList; @@ -105,6 +109,7 @@ public class ApplicationRepositoryTest { private SessionHandlerTest.MockProvisioner provisioner; private OrchestratorMock orchestrator; private TimeoutBudget timeoutBudget; + private Curator curator; private ConfigCurator configCurator; @Rule @@ -119,7 +124,7 @@ public class ApplicationRepositoryTest { } public void setup(FlagSource flagSource) throws IOException { - Curator curator = new MockCurator(); + curator = new MockCurator(); configCurator = ConfigCurator.create(curator); ConfigserverConfig configserverConfig = new ConfigserverConfig.Builder() .payloadCompressionType(ConfigserverConfig.PayloadCompressionType.Enum.UNCOMPRESSED) @@ -157,8 +162,7 @@ public class ApplicationRepositoryTest { assertTrue(result.configChangeActions().getRefeedActions().isEmpty()); assertTrue(result.configChangeActions().getRestartActions().isEmpty()); - TenantName tenantName = applicationId().tenant(); - Tenant tenant = tenantRepository.getTenant(tenantName); + Tenant tenant = applicationRepository.getTenant(applicationId()); LocalSession session = tenant.getSessionRepository().getLocalSession(tenant.getApplicationRepo() .requireActiveSessionOf(applicationId())); session.getAllocatedHosts(); @@ -189,8 +193,7 @@ public class ApplicationRepositoryTest { long secondSessionId = result2.sessionId(); assertNotEquals(firstSessionId, secondSessionId); - TenantName tenantName = applicationId().tenant(); - Tenant tenant = tenantRepository.getTenant(tenantName); + Tenant tenant = applicationRepository.getTenant(applicationId()); LocalSession session = tenant.getSessionRepository().getLocalSession( tenant.getApplicationRepo().requireActiveSessionOf(applicationId())); assertEquals(firstSessionId, session.getMetaData().getPreviousActiveGeneration()); @@ -273,8 +276,7 @@ public class ApplicationRepositoryTest { @Test public void delete() { - TenantName tenantName = applicationId().tenant(); - Tenant tenant = tenantRepository.getTenant(tenantName); + Tenant tenant = applicationRepository.getTenant(applicationId()); SessionRepository sessionRepository = tenant.getSessionRepository(); { PrepareResult result = deployApp(testApp); @@ -347,9 +349,10 @@ public class ApplicationRepositoryTest { @Test public void testDeletingInactiveSessions() throws IOException { + File serverdb = temporaryFolder.newFolder("serverdb"); ConfigserverConfig configserverConfig = new ConfigserverConfig(new ConfigserverConfig.Builder() - .configServerDBDir(temporaryFolder.newFolder("serverdb").getAbsolutePath()) + .configServerDBDir(serverdb.getAbsolutePath()) .configDefinitionsDir(temporaryFolder.newFolder("configdefinitions").getAbsolutePath()) .sessionLifetime(60)); DeployTester tester = new DeployTester(configserverConfig, clock); @@ -387,6 +390,8 @@ public class ApplicationRepositoryTest { // There should be no expired remote sessions in the common case assertEquals(0, tester.applicationRepository().deleteExpiredRemoteSessions(clock, Duration.ofSeconds(0))); + assertEquals(1, sessionRepository.getLocalSessions().size()); + // Deploy, but do not activate Optional<com.yahoo.config.provision.Deployment> deployment4 = tester.redeployFromLocalActive(); assertTrue(deployment4.isPresent()); @@ -396,6 +401,24 @@ public class ApplicationRepositoryTest { sessionRepository.deleteLocalSession(localSession); assertEquals(1, sessionRepository.getLocalSessions().size()); + // Create a local session without any data in zookeeper (corner case seen in production occasionally) + // and check that expiring local sessions still work + int sessionId = 6; + Files.createDirectory(new TenantFileSystemDirs(serverdb, tenant1).getUserApplicationDir(sessionId).toPath()); + LocalSession localSession2 = new LocalSession(tenant1, + sessionId, + FilesApplicationPackage.fromFile(testApp), + new SessionZooKeeperClient(curator, + configCurator, + sessionRepository.getSessionPath(sessionId), + ConfigUtils.getCanonicalHostName())); + sessionRepository.addLocalSession(localSession2); + assertEquals(2, sessionRepository.getLocalSessions().size()); + + // Check that trying to expire local session when there exists a local session with no zookeeper data works + tester.applicationRepository().deleteExpiredLocalSessions(); + assertEquals(1, sessionRepository.getLocalSessions().size()); + // Check that trying to expire when there are no active sessions works tester.applicationRepository().deleteExpiredLocalSessions(); } @@ -424,7 +447,7 @@ public class ApplicationRepositoryTest { @Test public void deletesApplicationRoles() { - var tenant = tenantRepository.getTenant(tenant1); + var tenant = applicationRepository.getTenant(applicationId()); var applicationId = applicationId(tenant1); var prepareParams = new PrepareParams.Builder().applicationId(applicationId) .applicationRoles(ApplicationRoles.fromString("hostRole","containerRole")).build(); @@ -447,8 +470,7 @@ public class ApplicationRepositoryTest { public void require_that_provision_info_can_be_read() { prepareAndActivate(testAppJdiscOnly); - TenantName tenantName = applicationId().tenant(); - Tenant tenant = tenantRepository.getTenant(tenantName); + Tenant tenant = applicationRepository.getTenant(applicationId()); LocalSession session = tenant.getSessionRepository().getLocalSession(tenant.getApplicationRepo().requireActiveSessionOf(applicationId())); List<NetworkPorts.Allocation> list = new ArrayList<>(); @@ -494,7 +516,7 @@ public class ApplicationRepositoryTest { long sessionId = applicationRepository.createSession(applicationId(), timeoutBudget, testAppJdiscOnly); exceptionRule.expect(IllegalStateException.class); exceptionRule.expectMessage(containsString("tenant:test1 Session 3 is not prepared")); - applicationRepository.activate(tenantRepository.getTenant(tenant1), sessionId, timeoutBudget, false); + applicationRepository.activate(applicationRepository.getTenant(applicationId()), sessionId, timeoutBudget, false); RemoteSession activeSession = applicationRepository.getActiveSession(applicationId()); assertEquals(firstSession, activeSession.getSessionId()); @@ -508,10 +530,10 @@ public class ApplicationRepositoryTest { long firstSession = result.sessionId(); long sessionId = applicationRepository.createSession(applicationId(), timeoutBudget, testAppJdiscOnly); - applicationRepository.prepare(tenantRepository.getTenant(tenant1), sessionId, prepareParams(), clock.instant()); + applicationRepository.prepare(applicationRepository.getTenant(applicationId()), sessionId, prepareParams(), clock.instant()); exceptionRule.expect(RuntimeException.class); exceptionRule.expectMessage(containsString("Timeout exceeded when trying to activate 'test1.testapp'")); - applicationRepository.activate(tenantRepository.getTenant(tenant1), sessionId, new TimeoutBudget(clock, Duration.ofSeconds(0)), false); + applicationRepository.activate(applicationRepository.getTenant(applicationId()), sessionId, new TimeoutBudget(clock, Duration.ofSeconds(0)), false); RemoteSession activeSession = applicationRepository.getActiveSession(applicationId()); assertEquals(firstSession, activeSession.getSessionId()); @@ -533,10 +555,10 @@ public class ApplicationRepositoryTest { PrepareResult result2 = deployApp(testAppJdiscOnly); result2.sessionId(); - applicationRepository.prepare(tenantRepository.getTenant(tenant1), sessionId2, prepareParams(), clock.instant()); + applicationRepository.prepare(applicationRepository.getTenant(applicationId()), sessionId2, prepareParams(), clock.instant()); exceptionRule.expect(ActivationConflictException.class); exceptionRule.expectMessage(containsString("tenant:test1 app:testapp:default Cannot activate session 3 because the currently active session (4) has changed since session 3 was created (was 2 at creation time)")); - applicationRepository.activate(tenantRepository.getTenant(tenant1), sessionId2, timeoutBudget, false); + applicationRepository.activate(applicationRepository.getTenant(applicationId()), sessionId2, timeoutBudget, false); } @Test @@ -546,11 +568,11 @@ public class ApplicationRepositoryTest { exceptionRule.expect(IllegalStateException.class); exceptionRule.expectMessage(containsString("Session is active: 2")); - applicationRepository.prepare(tenantRepository.getTenant(tenant1), sessionId, prepareParams(), clock.instant()); + applicationRepository.prepare(applicationRepository.getTenant(applicationId()), sessionId, prepareParams(), clock.instant()); exceptionRule.expect(IllegalStateException.class); exceptionRule.expectMessage(containsString("tenant:test1 app:testapp:default Session 2 is already active")); - applicationRepository.activate(tenantRepository.getTenant(tenant1), sessionId, timeoutBudget, false); + applicationRepository.activate(applicationRepository.getTenant(applicationId()), sessionId, timeoutBudget, false); } @Test diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java index 8bf5215a696..0db6aea312d 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java @@ -45,13 +45,12 @@ public class ApplicationContentHandlerTest extends ContentHandlerTestBase { private final ApplicationId appId1 = new ApplicationId.Builder().tenant(tenantName1).applicationName("foo").instanceName("quux").build(); private final ApplicationId appId2 = new ApplicationId.Builder().tenant(tenantName2).applicationName("foo").instanceName("quux").build(); - private TenantRepository tenantRepository; private ApplicationRepository applicationRepository; private ApplicationHandler handler; @Before public void setupHandler() { - tenantRepository = new TenantRepository(componentRegistry, false); + TenantRepository tenantRepository = new TenantRepository(componentRegistry, false); tenantRepository.addTenant(tenantName1); tenantRepository.addTenant(tenantName2); @@ -106,7 +105,7 @@ public class ApplicationContentHandlerTest extends ContentHandlerTestBase { @Test public void require_that_get_does_not_set_write_flag() throws IOException { - Tenant tenant1 = tenantRepository.getTenant(tenantName1); + Tenant tenant1 = applicationRepository.getTenant(appId1); LocalSession session = applicationRepository.getActiveLocalSession(tenant1, appId1); assertContent("/test.txt", "foo\n"); assertThat(session.getStatus(), is(Session.Status.ACTIVATE)); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java index 1a558f89284..b6a96d680ec 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java @@ -113,7 +113,7 @@ public class ApplicationHandlerTest { { applicationRepository.deploy(testApp, prepareParams(applicationId)); - Tenant mytenant = tenantRepository.getTenant(applicationId.tenant()); + Tenant mytenant = applicationRepository.getTenant(applicationId); deleteAndAssertOKResponse(mytenant, applicationId); } @@ -301,9 +301,10 @@ public class ApplicationHandlerTest { } private void deleteAndAssertOKResponseMocked(ApplicationId applicationId, boolean fullAppIdInUrl) throws IOException { - long sessionId = tenantRepository.getTenant(applicationId.tenant()).getApplicationRepo().requireActiveSessionOf(applicationId); + Tenant tenant = applicationRepository.getTenant(applicationId); + long sessionId = tenant.getApplicationRepo().requireActiveSessionOf(applicationId); deleteAndAssertResponse(applicationId, Zone.defaultZone(), Response.Status.OK, null, fullAppIdInUrl); - assertNull(tenantRepository.getTenant(applicationId.tenant()).getSessionRepository().getLocalSession(sessionId)); + assertNull(tenant.getSessionRepository().getLocalSession(sessionId)); } private void deleteAndAssertOKResponse(Tenant tenant, ApplicationId applicationId) throws IOException { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java index 364e7372e20..5bf5e1f2229 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java @@ -39,7 +39,6 @@ public class HostHandlerTest { private HostHandler handler; private final static TenantName mytenant = TenantName.from("mytenant"); private final static Zone zone = Zone.defaultZone(); - private TenantRepository tenantRepository; private ApplicationRepository applicationRepository; @Before @@ -47,7 +46,7 @@ public class HostHandlerTest { TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder() .zone(zone) .build(); - tenantRepository = new TenantRepository(componentRegistry); + TenantRepository tenantRepository = new TenantRepository(componentRegistry); tenantRepository.addTenant(mytenant); applicationRepository = new ApplicationRepository(tenantRepository, new SessionHandlerTest.MockProvisioner(), @@ -60,7 +59,7 @@ public class HostHandlerTest { public void require_correct_tenant_and_application_for_hostname() throws Exception { ApplicationId applicationId = applicationId(); applicationRepository.deploy(testApp, new PrepareParams.Builder().applicationId(applicationId).build()); - Tenant tenant = tenantRepository.getTenant(mytenant); + Tenant tenant = applicationRepository.getTenant(applicationId); String hostname = applicationRepository.getCurrentActiveApplicationSet(tenant, applicationId).get().getAllHosts().iterator().next(); assertApplicationForHost(hostname, applicationId); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java index 53422df3fad..e48752afaf6 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java @@ -58,7 +58,6 @@ public class SessionActiveHandlerTest { private SessionHandlerTest.MockProvisioner hostProvisioner; private TestComponentRegistry componentRegistry; - private TenantRepository tenantRepository; private ApplicationRepository applicationRepository; private SessionActiveHandler handler; @@ -73,10 +72,10 @@ public class SessionActiveHandlerTest { .curator(new MockCurator()) .modelFactoryRegistry(new ModelFactoryRegistry(List.of((modelFactory)))) .build(); - tenantRepository = new TenantRepository(componentRegistry, false); + TenantRepository tenantRepository = new TenantRepository(componentRegistry, false); + tenantRepository.addTenant(tenantName); applicationRepository = new ApplicationRepository(tenantRepository, hostProvisioner, new OrchestratorMock(), componentRegistry.getClock()); - tenantRepository.addTenant(tenantName); handler = createHandler(); } @@ -124,7 +123,7 @@ public class SessionActiveHandlerTest { ApplicationMetaData getMetaData() { return metaData; } void invoke() { - Tenant tenant = tenantRepository.getTenant(tenantName); + Tenant tenant = applicationRepository.getTenant(applicationId()); long sessionId = applicationRepository.createSession(applicationId(), new TimeoutBudget(componentRegistry.getClock(), Duration.ofSeconds(10)), testApp); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java index 3ab56d3869a..cda62361256 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java @@ -54,7 +54,7 @@ public class SessionContentHandlerTest extends ContentHandlerTestBase { new OrchestratorMock(), Clock.systemUTC()); applicationRepository.deploy(testApp, new PrepareParams.Builder().applicationId(applicationId()).build()); - Tenant tenant = tenantRepository.getTenant(tenantName); + Tenant tenant = applicationRepository.getTenant(applicationId()); sessionId = applicationRepository.getActiveLocalSession(tenant, applicationId()).getSessionId(); handler = createHandler(); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/MockSessionZKClient.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/MockSessionZKClient.java index 692510b8b6d..296fbf56b3d 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/MockSessionZKClient.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/MockSessionZKClient.java @@ -6,6 +6,8 @@ import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.TenantName; import com.yahoo.vespa.config.server.tenant.TenantRepository; +import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; +import com.yahoo.vespa.config.util.ConfigUtils; import com.yahoo.vespa.curator.Curator; import java.util.Optional; @@ -30,7 +32,10 @@ public class MockSessionZKClient extends SessionZooKeeperClient { } MockSessionZKClient(Curator curator, TenantName tenantName, long sessionId, ApplicationPackage application) { - super(curator, TenantRepository.getSessionsPath(tenantName).append(String.valueOf(sessionId))); + super(curator, + ConfigCurator.create(curator), + TenantRepository.getSessionsPath(tenantName).append(String.valueOf(sessionId)), + ConfigUtils.getCanonicalHostName()); this.app = application; curator.create(TenantRepository.getSessionsPath(tenantName).append(String.valueOf(sessionId))); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java index e53f9270983..ee8f00f6bcf 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java @@ -43,6 +43,7 @@ import com.yahoo.vespa.config.server.tenant.ContainerEndpointsCache; import com.yahoo.vespa.config.server.tenant.EndpointCertificateMetadataStore; import com.yahoo.vespa.config.server.tenant.EndpointCertificateRetriever; import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; +import com.yahoo.vespa.config.util.ConfigUtils; import com.yahoo.vespa.curator.mock.MockCurator; import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.InMemoryFlagSource; @@ -172,7 +173,7 @@ public class SessionPreparerTest { HostRegistry<ApplicationId> hostValidator = new HostRegistry<>(); hostValidator.update(applicationId("foo"), Collections.singletonList("mytesthost")); preparer.prepare(hostValidator, new BaseDeployLogger(), new PrepareParams.Builder().applicationId(applicationId("default")).build(), - Optional.empty(), tenantPath, Instant.now(), app.getAppDir(), app, new SessionZooKeeperClient(curator, sessionsPath)); + Optional.empty(), tenantPath, Instant.now(), app.getAppDir(), app, createSessionZooKeeperClient()); } @Test @@ -187,7 +188,7 @@ public class SessionPreparerTest { hostValidator.update(applicationId, Collections.singletonList("mytesthost")); preparer.prepare(hostValidator, logger, new PrepareParams.Builder().applicationId(applicationId).build(), Optional.empty(), tenantPath, Instant.now(), app.getAppDir(), app, - new SessionZooKeeperClient(curator, sessionsPath)); + createSessionZooKeeperClient()); assertEquals(logged.toString(), ""); } @@ -199,9 +200,8 @@ public class SessionPreparerTest { .applicationName("foo").instanceName("quux").build(); PrepareParams params = new PrepareParams.Builder().applicationId(origId).build(); prepare(testApp, params); - SessionZooKeeperClient zkc = new SessionZooKeeperClient(curator, sessionsPath); assertTrue(configCurator.exists(sessionsPath.append(SessionZooKeeperClient.APPLICATION_ID_PATH).getAbsolute())); - assertThat(zkc.readApplicationId(), is(origId)); + assertThat(createSessionZooKeeperClient().readApplicationId().get(), is(origId)); } @Test @@ -336,7 +336,7 @@ public class SessionPreparerTest { FilesApplicationPackage applicationPackage = getApplicationPackage(app); return preparer.prepare(new HostRegistry<>(), getLogger(), params, Optional.empty(), tenantPath, Instant.now(), applicationPackage.getAppDir(), - applicationPackage, new SessionZooKeeperClient(curator, sessionsPath)); + applicationPackage, createSessionZooKeeperClient()); } private FilesApplicationPackage getApplicationPackage(File testFile) throws IOException { @@ -360,6 +360,10 @@ public class SessionPreparerTest { ApplicationName.from(applicationName), InstanceName.defaultName()); } + private SessionZooKeeperClient createSessionZooKeeperClient() { + return new SessionZooKeeperClient(curator, configCurator, sessionsPath, ConfigUtils.getCanonicalHostName()); + } + private static class FailWithTransientExceptionProvisioner implements Provisioner { @Override diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionRepositoryTest.java index 13a27e570c2..9b9682faf5f 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionRepositoryTest.java @@ -11,6 +11,8 @@ import com.yahoo.vespa.config.server.TestComponentRegistry; import com.yahoo.vespa.config.server.application.OrchestratorMock; import com.yahoo.vespa.config.server.http.SessionHandlerTest; import com.yahoo.vespa.config.server.tenant.TenantRepository; +import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; +import com.yahoo.vespa.config.util.ConfigUtils; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; import com.yahoo.vespa.flags.FlagSource; @@ -153,7 +155,10 @@ public class SessionRepositoryTest { } private void createSession(long sessionId, boolean wait, SessionRepository sessionRepository) { - SessionZooKeeperClient zkc = new SessionZooKeeperClient(curator, sessionRepository.getSessionPath(sessionId)); + SessionZooKeeperClient zkc = new SessionZooKeeperClient(curator, + ConfigCurator.create(curator), + sessionRepository.getSessionPath(sessionId), + ConfigUtils.getCanonicalHostName()); zkc.createNewSession(Instant.now()); if (wait) { Curator.CompletionWaiter waiter = zkc.getUploadWaiter(); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClientTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClientTest.java index 1d7df7acfd0..2fc4c0f456f 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClientTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClientTest.java @@ -6,6 +6,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.path.Path; import com.yahoo.text.Utf8; import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; +import com.yahoo.vespa.config.util.ConfigUtils; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; import org.junit.Before; @@ -107,12 +108,15 @@ public class SessionZooKeeperClientTest { SessionZooKeeperClient zkc = createSessionZKClient(sessionId); String path = "/" + sessionId + "/" + SessionZooKeeperClient.APPLICATION_ID_PATH; configCurator.putData(path, idString); - ApplicationId zkId = zkc.readApplicationId(); - assertThat(zkId.serializedForm(), is(expectedIdString)); + ApplicationId applicationId = zkc.readApplicationId().get(); + assertThat(applicationId.serializedForm(), is(expectedIdString)); } private SessionZooKeeperClient createSessionZKClient(String sessionId) { - SessionZooKeeperClient zkc = new SessionZooKeeperClient(curator, Path.fromString(sessionId)); + SessionZooKeeperClient zkc = new SessionZooKeeperClient(curator, + ConfigCurator.create(curator), + Path.fromString(sessionId), + ConfigUtils.getCanonicalHostName()); zkc.createNewSession(Instant.now()); return zkc; } diff --git a/document/src/tests/fieldsettest.cpp b/document/src/tests/fieldsettest.cpp index af23e713735..29581ff4549 100644 --- a/document/src/tests/fieldsettest.cpp +++ b/document/src/tests/fieldsettest.cpp @@ -31,7 +31,7 @@ TEST_F(FieldSetTest, testParsing) (void) dynamic_cast<NoFields&>(*FieldSetRepo::parse(docRepo, NoFields::NAME)); (void) dynamic_cast<DocIdOnly&>(*FieldSetRepo::parse(docRepo, DocIdOnly::NAME)); - auto set = FieldSetRepo::parse(docRepo, "testdoctype1:headerval,content"); + FieldSet::UP set = FieldSetRepo::parse(docRepo, "testdoctype1:headerval,content"); auto & coll = dynamic_cast<FieldCollection&>(*set); std::ostringstream ost; @@ -46,8 +46,8 @@ namespace { bool checkContains(const DocumentTypeRepo& repo, const std::string& str1, const std::string & str2) { - auto set1 = FieldSetRepo::parse(repo, str1); - auto set2 = FieldSetRepo::parse(repo, str2); + FieldSet::UP set1 = FieldSetRepo::parse(repo, str1); + FieldSet::UP set2 = FieldSetRepo::parse(repo, str2); return set1->contains(*set2); } @@ -141,7 +141,7 @@ FieldSetTest::doCopyFields(const Document& src, if (!dest) { dest = &destDoc; } - auto fset = FieldSetRepo::parse(docRepo, fieldSetStr); + FieldSet::UP fset = FieldSetRepo::parse(docRepo, fieldSetStr); FieldSet::copyFields(*dest, src, *fset); return stringifyFields(*dest); } @@ -152,7 +152,7 @@ FieldSetTest::doStripFields(const Document& doc, const std::string& fieldSetStr) { Document::UP copy(doc.clone()); - auto fset = FieldSetRepo::parse(docRepo, fieldSetStr); + FieldSet::UP fset = FieldSetRepo::parse(docRepo, fieldSetStr); FieldSet::stripFields(*copy, *fset); return stringifyFields(*copy); } @@ -198,7 +198,7 @@ FieldSetTest::doCopyDocument(const Document& src, const DocumentTypeRepo& docRepo, const std::string& fieldSetStr) { - auto fset = FieldSetRepo::parse(docRepo, fieldSetStr); + FieldSet::UP fset = FieldSetRepo::parse(docRepo, fieldSetStr); Document::UP doc(FieldSet::createDocumentSubsetCopy(src, *fset)); return stringifyFields(*doc); } @@ -244,9 +244,10 @@ TEST_F(FieldSetTest, testSerialize) "testdoctype1:content,hstringval" }; + FieldSetRepo repo; for (const char * fieldSet : fieldSets) { - auto fs = FieldSetRepo::parse(docRepo, fieldSet); - EXPECT_EQ(vespalib::string(fieldSet), FieldSetRepo::serialize(*fs)); + FieldSet::UP fs = FieldSetRepo::parse(docRepo, fieldSet); + EXPECT_EQ(vespalib::string(fieldSet), repo.serialize(*fs)); } } diff --git a/document/src/vespa/document/base/field.h b/document/src/vespa/document/base/field.h index 305021ef29a..7580b2b692f 100644 --- a/document/src/vespa/document/base/field.h +++ b/document/src/vespa/document/base/field.h @@ -88,6 +88,7 @@ public: */ Field(vespalib::stringref name, const DataType &dataType); + Field* clone() const override { return new Field(*this); } std::unique_ptr<FieldValue> createValue() const; // Note that only id is checked for equality. diff --git a/document/src/vespa/document/datatype/documenttype.h b/document/src/vespa/document/datatype/documenttype.h index fae65addb48..ed6e9e66ab5 100644 --- a/document/src/vespa/document/datatype/documenttype.h +++ b/document/src/vespa/document/datatype/documenttype.h @@ -61,10 +61,12 @@ public: DocumentType(); DocumentType(vespalib::stringref name, int32_t id); - DocumentType(vespalib::stringref name, int32_t id, const StructDataType& fields); + DocumentType(vespalib::stringref name, int32_t id, + const StructDataType& fields); explicit DocumentType(vespalib::stringref name); - DocumentType(vespalib::stringref name, const StructDataType& fields); + DocumentType(vespalib::stringref name, + const StructDataType& fields); ~DocumentType() override; @@ -99,7 +101,6 @@ public: DocumentType & addFieldSet(const vespalib::string & name, FieldSet::Fields fields); const FieldSet * getFieldSet(const vespalib::string & name) const; - const FieldSetMap & getFieldSets() const { return _fieldSets; } const ImportedFieldNames& imported_field_names() const noexcept { return _imported_field_names; diff --git a/document/src/vespa/document/fieldset/fieldset.h b/document/src/vespa/document/fieldset/fieldset.h index 3d74659ebf5..35c912c5e45 100644 --- a/document/src/vespa/document/fieldset/fieldset.h +++ b/document/src/vespa/document/fieldset/fieldset.h @@ -23,8 +23,7 @@ public: DOCID }; - using SP = std::shared_ptr<FieldSet>; - using UP = std::unique_ptr<FieldSet>; + typedef std::unique_ptr<FieldSet> UP; virtual ~FieldSet() = default; @@ -40,6 +39,11 @@ public: virtual Type getType() const = 0; /** + * @return Returns a copy of this object. + */ + virtual FieldSet* clone() const = 0; + + /** * Copy all fields from src into dest that are contained within the * given field set. If any copied field pre-exists in dest, it will * be overwritten. diff --git a/document/src/vespa/document/fieldset/fieldsetrepo.cpp b/document/src/vespa/document/fieldset/fieldsetrepo.cpp index 5bde291c8dd..33cbf6185c4 100644 --- a/document/src/vespa/document/fieldset/fieldsetrepo.cpp +++ b/document/src/vespa/document/fieldset/fieldsetrepo.cpp @@ -5,7 +5,6 @@ #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/document/repo/documenttyperepo.h> -#include <vespa/vespalib/stllike/hash_map.hpp> using vespalib::StringTokenizer; @@ -13,25 +12,27 @@ namespace document { namespace { -FieldSet::SP +FieldSet::UP parseSpecialValues(vespalib::stringref name) { + FieldSet::UP fs; if ((name.size() == 4) && (name[1] == 'i') && (name[2] == 'd') && (name[3] == ']')) { - return std::make_shared<DocIdOnly>(); + fs = std::make_unique<DocIdOnly>(); } else if ((name.size() == 5) && (name[1] == 'a') && (name[2] == 'l') && (name[3] == 'l') && (name[4] == ']')) { - return std::make_shared<AllFields>(); + fs = std::make_unique<AllFields>(); } else if ((name.size() == 6) && (name[1] == 'n') && (name[2] == 'o') && (name[3] == 'n') && (name[4] == 'e') && (name[5] == ']')) { - return std::make_shared<NoFields>(); + fs = std::make_unique<NoFields>(); } else if ((name.size() == 7) && (name[1] == 'd') && (name[2] == 'o') && (name[3] == 'c') && (name[4] == 'i') && (name[5] == 'd') && (name[6] == ']')) { - return std::make_shared<DocIdOnly>(); + fs = std::make_unique<DocIdOnly>(); } else { throw vespalib::IllegalArgumentException( "The only special names (enclosed in '[]') allowed are " "id, all, none, not '" + name + "'."); } + return fs; } -FieldSet::SP +FieldSet::UP parseFieldCollection(const DocumentTypeRepo& repo, vespalib::stringref docType, vespalib::stringref fieldNames) @@ -54,12 +55,12 @@ parseFieldCollection(const DocumentTypeRepo& repo, builder.add(&type.getField(token)); } } - return std::make_shared<FieldCollection>(type, builder.build()); + return std::make_unique<FieldCollection>(type, builder.build()); } } -FieldSet::SP +FieldSet::UP FieldSetRepo::parse(const DocumentTypeRepo& repo, vespalib::stringref str) { if (str[0] == '[') { @@ -110,31 +111,5 @@ FieldSetRepo::serialize(const FieldSet& fieldSet) } } - -FieldSetRepo::FieldSetRepo(const DocumentTypeRepo& repo) - : _doumentTyperepo(repo), - _configuredFieldSets() -{ - repo.forEachDocumentType(*vespalib::makeClosure(this, &FieldSetRepo::configureDocumentType)); -} -FieldSetRepo::~FieldSetRepo() = default; - -void -FieldSetRepo::configureDocumentType(const DocumentType & documentType) { - for (const auto & entry : documentType.getFieldSets()) { - vespalib::string fieldSetName(documentType.getName()); - fieldSetName.append(':').append(entry.first); - _configuredFieldSets[fieldSetName] = parse(_doumentTyperepo, fieldSetName); - } -} -FieldSet::SP -FieldSetRepo::getFieldSet(vespalib::stringref fieldSetString) const { - auto found = _configuredFieldSets.find(fieldSetString); - if (found != _configuredFieldSets.end()) { - return found->second; - } - return parse(_doumentTyperepo, fieldSetString); -} - } diff --git a/document/src/vespa/document/fieldset/fieldsetrepo.h b/document/src/vespa/document/fieldset/fieldsetrepo.h index d213230848a..a744aa572e5 100644 --- a/document/src/vespa/document/fieldset/fieldsetrepo.h +++ b/document/src/vespa/document/fieldset/fieldsetrepo.h @@ -16,17 +16,10 @@ class DocumentTypeRepo; class FieldSetRepo { public: - FieldSetRepo(const DocumentTypeRepo& repo); - ~FieldSetRepo(); + static FieldSet::UP parse(const DocumentTypeRepo& repo, + vespalib::stringref fieldSetString); - FieldSet::SP getFieldSet(vespalib::stringref fieldSetString) const; - - static FieldSet::SP parse(const DocumentTypeRepo& repo, vespalib::stringref fieldSetString); static vespalib::string serialize(const FieldSet& fs); -private: - void configureDocumentType(const DocumentType & documentType); - const DocumentTypeRepo & _doumentTyperepo; - vespalib::hash_map<vespalib::string, FieldSet::SP> _configuredFieldSets; }; } diff --git a/document/src/vespa/document/fieldset/fieldsets.h b/document/src/vespa/document/fieldset/fieldsets.h index e71a96e5a7e..9537a5bdf61 100644 --- a/document/src/vespa/document/fieldset/fieldsets.h +++ b/document/src/vespa/document/fieldset/fieldsets.h @@ -13,6 +13,7 @@ public: static constexpr const char * NAME = "[all]"; bool contains(const FieldSet&) const override { return true; } Type getType() const override { return Type::ALL; } + FieldSet* clone() const override { return new AllFields(); } }; class NoFields final : public FieldSet @@ -21,6 +22,7 @@ public: static constexpr const char * NAME = "[none]"; bool contains(const FieldSet& f) const override { return f.getType() == Type::NONE; } Type getType() const override { return Type::NONE; } + FieldSet* clone() const override { return new NoFields(); } }; class DocIdOnly final : public FieldSet @@ -31,6 +33,7 @@ public: return fields.getType() == Type::DOCID || fields.getType() == Type::NONE; } Type getType() const override { return Type::DOCID; } + FieldSet* clone() const override { return new DocIdOnly(); } }; class FieldCollection : public FieldSet @@ -46,8 +49,20 @@ public: bool contains(const FieldSet& fields) const override; Type getType() const override { return Type::SET; } - const DocumentType& getDocumentType() const { return _docType; } + /** + * @return Returns the document type the collection is associated with. + */ + const DocumentType& getDocumentType() const { + return _docType; + } + + /** + * Returns all the fields contained in this collection. + */ const Field::Set& getFields() const { return _set; } + + FieldSet* clone() const override { return new FieldCollection(*this); } + uint64_t hash() const { return _hash; } private: Field::Set _set; diff --git a/eval/CMakeLists.txt b/eval/CMakeLists.txt index 3e81521550a..fe9d9985c6a 100644 --- a/eval/CMakeLists.txt +++ b/eval/CMakeLists.txt @@ -52,6 +52,7 @@ vespa_define_module( src/tests/tensor/direct_dense_tensor_builder src/tests/tensor/direct_sparse_tensor_builder src/tests/tensor/index_lookup_table + src/tests/tensor/onnx_wrapper src/tests/tensor/tensor_add_operation src/tests/tensor/tensor_address src/tests/tensor/tensor_conformance diff --git a/eval/src/tests/tensor/onnx_wrapper/CMakeLists.txt b/eval/src/tests/tensor/onnx_wrapper/CMakeLists.txt new file mode 100644 index 00000000000..9c92f75476c --- /dev/null +++ b/eval/src/tests/tensor/onnx_wrapper/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(eval_onnx_wrapper_test_app TEST + SOURCES + onnx_wrapper_test.cpp + DEPENDS + vespaeval + GTest::GTest +) +vespa_add_test(NAME eval_onnx_wrapper_test_app COMMAND eval_onnx_wrapper_test_app) diff --git a/eval/src/tests/tensor/onnx_wrapper/onnx_wrapper_test.cpp b/eval/src/tests/tensor/onnx_wrapper/onnx_wrapper_test.cpp new file mode 100644 index 00000000000..07e065f9e39 --- /dev/null +++ b/eval/src/tests/tensor/onnx_wrapper/onnx_wrapper_test.cpp @@ -0,0 +1,119 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/eval/tensor/dense/onnx_wrapper.h> +#include <vespa/eval/tensor/dense/dense_tensor_view.h> +#include <vespa/eval/tensor/dense/mutable_dense_tensor_view.h> +#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/gtest/gtest.h> + +using namespace vespalib::eval; +using namespace vespalib::tensor; + +using vespalib::make_string_short::fmt; + +std::string get_source_dir() { + const char *dir = getenv("SOURCE_DIRECTORY"); + return (dir ? dir : "."); +} +std::string source_dir = get_source_dir(); +std::string vespa_dir = source_dir + "/" + "../../../../.."; +std::string simple_model = vespa_dir + "/" + "model-integration/src/test/models/onnx/simple/simple.onnx"; + +vespalib::string to_str(const std::vector<size_t> &dim_sizes) { + vespalib::string res; + for (size_t dim_size: dim_sizes) { + if (dim_size == 0) { + res += "[]"; + } else { + res += fmt("[%zu]", dim_size); + } + } + return res; +} + +vespalib::string to_str(OnnxWrapper::TensorInfo::ElementType element_type) { + if (element_type == OnnxWrapper::TensorInfo::ElementType::FLOAT) { + return "float"; + } + if (element_type == OnnxWrapper::TensorInfo::ElementType::DOUBLE) { + return "double"; + } + return "???"; +} + +void dump_info(const char *ctx, const std::vector<OnnxWrapper::TensorInfo> &info) { + fprintf(stderr, "%s:\n", ctx); + for (size_t i = 0; i < info.size(); ++i) { + fprintf(stderr, " %s[%zu]: '%s' %s%s\n", ctx, i, info[i].name.c_str(), + to_str(info[i].elements).c_str(),to_str(info[i].dimensions).c_str()); + } +} + +TEST(OnnxWrapperTest, onnx_model_can_be_inspected) +{ + OnnxWrapper wrapper(simple_model, OnnxWrapper::Optimize::DISABLE); + dump_info("inputs", wrapper.inputs()); + dump_info("outputs", wrapper.outputs()); + ASSERT_EQ(wrapper.inputs().size(), 3); + ASSERT_EQ(wrapper.outputs().size(), 1); + //------------------------------------------------------------------------- + EXPECT_EQ( wrapper.inputs()[0].name, "query_tensor"); + EXPECT_EQ(to_str(wrapper.inputs()[0].dimensions), "[1][4]"); + EXPECT_EQ(to_str(wrapper.inputs()[0].elements), "float"); + //------------------------------------------------------------------------- + EXPECT_EQ( wrapper.inputs()[1].name, "attribute_tensor"); + EXPECT_EQ(to_str(wrapper.inputs()[1].dimensions), "[4][1]"); + EXPECT_EQ(to_str(wrapper.inputs()[1].elements), "float"); + //------------------------------------------------------------------------- + EXPECT_EQ( wrapper.inputs()[2].name, "bias_tensor"); + EXPECT_EQ(to_str(wrapper.inputs()[2].dimensions), "[1][1]"); + EXPECT_EQ(to_str(wrapper.inputs()[2].elements), "float"); + //------------------------------------------------------------------------- + EXPECT_EQ( wrapper.outputs()[0].name, "output"); + EXPECT_EQ(to_str(wrapper.outputs()[0].dimensions), "[1][1]"); + EXPECT_EQ(to_str(wrapper.outputs()[0].elements), "float"); +} + +TEST(OnnxWrapperTest, onnx_model_can_be_evaluated) +{ + OnnxWrapper wrapper(simple_model, OnnxWrapper::Optimize::ENABLE); + + ValueType query_type = ValueType::from_spec("tensor<float>(a[1],b[4])"); + std::vector<float> query_values({1.0, 2.0, 3.0, 4.0}); + DenseTensorView query(query_type, TypedCells(query_values)); + EXPECT_TRUE(wrapper.inputs()[0].is_compatible(query_type)); + EXPECT_FALSE(wrapper.inputs()[1].is_compatible(query_type)); + EXPECT_FALSE(wrapper.inputs()[2].is_compatible(query_type)); + + ValueType attribute_type = ValueType::from_spec("tensor<float>(a[4],b[1])"); + std::vector<float> attribute_values({5.0, 6.0, 7.0, 8.0}); + DenseTensorView attribute(attribute_type, TypedCells(attribute_values)); + EXPECT_FALSE(wrapper.inputs()[0].is_compatible(attribute_type)); + EXPECT_TRUE(wrapper.inputs()[1].is_compatible(attribute_type)); + EXPECT_FALSE(wrapper.inputs()[2].is_compatible(attribute_type)); + + ValueType bias_type = ValueType::from_spec("tensor<float>(a[1],b[1])"); + std::vector<float> bias_values({9.0}); + DenseTensorView bias(bias_type, TypedCells(bias_values)); + EXPECT_FALSE(wrapper.inputs()[0].is_compatible(bias_type)); + EXPECT_FALSE(wrapper.inputs()[1].is_compatible(bias_type)); + EXPECT_TRUE(wrapper.inputs()[2].is_compatible(bias_type)); + + MutableDenseTensorView output(wrapper.outputs()[0].make_compatible_type()); + EXPECT_EQ(output.fast_type().to_spec(), "tensor<float>(d0[1],d1[1])"); + + OnnxWrapper::Params params; + params.bind(0, query); + params.bind(1, attribute); + params.bind(2, bias); + auto result = wrapper.eval(params); + + EXPECT_EQ(result.num_values(), 1); + result.get(0, output); + auto cells = output.cellsRef(); + EXPECT_EQ(cells.type, ValueType::CellType::FLOAT); + EXPECT_EQ(cells.size, 1); + EXPECT_EQ(cells.get(0), 79.0); +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/eval/src/vespa/eval/CMakeLists.txt b/eval/src/vespa/eval/CMakeLists.txt index c28643e605e..04f151f7ced 100644 --- a/eval/src/vespa/eval/CMakeLists.txt +++ b/eval/src/vespa/eval/CMakeLists.txt @@ -12,6 +12,7 @@ vespa_add_library(vespaeval $<TARGET_OBJECTS:eval_tensor_sparse> INSTALL lib64 DEPENDS + onnxruntime ${VESPA_LLVM_LIB} ) diff --git a/eval/src/vespa/eval/tensor/dense/CMakeLists.txt b/eval/src/vespa/eval/tensor/dense/CMakeLists.txt index c4b8138148c..b4e849a1dde 100644 --- a/eval/src/vespa/eval/tensor/dense/CMakeLists.txt +++ b/eval/src/vespa/eval/tensor/dense/CMakeLists.txt @@ -30,6 +30,7 @@ vespa_add_library(eval_tensor_dense OBJECT dense_xw_product_function.cpp index_lookup_table.cpp mutable_dense_tensor_view.cpp + onnx_wrapper.cpp typed_cells.cpp typed_dense_tensor_builder.cpp vector_from_doubles_function.cpp diff --git a/eval/src/vespa/eval/tensor/dense/onnx_wrapper.cpp b/eval/src/vespa/eval/tensor/dense/onnx_wrapper.cpp new file mode 100644 index 00000000000..fa0379473c9 --- /dev/null +++ b/eval/src/vespa/eval/tensor/dense/onnx_wrapper.cpp @@ -0,0 +1,233 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "onnx_wrapper.h" +#include <vespa/eval/eval/value_type.h> +#include "dense_tensor_view.h" +#include "mutable_dense_tensor_view.h" +#include <vespa/vespalib/util/arrayref.h> +#include <vespa/vespalib/util/stringfmt.h> +#include <assert.h> +#include <cmath> +#include <stdlib.h> +#include <stdio.h> + +using vespalib::eval::ValueType; +using vespalib::make_string_short::fmt; + +namespace vespalib::tensor { + +namespace { + +ValueType::CellType as_cell_type(OnnxWrapper::TensorInfo::ElementType type) { + if (type == OnnxWrapper::TensorInfo::ElementType::FLOAT) { + return ValueType::CellType::FLOAT; + } + if (type == OnnxWrapper::TensorInfo::ElementType::DOUBLE) { + return ValueType::CellType::DOUBLE; + } + abort(); +} + +auto convert_optimize(OnnxWrapper::Optimize optimize) { + if (optimize == OnnxWrapper::Optimize::ENABLE) { + return ORT_ENABLE_ALL; + } else { + assert(optimize == OnnxWrapper::Optimize::DISABLE); + return ORT_DISABLE_ALL; + } +} + +class OnnxString { +private: + static Ort::AllocatorWithDefaultOptions _alloc; + char *_str; + void cleanup() { + if (_str != nullptr) { + _alloc.Free(_str); + _str = nullptr; + } + } + OnnxString(char *str) : _str(str) {} +public: + OnnxString(const OnnxString &rhs) = delete; + OnnxString &operator=(const OnnxString &rhs) = delete; + OnnxString(OnnxString &&rhs) : _str(rhs._str) { + rhs._str = nullptr; + } + OnnxString &operator=(OnnxString &&rhs) { + cleanup(); + _str = rhs._str; + rhs._str = nullptr; + return *this; + } + const char *get() const { return _str; } + ~OnnxString() { cleanup(); } + static OnnxString get_input_name(const Ort::Session &session, size_t idx) { + return OnnxString(session.GetInputName(idx, _alloc)); + } + static OnnxString get_output_name(const Ort::Session &session, size_t idx) { + return OnnxString(session.GetOutputName(idx, _alloc)); + } +}; +Ort::AllocatorWithDefaultOptions OnnxString::_alloc; + +std::vector<size_t> make_dimensions(const std::vector<int64_t> &shape) { + std::vector<size_t> result; + for (int64_t size: shape) { + result.push_back(std::max(size, 0L)); + } + return result; +} + +OnnxWrapper::TensorInfo::ElementType make_element_type(ONNXTensorElementDataType element_type) { + if (element_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT) { + return OnnxWrapper::TensorInfo::ElementType::FLOAT; + } else if (element_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE) { + return OnnxWrapper::TensorInfo::ElementType::DOUBLE; + } else { + return OnnxWrapper::TensorInfo::ElementType::UNKNOWN; + } +} + +OnnxWrapper::TensorInfo make_tensor_info(const OnnxString &name, const Ort::TypeInfo &type_info) { + auto tensor_info = type_info.GetTensorTypeAndShapeInfo(); + auto shape = tensor_info.GetShape(); + auto element_type = tensor_info.GetElementType(); + return OnnxWrapper::TensorInfo{vespalib::string(name.get()), make_dimensions(shape), make_element_type(element_type)}; +} + +} + +bool +OnnxWrapper::TensorInfo::is_compatible(const eval::ValueType &type) const +{ + if ((elements == ElementType::UNKNOWN) || dimensions.empty()) { + return false; + } + if (type.cell_type() != as_cell_type(elements)) { + return false; + } + if (type.dimensions().size() != dimensions.size()) { + return false; + } + for (size_t i = 0; i < dimensions.size(); ++i) { + if (type.dimensions()[i].size != dimensions[i]) { + return false; + } + } + return true; +} + +eval::ValueType +OnnxWrapper::TensorInfo::make_compatible_type() const +{ + if ((elements == ElementType::UNKNOWN) || dimensions.empty()) { + return ValueType::error_type(); + } + std::vector<ValueType::Dimension> dim_list; + for (size_t dim_size: dimensions) { + if ((dim_size == 0) || (dim_list.size() > 9)) { + return ValueType::error_type(); + } + dim_list.emplace_back(fmt("d%zu", dim_list.size()), dim_size); + } + return ValueType::tensor_type(std::move(dim_list), as_cell_type(elements)); +} + +OnnxWrapper::TensorInfo::~TensorInfo() = default; + +OnnxWrapper::Shared::Shared() + : _env(ORT_LOGGING_LEVEL_WARNING, "vespa-onnx-wrapper") +{ +} + +void +OnnxWrapper::Params::bind(size_t idx, const DenseTensorView &src) +{ + assert(idx == values.size()); + std::vector<int64_t> dim_sizes; + for (const auto &dim: src.fast_type().dimensions()) { + dim_sizes.push_back(dim.size); + } + auto memory_info = Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault); + if (src.fast_type().cell_type() == ValueType::CellType::FLOAT) { + // NB: create requires non-const input + auto cells = unconstify(src.cellsRef().typify<float>()); + values.push_back(Ort::Value::CreateTensor<float>(memory_info, cells.begin(), cells.size(), dim_sizes.data(), dim_sizes.size())); + } else if (src.fast_type().cell_type() == ValueType::CellType::DOUBLE) { + // NB: create requires non-const input + auto cells = unconstify(src.cellsRef().typify<double>()); + values.push_back(Ort::Value::CreateTensor<double>(memory_info, cells.begin(), cells.size(), dim_sizes.data(), dim_sizes.size())); + } +} + +void +OnnxWrapper::Result::get(size_t idx, MutableDenseTensorView &dst) +{ + assert(values[idx].IsTensor()); + auto meta = values[idx].GetTensorTypeAndShapeInfo(); + if (dst.fast_type().cell_type() == ValueType::CellType::FLOAT) { + assert(meta.GetElementType() == ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT); + ConstArrayRef<float> cells(values[idx].GetTensorMutableData<float>(), meta.GetElementCount()); + dst.setCells(TypedCells(cells)); + } else if (dst.fast_type().cell_type() == ValueType::CellType::DOUBLE) { + assert(meta.GetElementType() == ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE); + ConstArrayRef<double> cells(values[idx].GetTensorMutableData<double>(), meta.GetElementCount()); + dst.setCells(TypedCells(cells)); + } +} + +OnnxWrapper::Shared & +OnnxWrapper::Shared::get() { + static Shared shared; + return shared; +} + +void +OnnxWrapper::extract_meta_data() +{ + Ort::AllocatorWithDefaultOptions allocator; + size_t num_inputs = _session.GetInputCount(); + for (size_t i = 0; i < num_inputs; ++i) { + _inputs.push_back(make_tensor_info(OnnxString::get_input_name(_session, i), _session.GetInputTypeInfo(i))); + } + size_t num_outputs = _session.GetOutputCount(); + for (size_t i = 0; i < num_outputs; ++i) { + _outputs.push_back(make_tensor_info(OnnxString::get_output_name(_session, i), _session.GetOutputTypeInfo(i))); + } + for (const auto &input: _inputs) { + _input_name_refs.push_back(input.name.c_str()); + } + for (const auto &output: _outputs) { + _output_name_refs.push_back(output.name.c_str()); + } +} + +OnnxWrapper::OnnxWrapper(const vespalib::string &model_file, Optimize optimize) + : _shared(Shared::get()), + _options(), + _session(nullptr), + _inputs(), + _outputs(), + _input_name_refs(), + _output_name_refs() +{ + _options.SetIntraOpNumThreads(1); + _options.SetInterOpNumThreads(1); + _options.SetGraphOptimizationLevel(convert_optimize(optimize)); + _session = Ort::Session(_shared.env(), model_file.c_str(), _options); + extract_meta_data(); +} + +OnnxWrapper::~OnnxWrapper() = default; + +OnnxWrapper::Result +OnnxWrapper::eval(const Params ¶ms) +{ + assert(params.values.size() == _inputs.size()); + Ort::RunOptions run_opts(nullptr); + return Result(_session.Run(run_opts, _input_name_refs.data(), params.values.data(), _inputs.size(), + _output_name_refs.data(), _outputs.size())); +} + +} diff --git a/eval/src/vespa/eval/tensor/dense/onnx_wrapper.h b/eval/src/vespa/eval/tensor/dense/onnx_wrapper.h new file mode 100644 index 00000000000..67a64f2d318 --- /dev/null +++ b/eval/src/vespa/eval/tensor/dense/onnx_wrapper.h @@ -0,0 +1,84 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <onnxruntime/onnxruntime_cxx_api.h> +#include <vespa/vespalib/stllike/string.h> +#include <vespa/eval/eval/value_type.h> +#include <vector> + +namespace vespalib::tensor { + +class DenseTensorView; +class MutableDenseTensorView; + +/** + * Wrapper around an ONNX model handeled by onnxruntime. + **/ +class OnnxWrapper { +public: + // model optimization + enum class Optimize { ENABLE, DISABLE }; + + // information about a single input or output tensor + struct TensorInfo { + enum class ElementType { FLOAT, DOUBLE, UNKNOWN }; + vespalib::string name; + std::vector<size_t> dimensions; + ElementType elements; + bool is_compatible(const eval::ValueType &type) const; + eval::ValueType make_compatible_type() const; + ~TensorInfo(); + }; + + // used to build model parameters + class Params { + friend class OnnxWrapper; + private: + std::vector<Ort::Value> values; + public: + Params() : values() {} + void bind(size_t idx, const DenseTensorView &src); + }; + + // used to inspect model results + class Result { + friend class OnnxWrapper; + private: + std::vector<Ort::Value> values; + Result(std::vector<Ort::Value> values_in) : values(std::move(values_in)) {} + public: + size_t num_values() const { return values.size(); } + void get(size_t idx, MutableDenseTensorView &dst); + }; + +private: + // common stuff shared between model sessions + class Shared { + private: + Ort::Env _env; + Shared(); + public: + static Shared &get(); + Ort::Env &env() { return _env; } + }; + + Shared &_shared; + Ort::SessionOptions _options; + Ort::Session _session; + std::vector<TensorInfo> _inputs; + std::vector<TensorInfo> _outputs; + std::vector<const char *> _input_name_refs; + std::vector<const char *> _output_name_refs; + + void extract_meta_data(); + +public: + OnnxWrapper(const vespalib::string &model_file, Optimize optimize); + ~OnnxWrapper(); + const std::vector<TensorInfo> &inputs() const { return _inputs; } + const std::vector<TensorInfo> &outputs() const { return _outputs; } + Result eval(const Params ¶ms); // NB: Run requires non-const session +}; + +} diff --git a/model-integration/CMakeLists.txt b/model-integration/CMakeLists.txt index 26d5b3d1bbc..f8aa1c552a6 100644 --- a/model-integration/CMakeLists.txt +++ b/model-integration/CMakeLists.txt @@ -1,4 +1,6 @@ # Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. install_fat_java_artifact(model-integration) +vespa_install_script(src/main/python/vespa-convert-tf2onnx.py vespa-convert-tf2onnx bin) + install(FILES src/main/config/model-integration.xml DESTINATION conf/configserver-app)
\ No newline at end of file diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/tensorflow/TensorFlowImporter.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/tensorflow/TensorFlowImporter.java index 96ea58edc61..34b9c847a12 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/tensorflow/TensorFlowImporter.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/tensorflow/TensorFlowImporter.java @@ -98,7 +98,7 @@ public class TensorFlowImporter extends ModelImporter { private Pair<Integer, String> convertToOnnx(String savedModel, String output, int opset) throws IOException { ProcessExecuter executer = new ProcessExecuter(); - String job = "python3 -m tf2onnx.convert --saved-model " + savedModel + " --output " + output + " --opset " + opset; + String job = "vespa-convert-tf2onnx --saved-model " + savedModel + " --output " + output + " --opset " + opset; return executer.exec(job); } diff --git a/model-integration/src/main/python/vespa-convert-tf2onnx.py b/model-integration/src/main/python/vespa-convert-tf2onnx.py new file mode 100755 index 00000000000..e34610f6eb4 --- /dev/null +++ b/model-integration/src/main/python/vespa-convert-tf2onnx.py @@ -0,0 +1,60 @@ +#! /usr/bin/env python3 + +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +import sys +import onnx + +from tf2onnx import convert +from tensorflow.python.tools import saved_model_utils + + +def find(nodes, test): + return next((x for x in nodes if test(x)), None) + + +def make_alias(onnx_model, alias, output_name): + output = find(onnx_model.graph.output, lambda node: node.name == output_name) + if output is None: + print("Could not find output '{}' to alias from '{}'".format(output_name, alias)) + return + output_tensor = onnx.helper.make_empty_tensor_value_info("") + output_tensor.CopyFrom(output) + output_tensor.name = alias + onnx_model.graph.output.append(output_tensor) + onnx_model.graph.node.append(onnx.helper.make_node("Identity", [output_name], [alias])) + + +def verify_outputs(args, onnx_model): + tag_sets = saved_model_utils.get_saved_model_tag_sets(args.saved_model) + for tag_set in sorted(tag_sets): + tag_set = ','.join(tag_set) + meta_graph_def = saved_model_utils.get_meta_graph_def(args.saved_model, tag_set) + signature_def_map = meta_graph_def.signature_def + for signature_def_key in sorted(signature_def_map.keys()): + outputs_tensor_info = signature_def_map[signature_def_key].outputs + for output_key, output_tensor in sorted(outputs_tensor_info.items()): + output_key_exists_as_output = find(onnx_model.graph.output, lambda node: node.name == output_key) + if output_key_exists_as_output: + continue + make_alias(onnx_model, output_key, output_tensor.name) + + output_names = [ "'{}'".format(o.name) for o in onnx_model.graph.output ] + print("Outputs in model: {}".format(", ".join(output_names))) + + +def main(): + convert.main() + + args = convert.get_args() + onnx_model = onnx.load(args.output) + verify_outputs(args, onnx_model) + onnx.save(onnx_model, args.output) + + +if __name__ == "__main__": + main() + + + + diff --git a/persistence/src/tests/dummyimpl/dummyimpltest.cpp b/persistence/src/tests/dummyimpl/dummyimpltest.cpp index 7bc27266020..6cf0b53766f 100644 --- a/persistence/src/tests/dummyimpl/dummyimpltest.cpp +++ b/persistence/src/tests/dummyimpl/dummyimpltest.cpp @@ -14,9 +14,9 @@ namespace { struct DummyPersistenceFactory : public ConformanceTest::PersistenceFactory { using Repo = document::DocumentTypeRepo; - std::unique_ptr<PersistenceProvider> + PersistenceProvider::UP getPersistenceImplementation(const std::shared_ptr<const Repo>& repo, const Repo::DocumenttypesConfig&) override { - return std::make_unique<dummy::DummyPersistence>(repo, 4); + return PersistenceProvider::UP(new dummy::DummyPersistence(repo, 4)); } bool supportsActiveState() const override { return true; } diff --git a/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp b/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp index 1871d8943d8..3d15d09814f 100644 --- a/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp +++ b/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp @@ -27,15 +27,13 @@ using storage::spi::test::makeSpiBucket; namespace storage::spi { -using PersistenceProviderUP = std::unique_ptr<PersistenceProvider>; - namespace { LoadType defaultLoadType(0, "default"); -std::unique_ptr<PersistenceProvider> getSpi(ConformanceTest::PersistenceFactory &factory, +PersistenceProvider::UP getSpi(ConformanceTest::PersistenceFactory &factory, const document::TestDocMan &testDocMan) { - PersistenceProviderUP result(factory.getPersistenceImplementation( + PersistenceProvider::UP result(factory.getPersistenceImplementation( testDocMan.getTypeRepoSP(), *testDocMan.getTypeConfig())); EXPECT_TRUE(!result->initialize().hasError()); EXPECT_TRUE(!result->getPartitionStates().hasError()); @@ -55,15 +53,15 @@ createIterator(PersistenceProvider& spi, IncludedVersions versions = NEWEST_DOCUMENT_ONLY, int fields = ALL_FIELDS) { - document::FieldSet::SP fieldSet; + document::FieldSet::UP fieldSet; if (fields & ALL_FIELDS) { - fieldSet = std::make_shared<document::AllFields>(); + fieldSet.reset(new document::AllFields()); } else { - fieldSet = std::make_shared<document::DocIdOnly>(); + fieldSet.reset(new document::DocIdOnly()); } Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); - return spi.createIterator(b, std::move(fieldSet), sel, versions, context); + return spi.createIterator(b, *fieldSet, sel, versions, context); } Selection @@ -208,7 +206,7 @@ iterateBucket(PersistenceProvider& spi, Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); CreateIteratorResult iter = spi.createIterator( bucket, - std::make_shared<document::AllFields>(), + document::AllFields(), sel, versions, context); @@ -332,7 +330,7 @@ TEST_F(ConformanceTest, testBasics) { document::TestDocMan testDocMan; _factory->clear(); - PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); + PersistenceProvider::UP spi(getSpi(*_factory, testDocMan)); Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); Bucket bucket(makeSpiBucket(BucketId(8, 0x01))); @@ -360,7 +358,7 @@ TEST_F(ConformanceTest, testBasics) CreateIteratorResult iter = spi->createIterator( bucket, - std::make_shared<document::AllFields>(), + document::AllFields(), sel, includeRemoves ? NEWEST_DOCUMENT_OR_REMOVE : NEWEST_DOCUMENT_ONLY, @@ -417,7 +415,7 @@ TEST_F(ConformanceTest, testListBuckets) //TODO: enable when supported by provider in storage document::TestDocMan testDocMan; _factory->clear(); - PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); + PersistenceProvider::UP spi(getSpi(*_factory, testDocMan)); PartitionId partId(0); BucketId bucketId1(8, 0x01); @@ -462,7 +460,7 @@ TEST_F(ConformanceTest, testBucketInfo) { document::TestDocMan testDocMan; _factory->clear(); - PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); + PersistenceProvider::UP spi(getSpi(*_factory, testDocMan)); Bucket bucket(makeSpiBucket(BucketId(8, 0x01))); @@ -519,7 +517,7 @@ TEST_F(ConformanceTest, testOrderIndependentBucketInfo) { document::TestDocMan testDocMan; _factory->clear(); - PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); + PersistenceProvider::UP spi(getSpi(*_factory, testDocMan)); Bucket bucket(makeSpiBucket(BucketId(8, 0x01))); @@ -563,7 +561,7 @@ TEST_F(ConformanceTest, testPut) { document::TestDocMan testDocMan; _factory->clear(); - PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); + PersistenceProvider::UP spi(getSpi(*_factory, testDocMan)); Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); Bucket bucket(makeSpiBucket(BucketId(8, 0x01))); @@ -588,7 +586,7 @@ TEST_F(ConformanceTest, testPutNewDocumentVersion) { document::TestDocMan testDocMan; _factory->clear(); - PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); + PersistenceProvider::UP spi(getSpi(*_factory, testDocMan)); Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); Bucket bucket(makeSpiBucket(BucketId(8, 0x01))); @@ -639,7 +637,7 @@ TEST_F(ConformanceTest, testPutOlderDocumentVersion) { document::TestDocMan testDocMan; _factory->clear(); - PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); + PersistenceProvider::UP spi(getSpi(*_factory, testDocMan)); Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); Bucket bucket(makeSpiBucket(BucketId(8, 0x01))); @@ -683,7 +681,7 @@ TEST_F(ConformanceTest, testPutDuplicate) { document::TestDocMan testDocMan; _factory->clear(); - PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); + PersistenceProvider::UP spi(getSpi(*_factory, testDocMan)); Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); Bucket bucket(makeSpiBucket(BucketId(8, 0x01))); @@ -715,7 +713,7 @@ TEST_F(ConformanceTest, testRemove) { document::TestDocMan testDocMan; _factory->clear(); - PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); + PersistenceProvider::UP spi(getSpi(*_factory, testDocMan)); Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); Bucket bucket(makeSpiBucket(BucketId(8, 0x01))); @@ -808,7 +806,7 @@ TEST_F(ConformanceTest, testRemoveMerge) { document::TestDocMan testDocMan; _factory->clear(); - PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); + PersistenceProvider::UP spi(getSpi(*_factory, testDocMan)); Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); Bucket bucket(makeSpiBucket(BucketId(8, 0x01))); @@ -902,7 +900,7 @@ TEST_F(ConformanceTest, testUpdate) { document::TestDocMan testDocMan; _factory->clear(); - PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); + PersistenceProvider::UP spi(getSpi(*_factory, testDocMan)); Document::SP doc1 = testDocMan.createRandomDocumentAtLocation(0x01, 1); Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); @@ -1001,7 +999,7 @@ TEST_F(ConformanceTest, testGet) { document::TestDocMan testDocMan; _factory->clear(); - PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); + PersistenceProvider::UP spi(getSpi(*_factory, testDocMan)); Document::SP doc1 = testDocMan.createRandomDocumentAtLocation(0x01, 1); Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); @@ -1044,7 +1042,7 @@ TEST_F(ConformanceTest, testIterateCreateIterator) { document::TestDocMan testDocMan; _factory->clear(); - PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); + PersistenceProvider::UP spi(getSpi(*_factory, testDocMan)); Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); Bucket b(makeSpiBucket(BucketId(8, 0x1))); spi->createBucket(b, context); @@ -1063,7 +1061,7 @@ TEST_F(ConformanceTest, testIterateWithUnknownId) { document::TestDocMan testDocMan; _factory->clear(); - PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); + PersistenceProvider::UP spi(getSpi(*_factory, testDocMan)); Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); Bucket b(makeSpiBucket(BucketId(8, 0x1))); spi->createBucket(b, context); @@ -1077,7 +1075,7 @@ TEST_F(ConformanceTest, testIterateDestroyIterator) { document::TestDocMan testDocMan; _factory->clear(); - PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); + PersistenceProvider::UP spi(getSpi(*_factory, testDocMan)); Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); Bucket b(makeSpiBucket(BucketId(8, 0x1))); spi->createBucket(b, context); @@ -1109,7 +1107,7 @@ TEST_F(ConformanceTest, testIterateAllDocs) { document::TestDocMan testDocMan; _factory->clear(); - PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); + PersistenceProvider::UP spi(getSpi(*_factory, testDocMan)); Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); Bucket b(makeSpiBucket(BucketId(8, 0x1))); spi->createBucket(b, context); @@ -1127,7 +1125,7 @@ TEST_F(ConformanceTest, testIterateAllDocsNewestVersionOnly) { document::TestDocMan testDocMan; _factory->clear(); - PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); + PersistenceProvider::UP spi(getSpi(*_factory, testDocMan)); Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); Bucket b(makeSpiBucket(BucketId(8, 0x1))); spi->createBucket(b, context); @@ -1155,7 +1153,7 @@ TEST_F(ConformanceTest, testIterateChunked) { document::TestDocMan testDocMan; _factory->clear(); - PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); + PersistenceProvider::UP spi(getSpi(*_factory, testDocMan)); Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); Bucket b(makeSpiBucket(BucketId(8, 0x1))); spi->createBucket(b, context); @@ -1175,7 +1173,7 @@ TEST_F(ConformanceTest, testMaxByteSize) { document::TestDocMan testDocMan; _factory->clear(); - PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); + PersistenceProvider::UP spi(getSpi(*_factory, testDocMan)); Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); Bucket b(makeSpiBucket(BucketId(8, 0x1))); spi->createBucket(b, context); @@ -1201,7 +1199,7 @@ TEST_F(ConformanceTest, testIterateMatchTimestampRange) { document::TestDocMan testDocMan; _factory->clear(); - PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); + PersistenceProvider::UP spi(getSpi(*_factory, testDocMan)); Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); Bucket b(makeSpiBucket(BucketId(8, 0x1))); spi->createBucket(b, context); @@ -1239,7 +1237,7 @@ TEST_F(ConformanceTest, testIterateExplicitTimestampSubset) { document::TestDocMan testDocMan; _factory->clear(); - PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); + PersistenceProvider::UP spi(getSpi(*_factory, testDocMan)); Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); Bucket b(makeSpiBucket(BucketId(8, 0x1))); spi->createBucket(b, context); @@ -1288,7 +1286,7 @@ TEST_F(ConformanceTest, testIterateRemoves) { document::TestDocMan testDocMan; _factory->clear(); - PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); + PersistenceProvider::UP spi(getSpi(*_factory, testDocMan)); Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); Bucket b(makeSpiBucket(BucketId(8, 0x1))); spi->createBucket(b, context); @@ -1323,7 +1321,8 @@ TEST_F(ConformanceTest, testIterateRemoves) { Selection sel(createSelection("")); - CreateIteratorResult iter(createIterator(*spi, b, sel, NEWEST_DOCUMENT_OR_REMOVE)); + CreateIteratorResult iter( + createIterator(*spi, b, sel, NEWEST_DOCUMENT_OR_REMOVE)); std::vector<Chunk> chunks = doIterate(*spi, iter.getIteratorId(), 4096); std::vector<DocEntry::UP> entries = getEntriesFromChunks(chunks); @@ -1338,7 +1337,7 @@ TEST_F(ConformanceTest, testIterateMatchSelection) { document::TestDocMan testDocMan; _factory->clear(); - PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); + PersistenceProvider::UP spi(getSpi(*_factory, testDocMan)); Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); Bucket b(makeSpiBucket(BucketId(8, 0x1))); spi->createBucket(b, context); @@ -1352,11 +1351,15 @@ TEST_F(ConformanceTest, testIterateMatchSelection) spi->put(b, Timestamp(1000 + i), doc, context); if ((i % 3) == 0) { - docsToVisit.push_back(DocAndTimestamp(doc, Timestamp(1000 + i))); + docsToVisit.push_back( + DocAndTimestamp(doc, Timestamp(1000 + i))); } } - CreateIteratorResult iter(createIterator(*spi, b, createSelection("testdoctype1.headerval % 3 == 0"))); + CreateIteratorResult iter( + createIterator(*spi, + b, + createSelection("testdoctype1.headerval % 3 == 0"))); std::vector<Chunk> chunks = doIterate(*spi, iter.getIteratorId(), 2048 * 1024); verifyDocs(docsToVisit, chunks); @@ -1368,7 +1371,7 @@ TEST_F(ConformanceTest, testIterationRequiringDocumentIdOnlyMatching) { document::TestDocMan testDocMan; _factory->clear(); - PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); + PersistenceProvider::UP spi(getSpi(*_factory, testDocMan)); Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); Bucket b(makeSpiBucket(BucketId(8, 0x1))); spi->createBucket(b, context); @@ -1378,11 +1381,13 @@ TEST_F(ConformanceTest, testIterationRequiringDocumentIdOnlyMatching) // Document does not already exist, remove should create a // remove entry for it regardless. - EXPECT_TRUE(!spi->remove(b, Timestamp(2000), removedId, context).wasFound()); + EXPECT_TRUE( + !spi->remove(b, Timestamp(2000), removedId, context).wasFound()); Selection sel(createSelection("id == '" + removedId.toString() + "'")); - CreateIteratorResult iter(createIterator(*spi, b, sel, NEWEST_DOCUMENT_OR_REMOVE)); + CreateIteratorResult iter( + createIterator(*spi, b, sel, NEWEST_DOCUMENT_OR_REMOVE)); EXPECT_TRUE(iter.getErrorCode() == Result::ErrorType::NONE); std::vector<Chunk> chunks = doIterate(*spi, iter.getIteratorId(), 4096); @@ -1398,14 +1403,16 @@ TEST_F(ConformanceTest, testIterateBadDocumentSelection) { document::TestDocMan testDocMan; _factory->clear(); - PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); + PersistenceProvider::UP spi(getSpi(*_factory, testDocMan)); Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); Bucket b(makeSpiBucket(BucketId(8, 0x1))); spi->createBucket(b, context); { - CreateIteratorResult iter(createIterator(*spi, b, createSelection("the muppet show"))); + CreateIteratorResult iter( + createIterator(*spi, b, createSelection("the muppet show"))); if (iter.getErrorCode() == Result::ErrorType::NONE) { - IterateResult result(spi->iterate(iter.getIteratorId(), 4096, context)); + IterateResult result( + spi->iterate(iter.getIteratorId(), 4096, context)); EXPECT_EQ(Result::ErrorType::NONE, result.getErrorCode()); EXPECT_EQ(size_t(0), result.getEntries().size()); EXPECT_EQ(true, result.isCompleted()); @@ -1415,9 +1422,14 @@ TEST_F(ConformanceTest, testIterateBadDocumentSelection) } } { - CreateIteratorResult iter(createIterator(*spi, b, createSelection("unknownddoctype.something=thatthing"))); + CreateIteratorResult iter( + createIterator(*spi, + b, + createSelection( + "unknownddoctype.something=thatthing"))); if (iter.getErrorCode() == Result::ErrorType::NONE) { - IterateResult result(spi->iterate(iter.getIteratorId(), 4096, context)); + IterateResult result(spi->iterate( + iter.getIteratorId(), 4096, context)); EXPECT_EQ(Result::ErrorType::NONE, result.getErrorCode()); EXPECT_EQ(size_t(0), result.getEntries().size()); EXPECT_EQ(true, result.isCompleted()); @@ -1432,7 +1444,7 @@ TEST_F(ConformanceTest, testIterateAlreadyCompleted) { document::TestDocMan testDocMan; _factory->clear(); - PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); + PersistenceProvider::UP spi(getSpi(*_factory, testDocMan)); Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); Bucket b(makeSpiBucket(BucketId(8, 0x1))); spi->createBucket(b, context); @@ -1456,7 +1468,7 @@ TEST_F(ConformanceTest, testIterateEmptyBucket) { document::TestDocMan testDocMan; _factory->clear(); - PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); + PersistenceProvider::UP spi(getSpi(*_factory, testDocMan)); Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); Bucket b(makeSpiBucket(BucketId(8, 0x1))); spi->createBucket(b, context); @@ -1476,7 +1488,7 @@ TEST_F(ConformanceTest, testDeleteBucket) { document::TestDocMan testDocMan; _factory->clear(); - PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); + PersistenceProvider::UP spi(getSpi(*_factory, testDocMan)); Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); Document::SP doc1 = testDocMan.createRandomDocumentAtLocation(0x01, 1); @@ -1486,28 +1498,28 @@ TEST_F(ConformanceTest, testDeleteBucket) spi->put(bucket, Timestamp(3), doc1, context); spi->deleteBucket(bucket, context); - testDeleteBucketPostCondition(*spi, bucket, *doc1); + testDeleteBucketPostCondition(spi, bucket, *doc1); if (_factory->hasPersistence()) { spi.reset(); document::TestDocMan testDocMan2; spi = getSpi(*_factory, testDocMan2); - testDeleteBucketPostCondition(*spi, bucket, *doc1); + testDeleteBucketPostCondition(spi, bucket, *doc1); } } void ConformanceTest:: -testDeleteBucketPostCondition(const PersistenceProvider &spi, +testDeleteBucketPostCondition(const PersistenceProvider::UP &spi, const Bucket &bucket, const Document &doc1) { Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); { - GetResult result = spi.get(bucket, - document::AllFields(), - doc1.getId(), - context); + GetResult result = spi->get(bucket, + document::AllFields(), + doc1.getId(), + context); EXPECT_EQ(Result::ErrorType::NONE, result.getErrorCode()); EXPECT_EQ(Timestamp(0), result.getTimestamp()); @@ -1519,7 +1531,7 @@ TEST_F(ConformanceTest, testSplitNormalCase) { document::TestDocMan testDocMan; _factory->clear(); - PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); + PersistenceProvider::UP spi(getSpi(*_factory, testDocMan)); Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); Bucket bucketA(makeSpiBucket(BucketId(3, 0x02))); @@ -1540,41 +1552,51 @@ TEST_F(ConformanceTest, testSplitNormalCase) } spi->split(bucketC, bucketA, bucketB, context); - testSplitNormalCasePostCondition(*spi, bucketA, bucketB, bucketC, testDocMan); + testSplitNormalCasePostCondition(spi, bucketA, bucketB, bucketC, + testDocMan); if (_factory->hasPersistence()) { spi.reset(); document::TestDocMan testDocMan2; spi = getSpi(*_factory, testDocMan2); - testSplitNormalCasePostCondition(*spi, bucketA, bucketB, bucketC, testDocMan2); + testSplitNormalCasePostCondition(spi, bucketA, bucketB, bucketC, + testDocMan2); } } void ConformanceTest:: -testSplitNormalCasePostCondition(const PersistenceProvider &spi, +testSplitNormalCasePostCondition(const PersistenceProvider::UP &spi, const Bucket &bucketA, const Bucket &bucketB, const Bucket &bucketC, document::TestDocMan &testDocMan) { - EXPECT_EQ(10, (int)spi.getBucketInfo(bucketA).getBucketInfo().getDocumentCount()); - EXPECT_EQ(10, (int)spi.getBucketInfo(bucketB).getBucketInfo().getDocumentCount()); + EXPECT_EQ(10, (int)spi->getBucketInfo(bucketA).getBucketInfo(). + getDocumentCount()); + EXPECT_EQ(10, (int)spi->getBucketInfo(bucketB).getBucketInfo(). + getDocumentCount()); document::AllFields fs; Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); for (uint32_t i = 0; i < 10; ++i) { Document::UP doc1 = testDocMan.createRandomDocumentAtLocation(0x02, i); - EXPECT_TRUE(spi.get(bucketA, fs, doc1->getId(), context).hasDocument()); - EXPECT_TRUE(!spi.get(bucketC, fs, doc1->getId(), context).hasDocument()); - EXPECT_TRUE(!spi.get(bucketB, fs, doc1->getId(), context).hasDocument()); + EXPECT_TRUE( + spi->get(bucketA, fs, doc1->getId(), context).hasDocument()); + EXPECT_TRUE( + !spi->get(bucketC, fs, doc1->getId(), context).hasDocument()); + EXPECT_TRUE( + !spi->get(bucketB, fs, doc1->getId(), context).hasDocument()); } for (uint32_t i = 10; i < 20; ++i) { Document::UP doc1 = testDocMan.createRandomDocumentAtLocation(0x06, i); - EXPECT_TRUE(spi.get(bucketB, fs, doc1->getId(), context).hasDocument()); - EXPECT_TRUE(!spi.get(bucketA, fs, doc1->getId(), context).hasDocument()); - EXPECT_TRUE(!spi.get(bucketC, fs, doc1->getId(), context).hasDocument()); + EXPECT_TRUE( + spi->get(bucketB, fs, doc1->getId(), context).hasDocument()); + EXPECT_TRUE( + !spi->get(bucketA, fs, doc1->getId(), context).hasDocument()); + EXPECT_TRUE( + !spi->get(bucketC, fs, doc1->getId(), context).hasDocument()); } } @@ -1582,7 +1604,7 @@ TEST_F(ConformanceTest, testSplitTargetExists) { document::TestDocMan testDocMan; _factory->clear(); - PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); + PersistenceProvider::UP spi(getSpi(*_factory, testDocMan)); Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); Bucket bucketA(makeSpiBucket(BucketId(3, 0x02))); @@ -1616,27 +1638,29 @@ TEST_F(ConformanceTest, testSplitTargetExists) } spi->split(bucketC, bucketA, bucketB, context); - testSplitTargetExistsPostCondition(*spi, bucketA, bucketB, bucketC,testDocMan); + testSplitTargetExistsPostCondition(spi, bucketA, bucketB, bucketC, + testDocMan); if (_factory->hasPersistence()) { spi.reset(); document::TestDocMan testDocMan2; spi = getSpi(*_factory, testDocMan2); - testSplitTargetExistsPostCondition(*spi, bucketA, bucketB, bucketC,testDocMan2); + testSplitTargetExistsPostCondition(spi, bucketA, bucketB, bucketC, + testDocMan2); } } void ConformanceTest:: -testSplitTargetExistsPostCondition(const PersistenceProvider &spi, +testSplitTargetExistsPostCondition(const PersistenceProvider::UP &spi, const Bucket &bucketA, const Bucket &bucketB, const Bucket &bucketC, document::TestDocMan &testDocMan) { - EXPECT_EQ(10, (int)spi.getBucketInfo(bucketA).getBucketInfo(). + EXPECT_EQ(10, (int)spi->getBucketInfo(bucketA).getBucketInfo(). getDocumentCount()); - EXPECT_EQ(15, (int)spi.getBucketInfo(bucketB).getBucketInfo(). + EXPECT_EQ(15, (int)spi->getBucketInfo(bucketB).getBucketInfo(). getDocumentCount()); document::AllFields fs; @@ -1644,21 +1668,21 @@ testSplitTargetExistsPostCondition(const PersistenceProvider &spi, for (uint32_t i = 0; i < 10; ++i) { Document::UP doc1 = testDocMan.createRandomDocumentAtLocation(0x02, i); EXPECT_TRUE( - spi.get(bucketA, fs, doc1->getId(), context).hasDocument()); + spi->get(bucketA, fs, doc1->getId(), context).hasDocument()); EXPECT_TRUE( - !spi.get(bucketC, fs, doc1->getId(), context).hasDocument()); + !spi->get(bucketC, fs, doc1->getId(), context).hasDocument()); EXPECT_TRUE( - !spi.get(bucketB, fs, doc1->getId(), context).hasDocument()); + !spi->get(bucketB, fs, doc1->getId(), context).hasDocument()); } for (uint32_t i = 10; i < 25; ++i) { Document::UP doc1 = testDocMan.createRandomDocumentAtLocation(0x06, i); EXPECT_TRUE( - spi.get(bucketB, fs, doc1->getId(), context).hasDocument()); + spi->get(bucketB, fs, doc1->getId(), context).hasDocument()); EXPECT_TRUE( - !spi.get(bucketA, fs, doc1->getId(), context).hasDocument()); + !spi->get(bucketA, fs, doc1->getId(), context).hasDocument()); EXPECT_TRUE( - !spi.get(bucketC, fs, doc1->getId(), context).hasDocument()); + !spi->get(bucketC, fs, doc1->getId(), context).hasDocument()); } } @@ -1666,7 +1690,7 @@ TEST_F(ConformanceTest, testSplitSingleDocumentInSource) { document::TestDocMan testDocMan; _factory->clear(); - PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); + PersistenceProvider::UP spi(getSpi(*_factory, testDocMan)); Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); Bucket target1(makeSpiBucket(BucketId(3, 0x02))); @@ -1680,34 +1704,42 @@ TEST_F(ConformanceTest, testSplitSingleDocumentInSource) spi->put(source, Timestamp(1), doc, context); spi->split(source, target1, target2, context); - testSplitSingleDocumentInSourcePostCondition(*spi, source, target1, target2, testDocMan); + testSplitSingleDocumentInSourcePostCondition( + spi, source, target1, target2, testDocMan); if (_factory->hasPersistence()) { spi.reset(); document::TestDocMan testDocMan2; spi = getSpi(*_factory, testDocMan2); - testSplitSingleDocumentInSourcePostCondition(*spi, source, target1, target2, testDocMan2); + testSplitSingleDocumentInSourcePostCondition( + spi, source, target1, target2, testDocMan2); } } void ConformanceTest::testSplitSingleDocumentInSourcePostCondition( - const PersistenceProvider& spi, + const PersistenceProvider::UP& spi, const Bucket& source, const Bucket& target1, const Bucket& target2, document::TestDocMan& testDocMan) { - EXPECT_EQ(uint32_t(0), spi.getBucketInfo(source).getBucketInfo().getDocumentCount()); - EXPECT_EQ(uint32_t(0), spi.getBucketInfo(target1).getBucketInfo().getDocumentCount()); - EXPECT_EQ(uint32_t(1), spi.getBucketInfo(target2).getBucketInfo().getDocumentCount()); + EXPECT_EQ(uint32_t(0), + spi->getBucketInfo(source).getBucketInfo(). + getDocumentCount()); + EXPECT_EQ(uint32_t(0), + spi->getBucketInfo(target1).getBucketInfo(). + getDocumentCount()); + EXPECT_EQ(uint32_t(1), + spi->getBucketInfo(target2).getBucketInfo(). + getDocumentCount()); document::AllFields fs; Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); Document::UP doc = testDocMan.createRandomDocumentAtLocation(0x06, 0); - EXPECT_TRUE(spi.get(target2, fs, doc->getId(), context).hasDocument()); - EXPECT_TRUE(!spi.get(target1, fs, doc->getId(), context).hasDocument()); - EXPECT_TRUE(!spi.get(source, fs, doc->getId(), context).hasDocument()); + EXPECT_TRUE(spi->get(target2, fs, doc->getId(), context).hasDocument()); + EXPECT_TRUE(!spi->get(target1, fs, doc->getId(), context).hasDocument()); + EXPECT_TRUE(!spi->get(source, fs, doc->getId(), context).hasDocument()); } void @@ -1744,19 +1776,21 @@ ConformanceTest::doTestJoinNormalCase(const Bucket& source1, { document::TestDocMan testDocMan; _factory->clear(); - PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); + PersistenceProvider::UP spi(getSpi(*_factory, testDocMan)); createAndPopulateJoinSourceBuckets(*spi, source1, source2, testDocMan); Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); spi->join(source1, source2, target, context); - testJoinNormalCasePostCondition(*spi, source1, source2, target, testDocMan); + testJoinNormalCasePostCondition(spi, source1, source2, target, + testDocMan); if (_factory->hasPersistence()) { spi.reset(); document::TestDocMan testDocMan2; spi = getSpi(*_factory, testDocMan2); - testJoinNormalCasePostCondition(*spi, source1, source2, target, testDocMan2); + testJoinNormalCasePostCondition(spi, source1, source2, target, + testDocMan2); } } @@ -1778,13 +1812,14 @@ TEST_F(ConformanceTest, testJoinNormalCaseWithMultipleBitsDecreased) void ConformanceTest:: -testJoinNormalCasePostCondition(const PersistenceProvider &spi, +testJoinNormalCasePostCondition(const PersistenceProvider::UP &spi, const Bucket &bucketA, const Bucket &bucketB, const Bucket &bucketC, document::TestDocMan &testDocMan) { - EXPECT_EQ(20, (int)spi.getBucketInfo(bucketC).getBucketInfo().getDocumentCount()); + EXPECT_EQ(20, (int)spi->getBucketInfo(bucketC). + getBucketInfo().getDocumentCount()); document::AllFields fs; Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); @@ -1792,14 +1827,20 @@ testJoinNormalCasePostCondition(const PersistenceProvider &spi, Document::UP doc( testDocMan.createRandomDocumentAtLocation( bucketA.getBucketId().getId(), i)); - EXPECT_TRUE(spi.get(bucketC, fs, doc->getId(), context).hasDocument()); - EXPECT_TRUE(!spi.get(bucketA, fs, doc->getId(), context).hasDocument()); + EXPECT_TRUE( + spi->get(bucketC, fs, doc->getId(), context).hasDocument()); + EXPECT_TRUE( + !spi->get(bucketA, fs, doc->getId(), context).hasDocument()); } for (uint32_t i = 10; i < 20; ++i) { - Document::UP doc(testDocMan.createRandomDocumentAtLocation(bucketB.getBucketId().getId(), i)); - EXPECT_TRUE(spi.get(bucketC, fs, doc->getId(), context).hasDocument()); - EXPECT_TRUE(!spi.get(bucketB, fs, doc->getId(), context).hasDocument()); + Document::UP doc( + testDocMan.createRandomDocumentAtLocation( + bucketB.getBucketId().getId(), i)); + EXPECT_TRUE( + spi->get(bucketC, fs, doc->getId(), context).hasDocument()); + EXPECT_TRUE( + !spi->get(bucketB, fs, doc->getId(), context).hasDocument()); } } @@ -1808,7 +1849,7 @@ TEST_F(ConformanceTest, testJoinTargetExists) { document::TestDocMan testDocMan; _factory->clear(); - PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); + PersistenceProvider::UP spi(getSpi(*_factory, testDocMan)); Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); Bucket bucketA(makeSpiBucket(BucketId(3, 0x02))); @@ -1837,43 +1878,51 @@ TEST_F(ConformanceTest, testJoinTargetExists) } spi->join(bucketA, bucketB, bucketC, context); - testJoinTargetExistsPostCondition(*spi, bucketA, bucketB, bucketC, testDocMan); + testJoinTargetExistsPostCondition(spi, bucketA, bucketB, bucketC, + testDocMan); if (_factory->hasPersistence()) { spi.reset(); document::TestDocMan testDocMan2; spi = getSpi(*_factory, testDocMan2); - testJoinTargetExistsPostCondition(*spi, bucketA, bucketB, bucketC, testDocMan2); + testJoinTargetExistsPostCondition(spi, bucketA, bucketB, bucketC, + testDocMan2); } } void ConformanceTest:: -testJoinTargetExistsPostCondition(const PersistenceProvider &spi, +testJoinTargetExistsPostCondition(const PersistenceProvider::UP &spi, const Bucket &bucketA, const Bucket &bucketB, const Bucket &bucketC, document::TestDocMan &testDocMan) { - EXPECT_EQ(30, (int)spi.getBucketInfo(bucketC).getBucketInfo().getDocumentCount()); + EXPECT_EQ(30, (int)spi->getBucketInfo(bucketC).getBucketInfo(). + getDocumentCount()); document::AllFields fs; Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); for (uint32_t i = 0; i < 10; ++i) { Document::UP doc1 = testDocMan.createRandomDocumentAtLocation(0x02, i); - EXPECT_TRUE(spi.get(bucketC, fs, doc1->getId(), context).hasDocument()); - EXPECT_TRUE(!spi.get(bucketA, fs, doc1->getId(), context).hasDocument()); + EXPECT_TRUE( + spi->get(bucketC, fs, doc1->getId(), context).hasDocument()); + EXPECT_TRUE( + !spi->get(bucketA, fs, doc1->getId(), context).hasDocument()); } for (uint32_t i = 10; i < 20; ++i) { Document::UP doc1 = testDocMan.createRandomDocumentAtLocation(0x06, i); - EXPECT_TRUE(spi.get(bucketC, fs, doc1->getId(), context).hasDocument()); - EXPECT_TRUE(!spi.get(bucketB, fs, doc1->getId(), context).hasDocument()); + EXPECT_TRUE( + spi->get(bucketC, fs, doc1->getId(), context).hasDocument()); + EXPECT_TRUE( + !spi->get(bucketB, fs, doc1->getId(), context).hasDocument()); } for (uint32_t i = 20; i < 30; ++i) { Document::UP doc1 = testDocMan.createRandomDocumentAtLocation(0x06, i); - EXPECT_TRUE(spi.get(bucketC, fs, doc1->getId(), context).hasDocument()); + EXPECT_TRUE( + spi->get(bucketC, fs, doc1->getId(), context).hasDocument()); } } @@ -1888,7 +1937,8 @@ ConformanceTest::populateBucket(const Bucket& b, assert(from <= to); for (uint32_t i = from; i < to; ++i) { const uint32_t location = b.getBucketId().getId(); - Document::SP doc1 = testDocMan.createRandomDocumentAtLocation(location, i); + Document::SP doc1 = testDocMan.createRandomDocumentAtLocation( + location, i); spi.put(b, Timestamp(i + 1), doc1, context); } } @@ -1897,7 +1947,7 @@ TEST_F(ConformanceTest, testJoinOneBucket) { document::TestDocMan testDocMan; _factory->clear(); - PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); + PersistenceProvider::UP spi(getSpi(*_factory, testDocMan)); Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); Bucket bucketA(makeSpiBucket(BucketId(3, 0x02))); @@ -1909,37 +1959,40 @@ TEST_F(ConformanceTest, testJoinOneBucket) populateBucket(bucketA, *spi, context, 0, 10, testDocMan); spi->join(bucketA, bucketB, bucketC, context); - testJoinOneBucketPostCondition(*spi, bucketA, bucketC, testDocMan); + testJoinOneBucketPostCondition(spi, bucketA, bucketC, testDocMan); if (_factory->hasPersistence()) { spi.reset(); document::TestDocMan testDocMan2; spi = getSpi(*_factory, testDocMan2); - testJoinOneBucketPostCondition(*spi, bucketA, bucketC, testDocMan2); + testJoinOneBucketPostCondition(spi, bucketA, bucketC, testDocMan2); } } void ConformanceTest:: -testJoinOneBucketPostCondition(const PersistenceProvider &spi, +testJoinOneBucketPostCondition(const PersistenceProvider::UP &spi, const Bucket &bucketA, const Bucket &bucketC, document::TestDocMan &testDocMan) { - EXPECT_EQ(10, (int)spi.getBucketInfo(bucketC).getBucketInfo().getDocumentCount()); + EXPECT_EQ(10, (int)spi->getBucketInfo(bucketC).getBucketInfo(). + getDocumentCount()); document::AllFields fs; Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); for (uint32_t i = 0; i < 10; ++i) { Document::UP doc1 = testDocMan.createRandomDocumentAtLocation(0x02, i); - EXPECT_TRUE(spi.get(bucketC, fs, doc1->getId(), context).hasDocument()); - EXPECT_TRUE(!spi.get(bucketA, fs, doc1->getId(), context).hasDocument()); + EXPECT_TRUE( + spi->get(bucketC, fs, doc1->getId(), context).hasDocument()); + EXPECT_TRUE( + !spi->get(bucketA, fs, doc1->getId(), context).hasDocument()); } } void ConformanceTest:: testJoinSameSourceBucketsPostCondition( - const PersistenceProvider& spi, + const PersistenceProvider::UP& spi, const Bucket& source, const Bucket& target, document::TestDocMan& testDocMan) @@ -1949,23 +2002,25 @@ testJoinSameSourceBucketsPostCondition( } void -ConformanceTest::doTestJoinSameSourceBuckets(const Bucket& source, const Bucket& target) +ConformanceTest::doTestJoinSameSourceBuckets(const Bucket& source, + const Bucket& target) { document::TestDocMan testDocMan; _factory->clear(); - PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); + PersistenceProvider::UP spi(getSpi(*_factory, testDocMan)); Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); spi->createBucket(source, context); populateBucket(source, *spi, context, 0, 10, testDocMan); spi->join(source, source, target, context); - testJoinSameSourceBucketsPostCondition(*spi, source, target, testDocMan); + testJoinSameSourceBucketsPostCondition(spi, source, target, testDocMan); if (_factory->hasPersistence()) { spi.reset(); document::TestDocMan testDocMan2; spi = getSpi(*_factory, testDocMan2); - testJoinSameSourceBucketsPostCondition(*spi, source, target, testDocMan2); + testJoinSameSourceBucketsPostCondition( + spi, source, target, testDocMan2); } } @@ -1990,14 +2045,17 @@ ConformanceTest::testJoinSameSourceBucketsTargetExistsPostCondition( const Bucket& target, document::TestDocMan& testDocMan) { - EXPECT_EQ(20, (int)spi.getBucketInfo(target).getBucketInfo().getDocumentCount()); + EXPECT_EQ(20, (int)spi.getBucketInfo(target).getBucketInfo(). + getDocumentCount()); document::AllFields fs; Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); for (uint32_t i = 0; i < 20; ++i) { Document::UP doc1 = testDocMan.createRandomDocumentAtLocation(0x02, i); - EXPECT_TRUE(spi.get(target, fs, doc1->getId(), context).hasDocument()); - EXPECT_TRUE(!spi.get(source, fs, doc1->getId(), context).hasDocument()); + EXPECT_TRUE( + spi.get(target, fs, doc1->getId(), context).hasDocument()); + EXPECT_TRUE( + !spi.get(source, fs, doc1->getId(), context).hasDocument()); } } @@ -2005,7 +2063,7 @@ TEST_F(ConformanceTest, testJoinSameSourceBucketsTargetExists) { document::TestDocMan testDocMan; _factory->clear(); - PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); + PersistenceProvider::UP spi(getSpi(*_factory, testDocMan)); Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); Bucket source(makeSpiBucket(BucketId(3, 0x02))); @@ -2024,7 +2082,8 @@ TEST_F(ConformanceTest, testJoinSameSourceBucketsTargetExists) spi.reset(); document::TestDocMan testDocMan2; spi = getSpi(*_factory, testDocMan2); - testJoinSameSourceBucketsTargetExistsPostCondition(*spi, source, target, testDocMan2); + testJoinSameSourceBucketsTargetExistsPostCondition( + *spi, source, target, testDocMan2); } } @@ -2032,8 +2091,9 @@ TEST_F(ConformanceTest, testGetModifiedBuckets) { document::TestDocMan testDocMan; _factory->clear(); - PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); - EXPECT_EQ(0, (int)spi->getModifiedBuckets(makeBucketSpace()).getList().size()); + PersistenceProvider::UP spi(getSpi(*_factory, testDocMan)); + EXPECT_EQ(0, + (int)spi->getModifiedBuckets(makeBucketSpace()).getList().size()); } TEST_F(ConformanceTest, testBucketActivation) @@ -2044,7 +2104,7 @@ TEST_F(ConformanceTest, testBucketActivation) document::TestDocMan testDocMan; _factory->clear(); - PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); + PersistenceProvider::UP spi(getSpi(*_factory, testDocMan)); Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); Bucket bucket(makeSpiBucket(BucketId(8, 0x01))); @@ -2087,7 +2147,7 @@ TEST_F(SingleDocTypeConformanceTest, testBucketActivationSplitAndJoin) document::TestDocMan testDocMan; _factory->clear(); - PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); + PersistenceProvider::UP spi(getSpi(*_factory, testDocMan)); Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); Bucket bucketA(makeSpiBucket(BucketId(3, 0x02))); @@ -2165,7 +2225,7 @@ TEST_F(ConformanceTest, testRemoveEntry) } document::TestDocMan testDocMan; _factory->clear(); - PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); + PersistenceProvider::UP spi(getSpi(*_factory, testDocMan)); Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); Bucket bucket(makeSpiBucket(BucketId(8, 0x01))); @@ -2234,7 +2294,7 @@ TEST_F(ConformanceTest, testBucketSpaces) } document::TestDocMan testDocMan; _factory->clear(); - PersistenceProviderUP spi(getSpi(*_factory, testDocMan)); + PersistenceProvider::UP spi(getSpi(*_factory, testDocMan)); Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); BucketSpace bucketSpace0(makeBucketSpace("testdoctype1")); BucketSpace bucketSpace1(makeBucketSpace("testdoctype2")); diff --git a/persistence/src/vespa/persistence/conformancetest/conformancetest.h b/persistence/src/vespa/persistence/conformancetest/conformancetest.h index 19be63cf3cb..05c1bc87c9f 100644 --- a/persistence/src/vespa/persistence/conformancetest/conformancetest.h +++ b/persistence/src/vespa/persistence/conformancetest/conformancetest.h @@ -26,13 +26,12 @@ namespace storage::spi { class ConformanceTest : public ::testing::Test { public: - using PersistenceProviderUP = std::unique_ptr<PersistenceProvider>; struct PersistenceFactory { typedef std::unique_ptr<PersistenceFactory> UP; using DocumenttypesConfig = const document::internal::InternalDocumenttypesType; - virtual ~PersistenceFactory() = default; - virtual PersistenceProviderUP getPersistenceImplementation( + virtual ~PersistenceFactory() {} + virtual PersistenceProvider::UP getPersistenceImplementation( const std::shared_ptr<const document::DocumentTypeRepo> &repo, const DocumenttypesConfig &typesCfg) = 0; @@ -75,19 +74,19 @@ protected: document::TestDocMan& testDocMan); void - testDeleteBucketPostCondition(const PersistenceProvider &spi, + testDeleteBucketPostCondition(const PersistenceProvider::UP &spi, const Bucket &bucket, const Document &doc1); void - testSplitNormalCasePostCondition(const PersistenceProvider &spi, + testSplitNormalCasePostCondition(const PersistenceProvider::UP &spi, const Bucket &bucketA, const Bucket &bucketB, const Bucket &bucketC, document::TestDocMan &testDocMan); void - testSplitTargetExistsPostCondition(const PersistenceProvider &spi, + testSplitTargetExistsPostCondition(const PersistenceProvider::UP &spi, const Bucket &bucketA, const Bucket &bucketB, const Bucket &bucketC, @@ -95,7 +94,7 @@ protected: void testSplitSingleDocumentInSourcePostCondition( - const PersistenceProvider& spi, + const PersistenceProvider::UP& spi, const Bucket& source, const Bucket& target1, const Bucket& target2, @@ -114,21 +113,21 @@ protected: const Bucket& target); void - testJoinNormalCasePostCondition(const PersistenceProvider &spi, + testJoinNormalCasePostCondition(const PersistenceProvider::UP &spi, const Bucket &bucketA, const Bucket &bucketB, const Bucket &bucketC, document::TestDocMan &testDocMan); void - testJoinTargetExistsPostCondition(const PersistenceProvider &spi, + testJoinTargetExistsPostCondition(const PersistenceProvider::UP &spi, const Bucket &bucketA, const Bucket &bucketB, const Bucket &bucketC, document::TestDocMan &testDocMan); void - testJoinOneBucketPostCondition(const PersistenceProvider &spi, + testJoinOneBucketPostCondition(const PersistenceProvider::UP &spi, const Bucket &bucketA, const Bucket &bucketC, document::TestDocMan &testDocMan); @@ -139,7 +138,7 @@ protected: void testJoinSameSourceBucketsPostCondition( - const PersistenceProvider& spi, + const PersistenceProvider::UP& spi, const Bucket& source, const Bucket& target, document::TestDocMan& testDocMan); diff --git a/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp b/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp index 297a3319939..73783413061 100644 --- a/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp +++ b/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp @@ -547,7 +547,12 @@ DummyPersistence::get(const Bucket& b, const document::FieldSet& fieldSet, const } CreateIteratorResult -DummyPersistence::createIterator(const Bucket &b, FieldSetSP fs, const Selection &s, IncludedVersions v, Context &) +DummyPersistence::createIterator( + const Bucket& b, + const document::FieldSet& fs, + const Selection& s, + IncludedVersions v, + Context&) { DUMMYPERSISTENCE_VERIFY_INITIALIZED; LOG(debug, "createIterator(%s)", b.toString().c_str()); @@ -580,7 +585,7 @@ DummyPersistence::createIterator(const Bucket &b, FieldSetSP fs, const Selection } // Memory pointed to by 'it' should now be valid from here on out - it->_fieldSet = std::move(fs); + it->_fieldSet = std::unique_ptr<document::FieldSet>(fs.clone()); const BucketContent::GidMapType& gidMap((*bc)->_gidMap); if (s.getTimestampSubset().empty()) { @@ -595,7 +600,8 @@ DummyPersistence::createIterator(const Bucket &b, FieldSetSP fs, const Selection entry.getTimestamp() > s.getToTimestamp()) { continue; } - BucketContent::GidMapType::const_iterator gidIt(gidMap.find(bucketEntry.gid)); + BucketContent::GidMapType::const_iterator gidIt( + gidMap.find(bucketEntry.gid)); assert(gidIt != gidMap.end()); if (entry.isRemove()) { diff --git a/persistence/src/vespa/persistence/dummyimpl/dummypersistence.h b/persistence/src/vespa/persistence/dummyimpl/dummypersistence.h index c3a4991a590..94c2e1cd9a4 100644 --- a/persistence/src/vespa/persistence/dummyimpl/dummypersistence.h +++ b/persistence/src/vespa/persistence/dummyimpl/dummypersistence.h @@ -93,7 +93,7 @@ struct Iterator { using UP = std::unique_ptr<Iterator>; Bucket _bucket; std::vector<Timestamp> _leftToIterate; - std::shared_ptr<document::FieldSet> _fieldSet; + std::unique_ptr<document::FieldSet> _fieldSet; }; class DummyPersistence; @@ -158,8 +158,11 @@ public: RemoveResult remove(const Bucket& b, Timestamp t, const DocumentId& did, Context&) override; UpdateResult update(const Bucket&, Timestamp, DocumentUpdateSP, Context&) override; - CreateIteratorResult - createIterator(const Bucket &bucket, FieldSetSP fs, const Selection &, IncludedVersions, Context &context) override; + CreateIteratorResult createIterator(const Bucket&, + const document::FieldSet& fs, + const Selection&, + IncludedVersions, + Context&) override; IterateResult iterate(IteratorId, uint64_t maxByteSize, Context&) const override; Result destroyIterator(IteratorId, Context&) override; diff --git a/persistence/src/vespa/persistence/spi/persistenceprovider.h b/persistence/src/vespa/persistence/spi/persistenceprovider.h index 021ac6338eb..2bb91c776d0 100644 --- a/persistence/src/vespa/persistence/spi/persistenceprovider.h +++ b/persistence/src/vespa/persistence/spi/persistenceprovider.h @@ -56,8 +56,8 @@ namespace storage::spi { */ struct PersistenceProvider { + typedef std::unique_ptr<PersistenceProvider> UP; using BucketSpace = document::BucketSpace; - using FieldSetSP = std::shared_ptr<document::FieldSet>; virtual ~PersistenceProvider(); @@ -258,9 +258,12 @@ struct PersistenceProvider * error. Identifier must be non-zero, as zero is used internally to * signify an invalid iterator ID. */ - virtual CreateIteratorResult - createIterator(const Bucket &bucket, FieldSetSP fieldSet, const Selection &selection, - IncludedVersions versions, Context &context) = 0; + virtual CreateIteratorResult createIterator( + const Bucket&, + const document::FieldSet& fieldSet, + const Selection& selection, //TODO: Make AST + IncludedVersions versions, + Context&) = 0; /** * Iterate over a bucket's document space using a valid iterator id diff --git a/searchcore/src/apps/proton/downpersistence.cpp b/searchcore/src/apps/proton/downpersistence.cpp index 999cf6696ea..aa87c383c33 100644 --- a/searchcore/src/apps/proton/downpersistence.cpp +++ b/searchcore/src/apps/proton/downpersistence.cpp @@ -95,7 +95,8 @@ DownPersistence::get(const Bucket&, const document::FieldSet&, const DocumentId& } CreateIteratorResult -DownPersistence::createIterator(const Bucket &, FieldSetSP, const Selection &, IncludedVersions, Context &) +DownPersistence::createIterator(const Bucket&, const document::FieldSet&, + const Selection&, IncludedVersions, Context&) { return CreateIteratorResult(errorResult.getErrorCode(), errorResult.getErrorMessage()); } diff --git a/searchcore/src/apps/proton/downpersistence.h b/searchcore/src/apps/proton/downpersistence.h index d8b48172880..10e3d9c1ad7 100644 --- a/searchcore/src/apps/proton/downpersistence.h +++ b/searchcore/src/apps/proton/downpersistence.h @@ -39,9 +39,8 @@ public: UpdateResult update(const Bucket&, Timestamp timestamp, DocumentUpdateSP update, Context&) override; GetResult get(const Bucket&, const document::FieldSet& fieldSet, const DocumentId& id, Context&) const override; - CreateIteratorResult - createIterator(const Bucket &bucket, FieldSetSP fieldSet, const Selection &selection, IncludedVersions versions, - Context &context) override; + CreateIteratorResult createIterator(const Bucket&, const document::FieldSet& fieldSet, + const Selection& selection, IncludedVersions versions, Context&) override; IterateResult iterate(IteratorId id, uint64_t maxByteSize, Context&) const override; Result destroyIterator(IteratorId id, Context&) override; diff --git a/searchcore/src/apps/tests/persistenceconformance_test.cpp b/searchcore/src/apps/tests/persistenceconformance_test.cpp index 44fb2770594..217d1edcd57 100644 --- a/searchcore/src/apps/tests/persistenceconformance_test.cpp +++ b/searchcore/src/apps/tests/persistenceconformance_test.cpp @@ -362,11 +362,14 @@ public: ~MyPersistenceFactory() override { clear(); } - std::unique_ptr<PersistenceProvider> getPersistenceImplementation(const std::shared_ptr<const DocumentTypeRepo> &repo, + PersistenceProvider::UP getPersistenceImplementation(const std::shared_ptr<const DocumentTypeRepo> &repo, const DocumenttypesConfig &typesCfg) override { ConfigFactory cfgFactory(repo, std::make_shared<DocumenttypesConfig>(typesCfg), _schemaFactory); _docDbRepo = std::make_unique<DocumentDBRepo>(cfgFactory, _docDbFactory); - auto engine = std::make_unique<MyPersistenceEngine>(_engineOwner,_writeFilter,std::move(_docDbRepo), _docType); + PersistenceEngine::UP engine(new MyPersistenceEngine(_engineOwner, + _writeFilter, + std::move(_docDbRepo), + _docType)); assert( ! _docDbRepo); // Repo should be handed over return engine; } diff --git a/searchcore/src/tests/proton/document_iterator/document_iterator_test.cpp b/searchcore/src/tests/proton/document_iterator/document_iterator_test.cpp index 44ce55edfbd..147bd9afb84 100644 --- a/searchcore/src/tests/proton/document_iterator/document_iterator_test.cpp +++ b/searchcore/src/tests/proton/document_iterator/document_iterator_test.cpp @@ -447,14 +447,14 @@ TEST("require that custom retrievers work as expected") { } TEST("require that an empty list of retrievers can be iterated") { - DocumentIterator itr(bucket(5), std::make_shared<document::AllFields>(), selectAll(), newestV(), -1, false); + DocumentIterator itr(bucket(5), document::AllFields(), selectAll(), newestV(), -1, false); IterateResult res = itr.iterate(largeNum); EXPECT_EQUAL(0u, res.getEntries().size()); EXPECT_TRUE(res.isCompleted()); } TEST("require that a list of empty retrievers can be iterated") { - DocumentIterator itr(bucket(5), std::make_shared<document::AllFields>(), selectAll(), newestV(), -1, false); + DocumentIterator itr(bucket(5), document::AllFields(), selectAll(), newestV(), -1, false); itr.add(nil()); itr.add(nil()); itr.add(nil()); @@ -464,7 +464,7 @@ TEST("require that a list of empty retrievers can be iterated") { } TEST("require that normal documents can be iterated") { - DocumentIterator itr(bucket(5), std::make_shared<document::AllFields>(), selectAll(), newestV(), -1, false); + DocumentIterator itr(bucket(5), document::AllFields(), selectAll(), newestV(), -1, false); itr.add(doc("id:ns:document::1", Timestamp(2), bucket(5))); itr.add(cat(doc("id:ns:document::2", Timestamp(3), bucket(5)), doc("id:ns:document::3", Timestamp(4), bucket(5)))); @@ -487,12 +487,12 @@ void verifyIterateIgnoringStopSignal(DocumentIterator & itr) { } TEST("require that iterator stops at the end, and does not auto rewind") { - DocumentIterator itr(bucket(5), std::make_shared<document::AllFields>(), selectAll(), newestV(), -1, false); + DocumentIterator itr(bucket(5), document::AllFields(), selectAll(), newestV(), -1, false); TEST_DO(verifyIterateIgnoringStopSignal(itr)); } TEST("require that iterator ignoring maxbytes stops at the end, and does not auto rewind") { - DocumentIterator itr(bucket(5), std::make_shared<document::AllFields>(), selectAll(), newestV(), -1, true); + DocumentIterator itr(bucket(5), document::AllFields(), selectAll(), newestV(), -1, true); TEST_DO(verifyIterateIgnoringStopSignal(itr)); } @@ -515,12 +515,12 @@ void verifyStrongReadConsistency(DocumentIterator & itr) { } TEST("require that default readconsistency does commit") { - DocumentIterator itr(bucket(5), std::make_shared<document::AllFields>(), selectAll(), newestV(), -1, false); + DocumentIterator itr(bucket(5), document::AllFields(), selectAll(), newestV(), -1, false); TEST_DO(verifyStrongReadConsistency(itr)); } TEST("require that readconsistency::strong does commit") { - DocumentIterator itr(bucket(5), std::make_shared<document::AllFields>(), selectAll(), newestV(), -1, false, storage::spi::ReadConsistency::STRONG); + DocumentIterator itr(bucket(5), document::AllFields(), selectAll(), newestV(), -1, false, storage::spi::ReadConsistency::STRONG); TEST_DO(verifyStrongReadConsistency(itr)); } @@ -528,7 +528,7 @@ TEST("require that docid limit is honoured") { IDocumentRetriever::SP retriever = doc("id:ns:document::1", Timestamp(2), bucket(5)); auto & udr = dynamic_cast<UnitDR &>(*retriever); udr.docid = 7; - DocumentIterator itr(bucket(5), std::make_shared<document::AllFields>(), selectAll(), newestV(), -1, false); + DocumentIterator itr(bucket(5), document::AllFields(), selectAll(), newestV(), -1, false); itr.add(retriever); IterateResult res = itr.iterate(largeNum); EXPECT_TRUE(res.isCompleted()); @@ -536,7 +536,7 @@ TEST("require that docid limit is honoured") { TEST_DO(checkEntry(res, 0, Document(*DataType::DOCUMENT, DocumentId("id:ns:document::1")), Timestamp(2))); udr.setDocIdLimit(7); - DocumentIterator limited(bucket(5), std::make_shared<document::AllFields>(), selectAll(), newestV(), -1, false); + DocumentIterator limited(bucket(5), document::AllFields(), selectAll(), newestV(), -1, false); limited.add(retriever); res = limited.iterate(largeNum); EXPECT_TRUE(res.isCompleted()); @@ -544,7 +544,7 @@ TEST("require that docid limit is honoured") { } TEST("require that remove entries can be iterated") { - DocumentIterator itr(bucket(5), std::make_shared<document::AllFields>(), selectAll(), newestV(), -1, false); + DocumentIterator itr(bucket(5), document::AllFields(), selectAll(), newestV(), -1, false); itr.add(rem("id:ns:document::1", Timestamp(2), bucket(5))); itr.add(cat(rem("id:ns:document::2", Timestamp(3), bucket(5)), rem("id:ns:document::3", Timestamp(4), bucket(5)))); @@ -557,7 +557,7 @@ TEST("require that remove entries can be iterated") { } TEST("require that remove entries can be ignored") { - DocumentIterator itr(bucket(5), std::make_shared<document::AllFields>(), selectAll(), docV(), -1, false); + DocumentIterator itr(bucket(5), document::AllFields(), selectAll(), docV(), -1, false); itr.add(rem("id:ns:document::1", Timestamp(2), bucket(5))); itr.add(cat(doc("id:ns:document::2", Timestamp(3), bucket(5)), rem("id:ns:document::3", Timestamp(4), bucket(5)))); @@ -568,7 +568,7 @@ TEST("require that remove entries can be ignored") { } TEST("require that iterating all versions returns both documents and removes") { - DocumentIterator itr(bucket(5), std::make_shared<document::AllFields>(), selectAll(), allV(), -1, false); + DocumentIterator itr(bucket(5), document::AllFields(), selectAll(), allV(), -1, false); itr.add(rem("id:ns:document::1", Timestamp(2), bucket(5))); itr.add(cat(doc("id:ns:document::2", Timestamp(3), bucket(5)), rem("id:ns:document::3", Timestamp(4), bucket(5)))); @@ -581,7 +581,7 @@ TEST("require that iterating all versions returns both documents and removes") { } TEST("require that using an empty field set returns meta-data only") { - DocumentIterator itr(bucket(5), std::make_shared<document::NoFields>(), selectAll(), newestV(), -1, false); + DocumentIterator itr(bucket(5), document::NoFields(), selectAll(), newestV(), -1, false); itr.add(doc("id:ns:document::1", Timestamp(2), bucket(5))); itr.add(cat(doc("id:ns:document::2", Timestamp(3), bucket(5)), rem("id:ns:document::3", Timestamp(4), bucket(5)))); @@ -594,7 +594,7 @@ TEST("require that using an empty field set returns meta-data only") { } TEST("require that entries in other buckets are skipped") { - DocumentIterator itr(bucket(5), std::make_shared<document::AllFields>(), selectAll(), newestV(), -1, false); + DocumentIterator itr(bucket(5), document::AllFields(), selectAll(), newestV(), -1, false); itr.add(rem("id:ns:document::1", Timestamp(2), bucket(6))); itr.add(cat(doc("id:ns:document::2", Timestamp(3), bucket(5)), doc("id:ns:document::3", Timestamp(4), bucket(6)))); @@ -605,7 +605,7 @@ TEST("require that entries in other buckets are skipped") { } TEST("require that maxBytes splits iteration results") { - DocumentIterator itr(bucket(5), std::make_shared<document::AllFields>(), selectAll(), newestV(), -1, false); + DocumentIterator itr(bucket(5), document::AllFields(), selectAll(), newestV(), -1, false); itr.add(doc("id:ns:document::1", Timestamp(2), bucket(5))); itr.add(cat(rem("id:ns:document::2", Timestamp(3), bucket(5)), doc("id:ns:document::3", Timestamp(4), bucket(5)))); @@ -626,7 +626,7 @@ TEST("require that maxBytes splits iteration results") { } TEST("require that maxBytes splits iteration results for meta-data only iteration") { - DocumentIterator itr(bucket(5), std::make_shared<document::NoFields>(), selectAll(), newestV(), -1, false); + DocumentIterator itr(bucket(5), document::NoFields(), selectAll(), newestV(), -1, false); itr.add(doc("id:ns:document::1", Timestamp(2), bucket(5))); itr.add(cat(rem("id:ns:document::2", Timestamp(3), bucket(5)), doc("id:ns:document::3", Timestamp(4), bucket(5)))); @@ -646,7 +646,7 @@ TEST("require that maxBytes splits iteration results for meta-data only iteratio } TEST("require that at least one document is returned by visit") { - DocumentIterator itr(bucket(5), std::make_shared<document::AllFields>(), selectAll(), newestV(), -1, false); + DocumentIterator itr(bucket(5), document::AllFields(), selectAll(), newestV(), -1, false); itr.add(doc("id:ns:document::1", Timestamp(2), bucket(5))); itr.add(cat(rem("id:ns:document::2", Timestamp(3), bucket(5)), doc("id:ns:document::3", Timestamp(4), bucket(5)))); @@ -656,7 +656,7 @@ TEST("require that at least one document is returned by visit") { } TEST("require that documents outside the timestamp limits are ignored") { - DocumentIterator itr(bucket(5), std::make_shared<document::AllFields>(), selectTimestampRange(100, 200), newestV(), -1, false); + DocumentIterator itr(bucket(5), document::AllFields(), selectTimestampRange(100, 200), newestV(), -1, false); itr.add(doc("id:ns:document::1", Timestamp(99), bucket(5))); itr.add(doc("id:ns:document::2", Timestamp(100), bucket(5))); itr.add(doc("id:ns:document::3", Timestamp(200), bucket(5))); @@ -675,7 +675,7 @@ TEST("require that documents outside the timestamp limits are ignored") { } TEST("require that timestamp subset returns the appropriate documents") { - DocumentIterator itr(bucket(5), std::make_shared<document::AllFields>(), selectTimestampSet(200, 350, 400), newestV(), -1, false); + DocumentIterator itr(bucket(5), document::AllFields(), selectTimestampSet(200, 350, 400), newestV(), -1, false); itr.add(doc("id:ns:document::1", Timestamp(500), bucket(5))); itr.add(doc("id:ns:document::2", Timestamp(400), bucket(5))); itr.add(doc("id:ns:document::3", Timestamp(300), bucket(5))); @@ -693,7 +693,7 @@ TEST("require that timestamp subset returns the appropriate documents") { } TEST("require that document selection will filter results") { - DocumentIterator itr(bucket(5), std::make_shared<document::AllFields>(), selectDocs("id=\"id:ns:document::xxx*\""), newestV(), -1, false); + DocumentIterator itr(bucket(5), document::AllFields(), selectDocs("id=\"id:ns:document::xxx*\""), newestV(), -1, false); itr.add(doc("id:ns:document::xxx1", Timestamp(99), bucket(5))); itr.add(doc("id:ns:document::yyy1", Timestamp(100), bucket(5))); itr.add(doc("id:ns:document::xxx2", Timestamp(200), bucket(5))); @@ -712,7 +712,7 @@ TEST("require that document selection will filter results") { } TEST("require that document selection handles 'field == null'") { - DocumentIterator itr(bucket(5), std::make_shared<document::AllFields>(), selectDocs("foo.aa == null"), newestV(), -1, false); + DocumentIterator itr(bucket(5), document::AllFields(), selectDocs("foo.aa == null"), newestV(), -1, false); itr.add(doc_with_null_fields("id:ns:foo::xxx1", Timestamp(99), bucket(5))); itr.add(doc_with_null_fields("id:ns:foo::xxx2", Timestamp(100), bucket(5))); IterateResult res = itr.iterate(largeNum); @@ -725,7 +725,7 @@ TEST("require that document selection handles 'field == null'") { } TEST("require that invalid document selection returns no documents") { - DocumentIterator itr(bucket(5), std::make_shared<document::AllFields>(), selectDocs("=="), newestV(), -1, false); + DocumentIterator itr(bucket(5), document::AllFields(), selectDocs("=="), newestV(), -1, false); itr.add(doc("id:ns:document::xxx1", Timestamp(99), bucket(5))); itr.add(doc("id:ns:document::yyy1", Timestamp(100), bucket(5))); itr.add(doc("id:ns:document::xxx2", Timestamp(200), bucket(5))); @@ -740,7 +740,7 @@ TEST("require that invalid document selection returns no documents") { } TEST("require that document selection and timestamp range works together") { - DocumentIterator itr(bucket(5), std::make_shared<document::AllFields>(), selectDocsWithinRange("id=\"id:ns:document::xxx*\"", 100, 200), newestV(), -1, false); + DocumentIterator itr(bucket(5), document::AllFields(), selectDocsWithinRange("id=\"id:ns:document::xxx*\"", 100, 200), newestV(), -1, false); itr.add(doc("id:ns:document::xxx1", Timestamp(99), bucket(5))); itr.add(doc("id:ns:document::yyy1", Timestamp(100), bucket(5))); itr.add(doc("id:ns:document::xxx2", Timestamp(200), bucket(5))); @@ -757,8 +757,9 @@ TEST("require that document selection and timestamp range works together") { } TEST("require that fieldset limits fields returned") { - auto limited = std::make_shared<document::FieldCollection>(getDocType(),document::Field::Set::Builder().add(&getDocType().getField("header")).build()); - DocumentIterator itr(bucket(5), std::move(limited), selectAll(), newestV(), -1, false); + document::FieldCollection limited(getDocType(), + document::Field::Set::Builder().add(&getDocType().getField("header")).build()); + DocumentIterator itr(bucket(5), limited, selectAll(), newestV(), -1, false); itr.add(doc_with_fields("id:ns:foo::xxx1", Timestamp(1), bucket(5))); IterateResult res = itr.iterate(largeNum); EXPECT_TRUE(res.isCompleted()); @@ -776,7 +777,8 @@ bool contains(const Container& c, const T& value) { } TEST("require that userdoc-constrained selections pre-filter on GIDs") { - DocumentIterator itr(bucket(5), std::make_shared<document::AllFields>(), selectDocs("id.user=1234"), newestV(), -1, false); + DocumentIterator itr(bucket(5), document::AllFields(), + selectDocs("id.user=1234"), newestV(), -1, false); VisitRecordingUnitDR::VisitedLIDs visited_lids; // Even though GID filtering is probabilistic when it comes to filtering // user IDs that cover the 64-bit range, it's fully deterministic when the @@ -806,7 +808,7 @@ TEST("require that userdoc-constrained selections pre-filter on GIDs") { TEST("require that attributes are used") { UnitDR::reset(); - DocumentIterator itr(bucket(5), std::make_shared<document::AllFields>(), selectDocs("foo.aa == 45"), docV(), -1, false); + DocumentIterator itr(bucket(5), document::AllFields(), selectDocs("foo.aa == 45"), docV(), -1, false); itr.add(doc_with_attr_fields("id:ns:foo::xx1", Timestamp(1), bucket(5), 27, 28, 27, 2.7, 2.8, "x27", "x28")); itr.add(doc_with_attr_fields("id:ns:foo::xx2", Timestamp(2), bucket(5), @@ -836,7 +838,7 @@ TEST("require that attributes are used") TEST_DO(checkEntry(res, 0, expected1, Timestamp(2))); TEST_DO(checkEntry(res, 1, expected2, Timestamp(4))); - DocumentIterator itr2(bucket(5), std::make_shared<document::AllFields>(), selectDocs("foo.dd == 4.5"), docV(), -1, false); + DocumentIterator itr2(bucket(5), document::AllFields(), selectDocs("foo.dd == 4.5"), docV(), -1, false); itr2.add(doc_with_attr_fields("id:ns:foo::xx5", Timestamp(5), bucket(5), 27, 28, 27, 2.7, 2.8, "x27", "x28")); itr2.add(doc_with_attr_fields("id:ns:foo::xx6", Timestamp(6), bucket(5), @@ -866,7 +868,7 @@ TEST("require that attributes are used") TEST_DO(checkEntry(res2, 0, expected3, Timestamp(6))); TEST_DO(checkEntry(res2, 1, expected4, Timestamp(8))); - DocumentIterator itr3(bucket(5), std::make_shared<document::AllFields>(), selectDocs("foo.ss == \"x45\""), docV(), -1, false); + DocumentIterator itr3(bucket(5), document::AllFields(), selectDocs("foo.ss == \"x45\""), docV(), -1, false); itr3.add(doc_with_attr_fields("id:ns:foo::xx9", Timestamp(9), bucket(5), 27, 28, 27, 2.7, 2.8, "x27", "x28")); itr3.add(doc_with_attr_fields("id:ns:foo::xx10", Timestamp(10), bucket(5), diff --git a/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp b/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp index 2540a991015..a88b36ea489 100644 --- a/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp +++ b/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp @@ -404,8 +404,7 @@ public: void setPruneConfig(const DocumentDBPruneRemovedDocumentsConfig &pruneConfig) { - DocumentDBMaintenanceConfig::SP - newCfg(new DocumentDBMaintenanceConfig( + auto newCfg = std::make_shared<DocumentDBMaintenanceConfig>( pruneConfig, _mcCfg->getHeartBeatConfig(), _mcCfg->getSessionCachePruneInterval(), @@ -414,7 +413,7 @@ public: _mcCfg->getAttributeUsageFilterConfig(), _mcCfg->getAttributeUsageSampleInterval(), _mcCfg->getBlockableJobConfig(), - _mcCfg->getFlushConfig())); + _mcCfg->getFlushConfig()); _mcCfg = newCfg; forwardMaintenanceConfig(); } @@ -422,8 +421,7 @@ public: void setHeartBeatConfig(const DocumentDBHeartBeatConfig &heartBeatConfig) { - DocumentDBMaintenanceConfig::SP - newCfg(new DocumentDBMaintenanceConfig( + auto newCfg = std::make_shared<DocumentDBMaintenanceConfig>( _mcCfg->getPruneRemovedDocumentsConfig(), heartBeatConfig, _mcCfg->getSessionCachePruneInterval(), @@ -432,7 +430,7 @@ public: _mcCfg->getAttributeUsageFilterConfig(), _mcCfg->getAttributeUsageSampleInterval(), _mcCfg->getBlockableJobConfig(), - _mcCfg->getFlushConfig())); + _mcCfg->getFlushConfig()); _mcCfg = newCfg; forwardMaintenanceConfig(); } @@ -440,8 +438,7 @@ public: void setGroupingSessionPruneInterval(vespalib::duration groupingSessionPruneInterval) { - DocumentDBMaintenanceConfig::SP - newCfg(new DocumentDBMaintenanceConfig( + auto newCfg = std::make_shared<DocumentDBMaintenanceConfig>( _mcCfg->getPruneRemovedDocumentsConfig(), _mcCfg->getHeartBeatConfig(), groupingSessionPruneInterval, @@ -450,14 +447,13 @@ public: _mcCfg->getAttributeUsageFilterConfig(), _mcCfg->getAttributeUsageSampleInterval(), _mcCfg->getBlockableJobConfig(), - _mcCfg->getFlushConfig())); + _mcCfg->getFlushConfig()); _mcCfg = newCfg; forwardMaintenanceConfig(); } void setLidSpaceCompactionConfig(const DocumentDBLidSpaceCompactionConfig &cfg) { - DocumentDBMaintenanceConfig::SP - newCfg(new DocumentDBMaintenanceConfig( + auto newCfg = std::make_shared<DocumentDBMaintenanceConfig>( _mcCfg->getPruneRemovedDocumentsConfig(), _mcCfg->getHeartBeatConfig(), _mcCfg->getSessionCachePruneInterval(), @@ -466,21 +462,19 @@ public: _mcCfg->getAttributeUsageFilterConfig(), _mcCfg->getAttributeUsageSampleInterval(), _mcCfg->getBlockableJobConfig(), - _mcCfg->getFlushConfig())); + _mcCfg->getFlushConfig()); _mcCfg = newCfg; forwardMaintenanceConfig(); } void - performNotifyBucketStateChanged(document::BucketId bucketId, - BucketInfo::ActiveState newState) + performNotifyBucketStateChanged(document::BucketId bucketId, BucketInfo::ActiveState newState) { _bucketHandler.notifyBucketStateChanged(bucketId, newState); } void - notifyBucketStateChanged(const document::BucketId &bucketId, - BucketInfo::ActiveState newState) + notifyBucketStateChanged(const document::BucketId &bucketId, BucketInfo::ActiveState newState) { _executor.execute(makeTask(makeClosure(this, &MaintenanceControllerFixture:: @@ -494,7 +488,7 @@ public: MaintenanceDocumentSubDB MyDocumentSubDB::getSubDB() { - IDocumentRetriever::SP retriever(new MyDocumentRetriever(*this)); + auto retriever = std::make_shared<MyDocumentRetriever>(*this); return MaintenanceDocumentSubDB("my_sub_db", _subDBId, _metaStoreSP, diff --git a/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp b/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp index 6351c187b45..a31deca5d12 100644 --- a/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp +++ b/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp @@ -693,7 +693,7 @@ TEST_F("require that createIterator does", SimpleFixture) { storage::spi::LoadType loadType(0, "default"); Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); CreateIteratorResult result = - f.engine.createIterator(bucket1, std::make_shared<document::AllFields>(), selection, + f.engine.createIterator(bucket1, document::AllFields(), selection, storage::spi::NEWEST_DOCUMENT_ONLY, context); EXPECT_FALSE(result.hasError()); EXPECT_TRUE(result.getIteratorId()); @@ -707,10 +707,10 @@ TEST_F("require that iterator ids are unique", SimpleFixture) { storage::spi::LoadType loadType(0, "default"); Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); CreateIteratorResult result = - f.engine.createIterator(bucket1, std::make_shared<document::AllFields>(), selection, + f.engine.createIterator(bucket1, document::AllFields(), selection, storage::spi::NEWEST_DOCUMENT_ONLY, context); CreateIteratorResult result2 = - f.engine.createIterator(bucket1, std::make_shared<document::AllFields>(), selection, + f.engine.createIterator(bucket1, document::AllFields(), selection, storage::spi::NEWEST_DOCUMENT_ONLY, context); EXPECT_FALSE(result.hasError()); EXPECT_FALSE(result2.hasError()); @@ -727,7 +727,7 @@ TEST_F("require that iterate requires valid iterator", SimpleFixture) { EXPECT_EQUAL("Unknown iterator with id 1", it_result.getErrorMessage()); CreateIteratorResult result = - f.engine.createIterator(bucket1, std::make_shared<document::AllFields>(), selection, + f.engine.createIterator(bucket1, document::AllFields(), selection, storage::spi::NEWEST_DOCUMENT_ONLY, context); EXPECT_TRUE(result.getIteratorId()); @@ -743,7 +743,7 @@ TEST_F("require that iterate returns documents", SimpleFixture) { Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); uint64_t max_size = 1024; CreateIteratorResult result = - f.engine.createIterator(bucket1, std::make_shared<document::AllFields>(), selection, + f.engine.createIterator(bucket1, document::AllFields(), selection, storage::spi::NEWEST_DOCUMENT_ONLY, context); EXPECT_TRUE(result.getIteratorId()); @@ -758,7 +758,7 @@ TEST_F("require that destroyIterator prevents iteration", SimpleFixture) { storage::spi::LoadType loadType(0, "default"); Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); CreateIteratorResult create_result = - f.engine.createIterator(bucket1, std::make_shared<document::AllFields>(), selection, + f.engine.createIterator(bucket1, document::AllFields(), selection, storage::spi::NEWEST_DOCUMENT_ONLY, context); EXPECT_TRUE(create_result.getIteratorId()); @@ -779,7 +779,7 @@ TEST_F("require that buckets are frozen during iterator life", SimpleFixture) { storage::spi::LoadType loadType(0, "default"); Context context(loadType, storage::spi::Priority(0), storage::spi::Trace::TraceLevel(0)); CreateIteratorResult create_result = - f.engine.createIterator(bucket1, std::make_shared<document::AllFields>(), selection, + f.engine.createIterator(bucket1, document::AllFields(), selection, storage::spi::NEWEST_DOCUMENT_ONLY, context); EXPECT_TRUE(f.hset.handler1.isFrozen(bucket1)); EXPECT_TRUE(f.hset.handler2.isFrozen(bucket1)); diff --git a/searchcore/src/tests/proton/server/visibility_handler/visibility_handler_test.cpp b/searchcore/src/tests/proton/server/visibility_handler/visibility_handler_test.cpp index ca8d0a1d26a..468b3b5b29c 100644 --- a/searchcore/src/tests/proton/server/visibility_handler/visibility_handler_test.cpp +++ b/searchcore/src/tests/proton/server/visibility_handler/visibility_handler_test.cpp @@ -1,13 +1,16 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/log/log.h> -LOG_SETUP("visibility_handler_test"); + #include <vespa/vespalib/testkit/testapp.h> #include <vespa/searchcore/proton/server/visibilityhandler.h> #include <vespa/searchcore/proton/test/dummy_feed_view.h> #include <vespa/searchcore/proton/test/threading_service_observer.h> #include <vespa/searchcore/proton/server/executorthreadingservice.h> +#include <vespa/searchcore/proton/common/pendinglidtracker.h> #include <vespa/vespalib/util/lambdatask.h> +#include <vespa/log/log.h> +LOG_SETUP("visibility_handler_test"); + using search::SerialNum; using proton::IGetSerialNum; using proton::test::DummyFeedView; @@ -25,8 +28,7 @@ class MyGetSerialNum : public IGetSerialNum public: MyGetSerialNum() : _serialNum(0u) - { - } + {} SerialNum getSerialNum() const override { return _serialNum; } void setSerialNum(SerialNum serialNum) { _serialNum = serialNum; } }; @@ -78,8 +80,7 @@ public: _feedViewReal(std::make_shared<MyFeedView>()), _feedView(_feedViewReal), _visibilityHandler(_getSerialNum, _writeService, _feedView) - { - } + {} void checkCommitPostCondition(uint32_t expForceCommitCount, diff --git a/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt index 31bc8116700..be50b8d67a5 100644 --- a/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt @@ -15,6 +15,7 @@ vespa_add_library(searchcore_pcommon STATIC indexschema_inspector.cpp monitored_refcount.cpp operation_rate_tracker.cpp + pendinglidtracker.cpp select_utils.cpp selectcontext.cpp selectpruner.cpp diff --git a/searchcore/src/vespa/searchcore/proton/server/pendinglidtracker.cpp b/searchcore/src/vespa/searchcore/proton/common/pendinglidtracker.cpp index 79bf970aeac..79bf970aeac 100644 --- a/searchcore/src/vespa/searchcore/proton/server/pendinglidtracker.cpp +++ b/searchcore/src/vespa/searchcore/proton/common/pendinglidtracker.cpp diff --git a/searchcore/src/vespa/searchcore/proton/server/pendinglidtracker.h b/searchcore/src/vespa/searchcore/proton/common/pendinglidtracker.h index 4fc718fd465..4fc718fd465 100644 --- a/searchcore/src/vespa/searchcore/proton/server/pendinglidtracker.h +++ b/searchcore/src/vespa/searchcore/proton/common/pendinglidtracker.h diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.cpp b/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.cpp index 65a19a72fb9..20348ea4710 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.cpp +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.cpp @@ -66,7 +66,7 @@ DocumentIterator::checkMeta(const search::DocumentMetaData &meta) const } DocumentIterator::DocumentIterator(const storage::spi::Bucket &bucket, - document::FieldSet::SP fields, + const document::FieldSet& fields, const storage::spi::Selection &selection, storage::spi::IncludedVersions versions, ssize_t defaultSerializedSize, @@ -75,10 +75,10 @@ DocumentIterator::DocumentIterator(const storage::spi::Bucket &bucket, : _bucket(bucket), _selection(selection), _versions(versions), - _fields(std::move(fields)), + _fields(fields.clone()), _defaultSerializedSize((readConsistency == ReadConsistency::WEAK) ? defaultSerializedSize : -1), _readConsistency(readConsistency), - _metaOnly(_fields->getType() == document::FieldSet::Type::NONE), + _metaOnly(fields.getType() == document::FieldSet::Type::NONE), _ignoreMaxBytes((readConsistency == ReadConsistency::WEAK) && ignoreMaxBytes), _fetchedData(false), _sources(), @@ -90,9 +90,9 @@ DocumentIterator::DocumentIterator(const storage::spi::Bucket &bucket, DocumentIterator::~DocumentIterator() = default; void -DocumentIterator::add(const IDocumentRetriever::SP &retriever) +DocumentIterator::add(IDocumentRetriever::SP retriever) { - _sources.push_back(retriever); + _sources.push_back(std::move(retriever)); } IterateResult diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.h b/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.h index 4ae3839bfe8..ec63b52612d 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.h +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.h @@ -19,7 +19,7 @@ private: const storage::spi::Bucket _bucket;; const storage::spi::Selection _selection; const storage::spi::IncludedVersions _versions; - const document::FieldSet::SP _fields; + const document::FieldSet::UP _fields; const ssize_t _defaultSerializedSize; const ReadConsistency _readConsistency; const bool _metaOnly; @@ -35,12 +35,12 @@ private: bool isWeakRead() const { return _readConsistency == ReadConsistency::WEAK; } public: - DocumentIterator(const storage::spi::Bucket &bucket, document::FieldSet::SP fields, + DocumentIterator(const storage::spi::Bucket &bucket, const document::FieldSet& fields, const storage::spi::Selection &selection, storage::spi::IncludedVersions versions, ssize_t defaultSerializedSize, bool ignoreMaxBytes, ReadConsistency readConsistency=ReadConsistency::STRONG); ~DocumentIterator(); - void add(const IDocumentRetriever::SP &retriever); + void add(IDocumentRetriever::SP retriever); storage::spi::IterateResult iterate(size_t maxBytes); }; diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/ipersistencehandler.h b/searchcore/src/vespa/searchcore/proton/persistenceengine/ipersistencehandler.h index a4bede28d7b..7e3bb903f72 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/ipersistencehandler.h +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/ipersistencehandler.h @@ -26,6 +26,7 @@ protected: public: using UP = std::unique_ptr<IPersistenceHandler>; using SP = std::shared_ptr<IPersistenceHandler>; + /// Note that you can not move awaythe handlers in the vector. using RetrieversSP = std::shared_ptr<std::vector<IDocumentRetriever::SP> >; IPersistenceHandler(const IPersistenceHandler &) = delete; IPersistenceHandler & operator = (const IPersistenceHandler &) = delete; diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp index 14a56e7e0bb..91fb04df0f2 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp @@ -452,20 +452,20 @@ PersistenceEngine::get(const Bucket& b, const document::FieldSet& fields, const PersistenceEngine::CreateIteratorResult -PersistenceEngine::createIterator(const Bucket &bucket, FieldSetSP fields, const Selection &selection, - IncludedVersions versions, Context &context) +PersistenceEngine::createIterator(const Bucket &bucket, const document::FieldSet& fields, const Selection &selection, + IncludedVersions versions, Context & context) { std::shared_lock<std::shared_timed_mutex> rguard(_rwMutex); HandlerSnapshot snapshot = getHandlerSnapshot(rguard, bucket.getBucketSpace()); - auto entry = std::make_unique<IteratorEntry>(context.getReadConsistency(), bucket, std::move(fields), selection, + auto entry = std::make_unique<IteratorEntry>(context.getReadConsistency(), bucket, fields, selection, versions, _defaultSerializedSize, _ignoreMaxBytes); entry->bucket_guards.reserve(snapshot.size()); for (PersistenceHandlerSequence & handlers = snapshot.handlers(); handlers.valid(); handlers.next()) { entry->bucket_guards.push_back(handlers.get()->lockBucket(bucket)); IPersistenceHandler::RetrieversSP retrievers = handlers.get()->getDocumentRetrievers(context.getReadConsistency()); - for (size_t i = 0; i < retrievers->size(); ++i) { - entry->it.add((*retrievers)[i]); + for (const auto & retriever : *retrievers) { + entry->it.add(retriever); } } entry->handler_sequence = HandlerSnapshot::release(std::move(snapshot)); diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h index a874d91eb20..230f8c411aa 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h @@ -45,10 +45,10 @@ private: DocumentIterator it; bool in_use; std::vector<BucketGuard::UP> bucket_guards; - IteratorEntry(storage::spi::ReadConsistency readConsistency, const Bucket &b, FieldSetSP f, + IteratorEntry(storage::spi::ReadConsistency readConsistency, const Bucket &b, const document::FieldSet& f, const Selection &s, IncludedVersions v, ssize_t defaultSerializedSize, bool ignoreMaxBytes) : handler_sequence(), - it(b, std::move(f), s, v, defaultSerializedSize, ignoreMaxBytes, readConsistency), + it(b, f, s, v, defaultSerializedSize, ignoreMaxBytes, readConsistency), in_use(false), bucket_guards() {} }; @@ -105,8 +105,8 @@ public: void removeAsync(const Bucket&, Timestamp, const document::DocumentId&, Context&, OperationComplete::UP) override; void updateAsync(const Bucket&, Timestamp, storage::spi::DocumentUpdateSP, Context&, OperationComplete::UP) override; GetResult get(const Bucket&, const document::FieldSet&, const document::DocumentId&, Context&) const override; - CreateIteratorResult - createIterator(const Bucket &bucket, FieldSetSP, const Selection &, IncludedVersions, Context &context) override; + CreateIteratorResult createIterator(const Bucket&, const document::FieldSet&, const Selection&, + IncludedVersions, Context&) override; IterateResult iterate(IteratorId, uint64_t maxByteSize, Context&) const override; Result destroyIterator(IteratorId, Context&) override; diff --git a/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt index bfcd613ed0a..0a3c1015d3a 100644 --- a/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt @@ -70,7 +70,6 @@ vespa_add_library(searchcore_server STATIC minimal_document_retriever.cpp move_operation_limiter.cpp operationdonecontext.cpp - pendinglidtracker.cpp persistencehandlerproxy.cpp prepare_restart_handler.cpp proton.cpp diff --git a/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.cpp b/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.cpp index d25570794fe..2c3f9988fa3 100644 --- a/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.cpp @@ -37,7 +37,7 @@ CombiningFeedView::CombiningFeedView(const std::vector<IFeedView::SP> &views, _views(views), _metaStores(), _calc(calc), - _clusterUp(calc.get() != NULL && calc->clusterUp()), + _clusterUp(calc && calc->clusterUp()), _forceReady(!_clusterUp || !hasNotReadyFeedView()), _bucketSpace(bucketSpace) { @@ -45,21 +45,19 @@ CombiningFeedView::CombiningFeedView(const std::vector<IFeedView::SP> &views, for (const auto &view : views) { _metaStores.push_back(view->getDocumentMetaStorePtr()); } - assert(getReadyFeedView() != NULL); - assert(getRemFeedView() != NULL); + assert(getReadyFeedView() != nullptr); + assert(getRemFeedView() != nullptr); if (hasNotReadyFeedView()) { - assert(getNotReadyFeedView() != NULL); + assert(getNotReadyFeedView() != nullptr); } } -CombiningFeedView::~CombiningFeedView() -{ -} +CombiningFeedView::~CombiningFeedView() = default; const ISimpleDocumentMetaStore * CombiningFeedView::getDocumentMetaStorePtr() const { - return NULL; + return nullptr; } void @@ -76,12 +74,11 @@ CombiningFeedView::findPrevDbdId(const document::GlobalId &gid, if (subDbId == skipSubDbId) continue; const documentmetastore::IStore *metaStore = _metaStores[subDbId]; - if (metaStore == NULL) + if (metaStore == nullptr) continue; documentmetastore::IStore::Result inspectRes(metaStore->inspectExisting(gid)); if (inspectRes._found) { - op.setPrevDbDocumentId(DbDocumentId(subDbId, - inspectRes._lid)); + op.setPrevDbDocumentId(DbDocumentId(subDbId, inspectRes._lid)); op.setPrevMarkedAsRemoved(subDbId == getRemFeedViewId()); op.setPrevTimestamp(inspectRes._timestamp); break; diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp index 35c72db0c6f..6093ef06c5b 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp @@ -736,18 +736,7 @@ BucketGuard::UP DocumentDB::lockBucket(const document::BucketId &bucket) std::shared_ptr<std::vector<IDocumentRetriever::SP> > DocumentDB::getDocumentRetrievers(IDocumentRetriever::ReadConsistency consistency) { - std::shared_ptr<std::vector<IDocumentRetriever::SP> > list = _subDBs.getRetrievers(); - - if (consistency == IDocumentRetriever::ReadConsistency::STRONG) { - std::shared_ptr<std::vector<IDocumentRetriever::SP> > wrappedList = std::make_shared<std::vector<IDocumentRetriever::SP>>(); - wrappedList->reserve(list->size()); - for (const IDocumentRetriever::SP & retriever : *list) { - wrappedList->push_back(std::make_shared<CommitAndWaitDocumentRetriever>(retriever, _visibility)); - } - return wrappedList; - } else { - return list; - } + return _subDBs.getRetrievers(consistency, _visibility); } SerialNum diff --git a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp index db69704663f..536180fbab4 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp @@ -129,16 +129,32 @@ DocumentSubDBCollection::createRetrievers() namespace { IDocumentRetriever::SP -wrapRetriever(const IDocumentRetriever::SP &retriever, ICommitable &commit) +wrapRetriever(IDocumentRetriever::SP retriever, ICommitable &commit) { - return std::make_shared<CommitAndWaitDocumentRetriever>(retriever, commit); + return std::make_shared<CommitAndWaitDocumentRetriever>(std::move(retriever), commit); } } +DocumentSubDBCollection::RetrieversSP +DocumentSubDBCollection::getRetrievers(IDocumentRetriever::ReadConsistency consistency, ICommitable & visibilityHandler) { + RetrieversSP list = _retrievers.get(); + + if (consistency == IDocumentRetriever::ReadConsistency::STRONG) { + auto wrappedList = std::make_shared<std::vector<IDocumentRetriever::SP>>(); + wrappedList->reserve(list->size()); + assert(list->size() == 3); + wrappedList->push_back(wrapRetriever((*list)[_readySubDbId], visibilityHandler)); + wrappedList->push_back((*list)[_remSubDbId]); + wrappedList->push_back(wrapRetriever((*list)[_notReadySubDbId], visibilityHandler)); + return wrappedList; + } else { + return list; + } +} void DocumentSubDBCollection::maintenanceSync(MaintenanceController &mc, ICommitable &commit) { - RetrieversSP retrievers = getRetrievers(); + RetrieversSP retrievers = _retrievers.get(); MaintenanceDocumentSubDB readySubDB(getReadySubDB()->getName(), _readySubDbId, getReadySubDB()->getDocumentMetaStoreContext().getSP(), @@ -162,10 +178,9 @@ DocumentSubDBCollection::createInitializer(const DocumentDBConfig &configSnapsho SerialNum configSerialNum, const index::IndexConfig & indexCfg) { - DocumentSubDbCollectionInitializer::SP task = std::make_shared<DocumentSubDbCollectionInitializer>(); + auto task = std::make_shared<DocumentSubDbCollectionInitializer>(); for (auto subDb : _subDBs) { - DocumentSubDbInitializer::SP subTask(subDb->createInitializer(configSnapshot, configSerialNum, indexCfg)); - task->add(subTask); + task->add(subDb->createInitializer(configSnapshot, configSerialNum, indexCfg)); } return task; } diff --git a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h index 2936051538d..83ebef18274 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h @@ -4,6 +4,7 @@ #include <vespa/searchcore/proton/reprocessing/reprocessingrunner.h> #include <vespa/searchcore/proton/bucketdb/bucketdbhandler.h> #include <vespa/searchcore/proton/common/hw_info.h> +#include <vespa/searchcore/proton/persistenceengine/i_document_retriever.h> #include <vespa/searchcommon/common/growstrategy.h> #include <vespa/searchlib/common/serialnum.h> #include <vespa/vespalib/util/varholder.h> @@ -121,9 +122,7 @@ public: void maintenanceSync(MaintenanceController &mc, ICommitable &commit); // Internally synchronized - RetrieversSP getRetrievers() { - return _retrievers.get(); - } + RetrieversSP getRetrievers(IDocumentRetriever::ReadConsistency consistency, ICommitable & visibilityHandler); IDocumentSubDB *getReadySubDB() { return _subDBs[_readySubDbId]; } const IDocumentSubDB *getReadySubDB() const { return _subDBs[_readySubDbId]; } diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb_configurer.cpp b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb_configurer.cpp index 36d5ec31a71..697219db563 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb_configurer.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb_configurer.cpp @@ -20,7 +20,7 @@ FastAccessDocSubDBConfigurer::reconfigureFeedView(const FastAccessFeedView::SP & const std::shared_ptr<const DocumentTypeRepo> &repo, const IAttributeWriter::SP &writer) { - _feedView.set(FastAccessFeedView::SP(new FastAccessFeedView( + _feedView.set(std::make_shared<FastAccessFeedView>( StoreOnlyFeedView::Context(curr->getSummaryAdapter(), schema, curr->getDocumentMetaStore(), @@ -30,8 +30,7 @@ FastAccessDocSubDBConfigurer::reconfigureFeedView(const FastAccessFeedView::SP & curr->getLidReuseDelayer(), curr->getCommitTimeTracker()), curr->getPersistentParams(), - FastAccessFeedView::Context(writer, - curr->getDocIdLimit())))); + FastAccessFeedView::Context(writer,curr->getDocIdLimit()))); } FastAccessDocSubDBConfigurer::FastAccessDocSubDBConfigurer(FeedViewVarHolder &feedView, @@ -43,9 +42,7 @@ FastAccessDocSubDBConfigurer::FastAccessDocSubDBConfigurer(FeedViewVarHolder &fe { } -FastAccessDocSubDBConfigurer::~FastAccessDocSubDBConfigurer() -{ -} +FastAccessDocSubDBConfigurer::~FastAccessDocSubDBConfigurer() = default; IReprocessingInitializer::UP FastAccessDocSubDBConfigurer::reconfigure(const DocumentDBConfig &newConfig, @@ -53,8 +50,7 @@ FastAccessDocSubDBConfigurer::reconfigure(const DocumentDBConfig &newConfig, const AttributeCollectionSpec &attrSpec) { FastAccessFeedView::SP oldView = _feedView.get(); - IAttributeWriter::SP writer = - _factory->create(oldView->getAttributeWriter(), attrSpec); + IAttributeWriter::SP writer = _factory->create(oldView->getAttributeWriter(), attrSpec); reconfigureFeedView(oldView, newConfig.getSchemaSP(), newConfig.getDocumentTypeRepoSP(), writer); const document::DocumentType *newDocType = newConfig.getDocumentType(); diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h index be2ed9af126..1d48785b8a9 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h @@ -7,7 +7,7 @@ #include "isummaryadapter.h" #include "replaypacketdispatcher.h" #include "searchcontext.h" -#include "pendinglidtracker.h" +#include <vespa/searchcore/proton/common/pendinglidtracker.h> #include <vespa/searchcore/proton/common/doctypename.h> #include <vespa/searchcore/proton/attribute/ifieldupdatecallback.h> #include <vespa/searchcore/proton/common/feeddebugger.h> diff --git a/searchcore/src/vespa/searchcore/proton/server/visibilityhandler.cpp b/searchcore/src/vespa/searchcore/proton/server/visibilityhandler.cpp index 2865d8c0536..5cc5743041d 100644 --- a/searchcore/src/vespa/searchcore/proton/server/visibilityhandler.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/visibilityhandler.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "visibilityhandler.h" +#include <vespa/searchcore/proton/common/pendinglidtracker.h> #include <vespa/vespalib/util/isequencedtaskexecutor.h> #include <vespa/vespalib/util/closuretask.h> diff --git a/searchcore/src/vespa/searchcore/proton/test/dummy_feed_view.h b/searchcore/src/vespa/searchcore/proton/test/dummy_feed_view.h index 556f4188afa..63d292a8cfd 100644 --- a/searchcore/src/vespa/searchcore/proton/test/dummy_feed_view.h +++ b/searchcore/src/vespa/searchcore/proton/test/dummy_feed_view.h @@ -16,7 +16,7 @@ struct DummyFeedView : public IFeedView return _docTypeRepo; } const ISimpleDocumentMetaStore *getDocumentMetaStorePtr() const override { - return std::nullptr_t(); + return nullptr; } void preparePut(PutOperation &) override {} void handlePut(FeedToken, const PutOperation &) override {} diff --git a/storage/src/tests/common/teststorageapp.cpp b/storage/src/tests/common/teststorageapp.cpp index 1847de0e84f..9fcf1049e1b 100644 --- a/storage/src/tests/common/teststorageapp.cpp +++ b/storage/src/tests/common/teststorageapp.cpp @@ -171,17 +171,19 @@ TestServiceLayerApp::TestServiceLayerApp(DiskCount dc, NodeIndex index, assert(dc > 0); } -TestServiceLayerApp::~TestServiceLayerApp() = default; +TestServiceLayerApp::~TestServiceLayerApp() {} void TestServiceLayerApp::setupDummyPersistence() { - auto provider = std::make_unique<spi::dummy::DummyPersistence>(getTypeRepo(), _compReg.getDiskCount()); + spi::PersistenceProvider::UP provider(new spi::dummy::DummyPersistence( + getTypeRepo(), _compReg.getDiskCount())); setPersistenceProvider(std::move(provider)); } void -TestServiceLayerApp::setPersistenceProvider(PersistenceProviderUP provider) +TestServiceLayerApp::setPersistenceProvider( + spi::PersistenceProvider::UP provider) { _partitions = provider->getPartitionStates().getList(); assert(spi::PartitionId(_compReg.getDiskCount()) == _partitions.size()); diff --git a/storage/src/tests/common/teststorageapp.h b/storage/src/tests/common/teststorageapp.h index 218e7352f04..e567206c371 100644 --- a/storage/src/tests/common/teststorageapp.h +++ b/storage/src/tests/common/teststorageapp.h @@ -107,9 +107,8 @@ private: class TestServiceLayerApp : public TestStorageApp { - using PersistenceProviderUP = std::unique_ptr<spi::PersistenceProvider>; ServiceLayerComponentRegisterImpl& _compReg; - PersistenceProviderUP _persistenceProvider; + spi::PersistenceProvider::UP _persistenceProvider; spi::PartitionStateList _partitions; public: @@ -119,7 +118,7 @@ public: ~TestServiceLayerApp(); void setupDummyPersistence(); - void setPersistenceProvider(PersistenceProviderUP); + void setPersistenceProvider(spi::PersistenceProvider::UP); ServiceLayerComponentRegisterImpl& getComponentRegister() { return _compReg; } diff --git a/storage/src/tests/persistence/common/persistenceproviderwrapper.cpp b/storage/src/tests/persistence/common/persistenceproviderwrapper.cpp index 67a1c41a9ef..dd9ce6e6cba 100644 --- a/storage/src/tests/persistence/common/persistenceproviderwrapper.cpp +++ b/storage/src/tests/persistence/common/persistenceproviderwrapper.cpp @@ -147,9 +147,11 @@ PersistenceProviderWrapper::get(const spi::Bucket& bucket, } spi::CreateIteratorResult -PersistenceProviderWrapper::createIterator(const spi::Bucket &bucket, FieldSetSP fields, const spi::Selection &sel, +PersistenceProviderWrapper::createIterator(const spi::Bucket& bucket, + const document::FieldSet& fields, + const spi::Selection& sel, spi::IncludedVersions versions, - spi::Context &context) + spi::Context& context) { // TODO: proper printing of FieldSet and Selection diff --git a/storage/src/tests/persistence/common/persistenceproviderwrapper.h b/storage/src/tests/persistence/common/persistenceproviderwrapper.h index 75712750d68..21e5d8016aa 100644 --- a/storage/src/tests/persistence/common/persistenceproviderwrapper.h +++ b/storage/src/tests/persistence/common/persistenceproviderwrapper.h @@ -100,9 +100,8 @@ public: spi::UpdateResult update(const spi::Bucket&, spi::Timestamp, spi::DocumentUpdateSP, spi::Context&) override; spi::GetResult get(const spi::Bucket&, const document::FieldSet&, const spi::DocumentId&, spi::Context&) const override; - spi::CreateIteratorResult - createIterator(const spi::Bucket &bucket, FieldSetSP, const spi::Selection &, spi::IncludedVersions versions, - spi::Context &context) override; + spi::CreateIteratorResult createIterator(const spi::Bucket&, const document::FieldSet&, const spi::Selection&, + spi::IncludedVersions versions, spi::Context&) override; spi::IterateResult iterate(spi::IteratorId, uint64_t maxByteSize, spi::Context&) const override; spi::Result destroyIterator(spi::IteratorId, spi::Context&) override; diff --git a/storage/src/tests/persistence/filestorage/mergeblockingtest.cpp b/storage/src/tests/persistence/filestorage/mergeblockingtest.cpp index c73ae7e506c..d9582cec585 100644 --- a/storage/src/tests/persistence/filestorage/mergeblockingtest.cpp +++ b/storage/src/tests/persistence/filestorage/mergeblockingtest.cpp @@ -15,7 +15,9 @@ namespace storage { struct MergeBlockingTest : public FileStorTestFixture { void setupDisks() { FileStorTestFixture::setupPersistenceThreads(1); - _node->setPersistenceProvider(std::make_unique<spi::dummy::DummyPersistence>(_node->getTypeRepo(), 1)); + _node->setPersistenceProvider( + spi::PersistenceProvider::UP( + new spi::dummy::DummyPersistence(_node->getTypeRepo(), 1))); } void SetUp() override; diff --git a/storage/src/tests/persistence/filestorage/operationabortingtest.cpp b/storage/src/tests/persistence/filestorage/operationabortingtest.cpp index 7810a595012..93c484368de 100644 --- a/storage/src/tests/persistence/filestorage/operationabortingtest.cpp +++ b/storage/src/tests/persistence/filestorage/operationabortingtest.cpp @@ -77,19 +77,18 @@ spi::LoadType defaultLoadType(0, "default"); } struct OperationAbortingTest : FileStorTestFixture { - std::unique_ptr<spi::dummy::DummyPersistence> _dummyProvider; - BlockingMockProvider * _blockingProvider; + spi::PersistenceProvider::UP _dummyProvider; + BlockingMockProvider* _blockingProvider; std::unique_ptr<vespalib::Barrier> _queueBarrier; std::unique_ptr<vespalib::Barrier> _completionBarrier; void setupProviderAndBarriers(uint32_t queueBarrierThreads) { FileStorTestFixture::setupPersistenceThreads(1); - _dummyProvider = std::make_unique<spi::dummy::DummyPersistence>(_node->getTypeRepo(), 1); - _queueBarrier = std::make_unique<vespalib::Barrier>(queueBarrierThreads); - _completionBarrier = std::make_unique<vespalib::Barrier>(2); - auto blockingProvider = std::make_unique<BlockingMockProvider>(*_dummyProvider, *_queueBarrier, *_completionBarrier); - _blockingProvider = blockingProvider.get(); - _node->setPersistenceProvider(std::move(blockingProvider)); + _dummyProvider.reset(new spi::dummy::DummyPersistence(_node->getTypeRepo(), 1)); + _queueBarrier.reset(new vespalib::Barrier(queueBarrierThreads)); + _completionBarrier.reset(new vespalib::Barrier(2)); + _blockingProvider = new BlockingMockProvider(*_dummyProvider, *_queueBarrier, *_completionBarrier); + _node->setPersistenceProvider(spi::PersistenceProvider::UP(_blockingProvider)); } void validateReplies(DummyStorageLink& link, size_t repliesTotal, diff --git a/storage/src/tests/persistence/persistencetestutils.cpp b/storage/src/tests/persistence/persistencetestutils.cpp index f50fbb0c8e8..504767e68c7 100644 --- a/storage/src/tests/persistence/persistencetestutils.cpp +++ b/storage/src/tests/persistence/persistencetestutils.cpp @@ -231,9 +231,9 @@ PersistenceTestUtils::doGetOnDisk( document::DocumentUpdate::SP PersistenceTestUtils::createBodyUpdate(const document::DocumentId& docId, const document::FieldValue& updateValue) { - const DocumentType* docType(getTypeRepo()->getDocumentType("testdoctype1")); - auto update = std::make_shared<document::DocumentUpdate>(*getTypeRepo(), *docType, docId); - auto assignUpdate = std::make_shared<document::AssignValueUpdate>(updateValue); + const DocumentType* docType(_env->_component.getTypeRepo()->getDocumentType("testdoctype1")); + document::DocumentUpdate::SP update(new document::DocumentUpdate(*_env->_component.getTypeRepo(), *docType, docId)); + std::shared_ptr<document::AssignValueUpdate> assignUpdate(new document::AssignValueUpdate(updateValue)); document::FieldUpdate fieldUpdate(docType->getField("content")); fieldUpdate.addUpdate(*assignUpdate); update->addUpdate(fieldUpdate); @@ -243,9 +243,9 @@ PersistenceTestUtils::createBodyUpdate(const document::DocumentId& docId, const document::DocumentUpdate::SP PersistenceTestUtils::createHeaderUpdate(const document::DocumentId& docId, const document::FieldValue& updateValue) { - const DocumentType* docType(getTypeRepo()->getDocumentType("testdoctype1")); - auto update = std::make_shared<document::DocumentUpdate>(*getTypeRepo(), *docType, docId); - auto assignUpdate = std::make_shared<document::AssignValueUpdate>(updateValue); + const DocumentType* docType(_env->_component.getTypeRepo()->getDocumentType("testdoctype1")); + document::DocumentUpdate::SP update(new document::DocumentUpdate(*_env->_component.getTypeRepo(), *docType, docId)); + std::shared_ptr<document::AssignValueUpdate> assignUpdate(new document::AssignValueUpdate(updateValue)); document::FieldUpdate fieldUpdate(docType->getField("headerval")); fieldUpdate.addUpdate(*assignUpdate); update->addUpdate(fieldUpdate); @@ -253,7 +253,8 @@ PersistenceTestUtils::createHeaderUpdate(const document::DocumentId& docId, cons } uint16_t -PersistenceTestUtils::getDiskFromBucketDatabaseIfUnset(const document::Bucket& bucket, uint16_t disk) +PersistenceTestUtils::getDiskFromBucketDatabaseIfUnset(const document::Bucket& bucket, + uint16_t disk) { if (disk == 0xffff) { StorBucketDatabase::WrappedEntry entry( @@ -341,7 +342,7 @@ PersistenceTestUtils::clearBody(document::Document& doc) //doc->getBody().clear(); vespalib::nbostream stream; doc.serializeHeader(stream); - doc.deserialize(*getTypeRepo(), stream); + doc.deserialize(*_env->_component.getTypeRepo(), stream); } document::Document::UP diff --git a/storage/src/tests/persistence/persistencetestutils.h b/storage/src/tests/persistence/persistencetestutils.h index 3d25a205017..6cee3b79ab8 100644 --- a/storage/src/tests/persistence/persistencetestutils.h +++ b/storage/src/tests/persistence/persistencetestutils.h @@ -101,7 +101,7 @@ public: FileStorHandler& fsHandler() { return *_env->_handler; } FileStorMetrics& metrics() { return _env->_metrics; } MessageKeeper& messageKeeper() { return _env->_messageKeeper; } - std::shared_ptr<const document::DocumentTypeRepo> getTypeRepo() { return _env->_component.getTypeRepo()->documentTypeRepo; } + std::shared_ptr<const document::DocumentTypeRepo> getTypeRepo() { return _env->_component.getTypeRepo(); } StorageComponent& getComponent() { return _env->_component; } TestServiceLayerApp& getNode() { return _env->_node; } diff --git a/storage/src/vespa/storage/common/storagecomponent.cpp b/storage/src/vespa/storage/common/storagecomponent.cpp index 3846fe3a9c0..21a4b8eea64 100644 --- a/storage/src/vespa/storage/common/storagecomponent.cpp +++ b/storage/src/vespa/storage/common/storagecomponent.cpp @@ -2,22 +2,17 @@ #include "storagecomponent.h" #include <vespa/storage/storageserver/prioritymapper.h> + #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vdslib/distribution/distribution.h> -#include <vespa/document/fieldset/fieldsetrepo.h> namespace storage { -StorageComponent::Repos::Repos(std::shared_ptr<const document::DocumentTypeRepo> repo) - : documentTypeRepo(std::move(repo)), - fieldSetRepo(std::make_shared<document::FieldSetRepo>(*documentTypeRepo)) -{} - -StorageComponent::Repos::~Repos() = default; - // Defined in cpp file to allow unique pointers of unknown type in header. -StorageComponent::~StorageComponent() = default; +StorageComponent::~StorageComponent() +{ +} void StorageComponent::setNodeInfo(vespalib::stringref clusterName, @@ -31,11 +26,10 @@ StorageComponent::setNodeInfo(vespalib::stringref clusterName, } void -StorageComponent::setDocumentTypeRepo(std::shared_ptr<const document::DocumentTypeRepo> docTypeRepo) +StorageComponent::setDocumentTypeRepo(DocumentTypeRepoSP repo) { - auto repo = std::make_shared<Repos>(std::move(docTypeRepo)); std::lock_guard guard(_lock); - _repos = std::move(repo); + _docTypeRepo = repo; } void @@ -84,7 +78,7 @@ StorageComponent::StorageComponent(StorageComponentRegister& compReg, _clusterName(), _nodeType(nullptr), _index(0), - _repos(), + _docTypeRepo(), _loadTypes(), _priorityMapper(new PriorityMapper), _bucketIdFactory(), @@ -122,11 +116,11 @@ StorageComponent::getPriority(const documentapi::LoadType& lt) const return _priorityMapper->getPriority(lt); } -std::shared_ptr<StorageComponent::Repos> +StorageComponent::DocumentTypeRepoSP StorageComponent::getTypeRepo() const { std::lock_guard guard(_lock); - return _repos; + return _docTypeRepo; } StorageComponent::LoadTypeSetSP diff --git a/storage/src/vespa/storage/common/storagecomponent.h b/storage/src/vespa/storage/common/storagecomponent.h index e0b1dc74d7f..821cd43f21d 100644 --- a/storage/src/vespa/storage/common/storagecomponent.h +++ b/storage/src/vespa/storage/common/storagecomponent.h @@ -42,7 +42,6 @@ namespace vespa::config::content::core::internal { } namespace document { class DocumentTypeRepo; - class FieldSetRepo; } namespace documentapi { class LoadType; @@ -59,14 +58,9 @@ struct StorageComponentRegister; class StorageComponent : public framework::Component { public: - struct Repos { - explicit Repos(std::shared_ptr<const document::DocumentTypeRepo> repo); - ~Repos(); - const std::shared_ptr<const document::DocumentTypeRepo> documentTypeRepo; - const std::shared_ptr<const document::FieldSetRepo> fieldSetRepo; - }; using UP = std::unique_ptr<StorageComponent>; using PriorityConfig = vespa::config::content::core::internal::InternalStorPrioritymappingType; + using DocumentTypeRepoSP = std::shared_ptr<const document::DocumentTypeRepo>; using LoadTypeSetSP = std::shared_ptr<documentapi::LoadTypeSet>; using DistributionSP = std::shared_ptr<lib::Distribution>; @@ -74,7 +68,9 @@ public: * Node type is supposed to be set immediately, and never be updated. * Thus it does not need to be threadsafe. Should never be used before set. */ - void setNodeInfo(vespalib::stringref clusterName, const lib::NodeType& nodeType, uint16_t index); + void setNodeInfo(vespalib::stringref clusterName, + const lib::NodeType& nodeType, + uint16_t index); /** * Node state updater is supposed to be set immediately, and never be @@ -82,14 +78,14 @@ public: * before set. */ void setNodeStateUpdater(NodeStateUpdater& updater); - void setDocumentTypeRepo(std::shared_ptr<const document::DocumentTypeRepo>); + void setDocumentTypeRepo(DocumentTypeRepoSP); void setLoadTypes(LoadTypeSetSP); void setPriorityConfig(const PriorityConfig&); void setBucketIdFactory(const document::BucketIdFactory&); void setDistribution(DistributionSP); StorageComponent(StorageComponentRegister&, vespalib::stringref name); - ~StorageComponent() override; + virtual ~StorageComponent(); vespalib::string getClusterName() const { return _clusterName; } const lib::NodeType& getNodeType() const { return *_nodeType; } @@ -98,7 +94,7 @@ public: vespalib::string getIdentity() const; - std::shared_ptr<Repos> getTypeRepo() const; + DocumentTypeRepoSP getTypeRepo() const; LoadTypeSetSP getLoadTypes() const; const document::BucketIdFactory& getBucketIdFactory() const { return _bucketIdFactory; } @@ -110,8 +106,7 @@ private: vespalib::string _clusterName; const lib::NodeType* _nodeType; uint16_t _index; - std::shared_ptr<Repos> _repos; - // TODO: move loadTypes and _distribution in to _repos so lock will only taken once and only copying one shared_ptr. + DocumentTypeRepoSP _docTypeRepo; LoadTypeSetSP _loadTypes; std::unique_ptr<PriorityMapper> _priorityMapper; document::BucketIdFactory _bucketIdFactory; diff --git a/storage/src/vespa/storage/config/distributorconfiguration.cpp b/storage/src/vespa/storage/config/distributorconfiguration.cpp index aa606cdc8b9..0c9988421a3 100644 --- a/storage/src/vespa/storage/config/distributorconfiguration.cpp +++ b/storage/src/vespa/storage/config/distributorconfiguration.cpp @@ -70,7 +70,7 @@ DistributorConfiguration::containsTimeStatement(const std::string& documentSelec { TimeVisitor visitor; try { - document::select::Parser parser(*_component.getTypeRepo()->documentTypeRepo, _component.getBucketIdFactory()); + document::select::Parser parser(*_component.getTypeRepo(), _component.getBucketIdFactory()); std::unique_ptr<document::select::Node> node = parser.parse(documentSelection); node->visit(visitor); diff --git a/storage/src/vespa/storage/distributor/distributor.cpp b/storage/src/vespa/storage/distributor/distributor.cpp index cfd8d7f1753..c74d4135556 100644 --- a/storage/src/vespa/storage/distributor/distributor.cpp +++ b/storage/src/vespa/storage/distributor/distributor.cpp @@ -108,7 +108,8 @@ Distributor::Distributor(DistributorComponentRegister& compReg, _must_send_updated_host_info(false) { _component.registerMetric(*_metrics); - _component.registerMetricUpdateHook(_metricUpdateHook, framework::SecondTime(0)); + _component.registerMetricUpdateHook(_metricUpdateHook, + framework::SecondTime(0)); _distributorStatusDelegate.registerStatusPage(); _bucketDBStatusDelegate.registerStatusPage(); hostInfoReporterRegistrar.registerReporter(&_hostInfoReporter); diff --git a/storage/src/vespa/storage/distributor/operations/external/removelocationoperation.cpp b/storage/src/vespa/storage/distributor/operations/external/removelocationoperation.cpp index 4c762cf4c23..ca1b6f266d6 100644 --- a/storage/src/vespa/storage/distributor/operations/external/removelocationoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/external/removelocationoperation.cpp @@ -38,7 +38,8 @@ RemoveLocationOperation::getBucketId( DistributorComponent& manager, const api::RemoveLocationCommand& cmd, document::BucketId& bid) { - document::select::Parser parser(*manager.getTypeRepo()->documentTypeRepo, manager.getBucketIdFactory()); + std::shared_ptr<const document::DocumentTypeRepo> repo = manager.getTypeRepo(); + document::select::Parser parser(*repo, manager.getBucketIdFactory()); document::BucketSelector bucketSel(manager.getBucketIdFactory()); std::unique_ptr<document::BucketSelector::BucketVector> exprResult diff --git a/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp b/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp index 3866ee4e6f7..41f452df801 100644 --- a/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp @@ -576,7 +576,7 @@ TwoPhaseUpdateOperation::processAndMatchTasCondition(DistributorMessageSender& s return true; // No condition; nothing to do here. } - document::select::Parser parser(*_manager.getTypeRepo()->documentTypeRepo, _manager.getBucketIdFactory()); + document::select::Parser parser(*_manager.getTypeRepo(), _manager.getBucketIdFactory()); std::unique_ptr<document::select::Node> selection; try { selection = parser.parse(_updateCmd->getCondition().getSelection()); diff --git a/storage/src/vespa/storage/persistence/bucketprocessor.cpp b/storage/src/vespa/storage/persistence/bucketprocessor.cpp index ea09fcfc348..c88b08612d7 100644 --- a/storage/src/vespa/storage/persistence/bucketprocessor.cpp +++ b/storage/src/vespa/storage/persistence/bucketprocessor.cpp @@ -47,11 +47,11 @@ BucketProcessor::iterateAll(spi::PersistenceProvider& provider, spi::Selection sel = spi::Selection(spi::DocumentSelection(documentSelection)); spi::CreateIteratorResult createIterResult(provider.createIterator( - bucket, - std::make_shared<document::AllFields>(), - sel, - versions, - context)); + bucket, + document::AllFields(), + sel, + versions, + context)); if (createIterResult.getErrorCode() != spi::Result::ErrorType::NONE) { vespalib::asciistream ss; diff --git a/storage/src/vespa/storage/persistence/mergehandler.cpp b/storage/src/vespa/storage/persistence/mergehandler.cpp index 612d4545a8a..70894858887 100644 --- a/storage/src/vespa/storage/persistence/mergehandler.cpp +++ b/storage/src/vespa/storage/persistence/mergehandler.cpp @@ -1,5 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + #include "mergehandler.h" #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vdslib/distribution/distribution.h> @@ -13,14 +14,17 @@ LOG_SETUP(".persistence.mergehandler"); namespace storage { -MergeHandler::MergeHandler(spi::PersistenceProvider& spi, PersistenceUtil& env) +MergeHandler::MergeHandler(spi::PersistenceProvider& spi, + PersistenceUtil& env) : _spi(spi), _env(env), _maxChunkSize(env._config.bucketMergeChunkSize) { } -MergeHandler::MergeHandler(spi::PersistenceProvider& spi, PersistenceUtil& env, uint32_t maxChunkSize) +MergeHandler::MergeHandler(spi::PersistenceProvider& spi, + PersistenceUtil& env, + uint32_t maxChunkSize) : _spi(spi), _env(env), _maxChunkSize(maxChunkSize) @@ -54,7 +58,9 @@ checkResult(const spi::Result& result, } void -checkResult(const spi::Result& result, const spi::Bucket& bucket, const char* op) +checkResult(const spi::Result& result, + const spi::Bucket& bucket, + const char* op) { if (result.hasError()) { vespalib::asciistream ss; @@ -118,11 +124,11 @@ MergeHandler::populateMetaData( spi::Selection sel(docSel); sel.setToTimestamp(spi::Timestamp(maxTimestamp.getTime())); spi::CreateIteratorResult createIterResult(_spi.createIterator( - bucket, - std::make_shared<document::NoFields>(), - sel, - spi::ALL_VERSIONS, - context)); + bucket, + document::NoFields(), + sel, + spi::ALL_VERSIONS, + context)); if (createIterResult.getErrorCode() != spi::Result::ErrorType::NONE) { std::ostringstream ss; @@ -136,7 +142,8 @@ MergeHandler::populateMetaData( IteratorGuard iteratorGuard(_spi, iteratorId, context); while (true) { - spi::IterateResult result(_spi.iterate(iteratorId, UINT64_MAX, context)); + spi::IterateResult result( + _spi.iterate(iteratorId, UINT64_MAX, context)); if (result.getErrorCode() != spi::Result::ErrorType::NONE) { std::ostringstream ss; ss << "Failed to iterate for " @@ -293,7 +300,8 @@ namespace { } int - countUnfilledEntries(const std::vector<api::ApplyBucketDiffCommand::Entry>& diff) + countUnfilledEntries( + const std::vector<api::ApplyBucketDiffCommand::Entry>& diff) { int count = 0; @@ -315,9 +323,11 @@ namespace { return value; } - api::StorageMessageAddress createAddress(const std::string& clusterName, uint16_t node) + api::StorageMessageAddress createAddress(const std::string& clusterName, + uint16_t node) { - return api::StorageMessageAddress(clusterName, lib::NodeType::STORAGE, node); + return api::StorageMessageAddress( + clusterName, lib::NodeType::STORAGE, node); } void assertContainedInBucket(const document::DocumentId& docId, @@ -360,11 +370,14 @@ MergeHandler::fetchLocalData( alreadyFilled += e._headerBlob.size() + e._bodyBlob.size(); } } - uint32_t remainingSize = _maxChunkSize - std::min(_maxChunkSize, alreadyFilled); - LOG(debug, "Diff of %s has already filled %u of max %u bytes, remaining size to fill is %u", + uint32_t remainingSize = _maxChunkSize - std::min(_maxChunkSize, + alreadyFilled); + LOG(debug, "Diff of %s has already filled %u of max %u bytes, " + "remaining size to fill is %u", bucket.toString().c_str(), alreadyFilled, _maxChunkSize, remainingSize); if (remainingSize == 0) { - LOG(debug, "Diff already at max chunk size, not fetching any local data"); + LOG(debug, + "Diff already at max chunk size, not fetching any local data"); return; } @@ -374,7 +387,7 @@ MergeHandler::fetchLocalData( sel.setTimestampSubset(slots); spi::CreateIteratorResult createIterResult( _spi.createIterator(bucket, - std::make_shared<document::AllFields>(), + document::AllFields(), sel, spi::NEWEST_DOCUMENT_OR_REMOVE, context)); @@ -396,7 +409,8 @@ MergeHandler::fetchLocalData( bool fetchedAllLocalData = false; bool chunkLimitReached = false; while (true) { - spi::IterateResult result(_spi.iterate(iteratorId, remainingSize, context)); + spi::IterateResult result( + _spi.iterate(iteratorId, remainingSize, context)); if (result.getErrorCode() != spi::Result::ErrorType::NONE) { std::ostringstream ss; ss << "Failed to iterate for " @@ -412,7 +426,8 @@ MergeHandler::fetchLocalData( { remainingSize -= list[i]->getSize(); LOG(spam, "Added %s, remainingSize is %u", - entries.back()->toString().c_str(), remainingSize); + entries.back()->toString().c_str(), + remainingSize); entries.push_back(std::move(list[i])); } else { LOG(spam, "Adding %s would exceed chunk size limit of %u; " @@ -438,7 +453,8 @@ MergeHandler::fetchLocalData( docEntry.toString().c_str()); std::vector<api::ApplyBucketDiffCommand::Entry>::iterator iter( - std::lower_bound(diff.begin(), diff.end(), + std::lower_bound(diff.begin(), + diff.end(), api::Timestamp(docEntry.getTimestamp()), DiffEntryTimestampPredicate())); assert(iter != diff.end()); @@ -550,8 +566,8 @@ MergeHandler::applyDiffLocally( std::vector<spi::DocEntry::UP> entries; populateMetaData(bucket, MAX_TIMESTAMP, entries, context); - std::shared_ptr<const document::DocumentTypeRepo> repo(_env._component.getTypeRepo()->documentTypeRepo); - assert(repo); + std::shared_ptr<const document::DocumentTypeRepo> repo(_env._component.getTypeRepo()); + assert(repo.get() != nullptr); uint32_t existingCount = entries.size(); uint32_t i = 0, j = 0; @@ -711,7 +727,8 @@ MergeHandler::processBucketMerge(const spi::Bucket& bucket, MergeStatus& status, // If nothing to update, we're done. if (status.diff.size() == 0) { - LOG(debug, "Done with merge of %s. No more entries in diff.", bucket.toString().c_str()); + LOG(debug, "Done with merge of %s. No more entries in diff.", + bucket.toString().c_str()); return status.reply; } @@ -738,8 +755,10 @@ MergeHandler::processBucketMerge(const spi::Bucket& bucket, MergeStatus& status, ? std::numeric_limits<uint32_t>().max() : _maxChunkSize); - cmd = std::make_shared<api::ApplyBucketDiffCommand>(bucket.getBucket(), nodes, maxSize); - cmd->setAddress(createAddress(_env._component.getClusterName(), nodes[1].index)); + cmd.reset(new api::ApplyBucketDiffCommand( + bucket.getBucket(), nodes, maxSize)); + cmd->setAddress(createAddress(_env._component.getClusterName(), + nodes[1].index)); findCandidates(bucket.getBucketId(), status, true, @@ -779,7 +798,8 @@ MergeHandler::processBucketMerge(const spi::Bucket& bucket, MergeStatus& status, for (std::map<uint16_t, uint32_t>::const_iterator it = counts.begin(); it != counts.end(); ++it) { - if (it->second >= uint32_t(_env._config.commonMergeChainOptimalizationMinimumSize) + if (it->second >= uint32_t( + _env._config.commonMergeChainOptimalizationMinimumSize) || counts.size() == 1) { LOG(spam, "Sending separate apply bucket diff for path %x " @@ -812,11 +832,15 @@ MergeHandler::processBucketMerge(const spi::Bucket& bucket, MergeStatus& status, (_env._config.enableMergeLocalNodeChooseDocsOptimalization ? std::numeric_limits<uint32_t>().max() : _maxChunkSize); - cmd = std::make_shared<api::ApplyBucketDiffCommand>(bucket.getBucket(), nodes, maxSize); - cmd->setAddress(createAddress(_env._component.getClusterName(), nodes[1].index)); + cmd.reset(new api::ApplyBucketDiffCommand( + bucket.getBucket(), nodes, maxSize)); + cmd->setAddress( + createAddress(_env._component.getClusterName(), + nodes[1].index)); // Add all the metadata, and thus use big limit. Max // data to fetch parameter will control amount added. - findCandidates(bucket.getBucketId(), status, true, it->first, newMask, maxSize, *cmd); + findCandidates(bucket.getBucketId(), status, true, + it->first, newMask, maxSize, *cmd); break; } } @@ -824,17 +848,22 @@ MergeHandler::processBucketMerge(const spi::Bucket& bucket, MergeStatus& status, // If we found no group big enough to handle on its own, do a common // merge to merge the remaining data. - if ( ! cmd ) { - cmd = std::make_shared<api::ApplyBucketDiffCommand>(bucket.getBucket(), status.nodeList, _maxChunkSize); - cmd->setAddress(createAddress(_env._component.getClusterName(), status.nodeList[1].index)); - findCandidates(bucket.getBucketId(), status, false, 0, 0, _maxChunkSize, *cmd); + if (cmd.get() == 0) { + cmd.reset(new api::ApplyBucketDiffCommand(bucket.getBucket(), + status.nodeList, + _maxChunkSize)); + cmd->setAddress(createAddress(_env._component.getClusterName(), + status.nodeList[1].index)); + findCandidates(bucket.getBucketId(), status, false, 0, 0, + _maxChunkSize, *cmd); } cmd->setPriority(status.context.getPriority()); cmd->setTimeout(status.timeout); if (applyDiffNeedLocalData(cmd->getDiff(), 0, true)) { framework::MilliSecTimer startTime(_env._component.getClock()); fetchLocalData(bucket, cmd->getLoadType(), cmd->getDiff(), 0, context); - _env._metrics.merge_handler_metrics.mergeDataReadLatency.addValue(startTime.getElapsedTimeAsDouble()); + _env._metrics.merge_handler_metrics.mergeDataReadLatency.addValue( + startTime.getElapsedTimeAsDouble()); } status.pendingId = cmd->getMsgId(); LOG(debug, "Sending %s", cmd->toString().c_str()); @@ -849,7 +878,8 @@ public: document::Bucket _bucket; bool _active; - MergeStateDeleter(FileStorHandler& handler, const document::Bucket& bucket) + MergeStateDeleter(FileStorHandler& handler, + const document::Bucket& bucket) : _handler(handler), _bucket(bucket), _active(true) @@ -876,7 +906,8 @@ MergeHandler::handleMergeBucket(api::MergeBucketCommand& cmd, MessageTracker::UP if (cmd.getNodes().size() < 2) { LOG(debug, "Attempt to merge a single instance of a bucket"); - tracker->fail(ReturnCode::ILLEGAL_PARAMETERS, "Cannot merge a single copy"); + tracker->fail(ReturnCode::ILLEGAL_PARAMETERS, + "Cannot merge a single copy"); return tracker; } @@ -923,7 +954,8 @@ MergeHandler::handleMergeBucket(api::MergeBucketCommand& cmd, MessageTracker::UP auto cmd2 = std::make_shared<api::GetBucketDiffCommand>(bucket.getBucket(), s->nodeList, s->maxTimestamp.getTime()); if (!buildBucketInfoList(bucket, cmd.getLoadType(), s->maxTimestamp, 0, cmd2->getDiff(), tracker->context())) { LOG(debug, "Bucket non-existing in db. Failing merge."); - tracker->fail(ReturnCode::BUCKET_DELETED, "Bucket not found in buildBucketInfo step"); + tracker->fail(ReturnCode::BUCKET_DELETED, + "Bucket not found in buildBucketInfo step"); return tracker; } _env._metrics.merge_handler_metrics.mergeMetadataReadLatency.addValue(s->startTime.getElapsedTimeAsDouble()); @@ -1084,7 +1116,8 @@ MergeHandler::handleGetBucketDiff(api::GetBucketDiffCommand& cmd, MessageTracker checkResult(_spi.createBucket(bucket, tracker->context()), bucket, "create bucket"); if (_env._fileStorHandler.isMerging(bucket.getBucket())) { - tracker->fail(ReturnCode::BUSY, "A merge is already running on this bucket."); + tracker->fail(ReturnCode::BUSY, + "A merge is already running on this bucket."); return tracker; } uint8_t index = findOwnIndex(cmd.getNodes(), _env._nodeIndex); @@ -1097,13 +1130,16 @@ MergeHandler::handleGetBucketDiff(api::GetBucketDiffCommand& cmd, MessageTracker index, local, tracker->context())) { LOG(debug, "Bucket non-existing in db. Failing merge."); - tracker->fail(ReturnCode::BUCKET_DELETED, "Bucket not found in buildBucketInfo step"); + tracker->fail(ReturnCode::BUCKET_DELETED, + "Bucket not found in buildBucketInfo step"); return tracker; } if (!mergeLists(remote, local, local)) { - LOG(error, "Diffing %s found suspect entries.", bucket.toString().c_str()); + LOG(error, "Diffing %s found suspect entries.", + bucket.toString().c_str()); } - _env._metrics.merge_handler_metrics.mergeMetadataReadLatency.addValue(startTime.getElapsedTimeAsDouble()); + _env._metrics.merge_handler_metrics.mergeMetadataReadLatency.addValue( + startTime.getElapsedTimeAsDouble()); // If last node in merge chain, we can send reply straight away if (index + 1u >= cmd.getNodes().size()) { @@ -1180,21 +1216,24 @@ namespace { bool operator()(const api::ApplyBucketDiffCommand::Entry& x, const api::ApplyBucketDiffCommand::Entry& y) { - return (x._entry._timestamp < y._entry._timestamp); + return (x._entry._timestamp + < y._entry._timestamp); } }; } // End of anonymous namespace void -MergeHandler::handleGetBucketDiffReply(api::GetBucketDiffReply& reply, MessageSender& sender) +MergeHandler::handleGetBucketDiffReply(api::GetBucketDiffReply& reply, + MessageSender& sender) { _env._metrics.getBucketDiffReply.inc(); spi::Bucket bucket(reply.getBucket(), spi::PartitionId(_env._partition)); LOG(debug, "GetBucketDiffReply(%s)", bucket.toString().c_str()); if (!_env._fileStorHandler.isMerging(bucket.getBucket())) { - LOG(warning, "Got GetBucketDiffReply for %s which we have no merge state for.", + LOG(warning, "Got GetBucketDiffReply for %s which we have no " + "merge state for.", bucket.toString().c_str()); return; } @@ -1348,7 +1387,8 @@ MergeHandler::handleApplyBucketDiff(api::ApplyBucketDiffCommand& cmd, MessageTra } void -MergeHandler::handleApplyBucketDiffReply(api::ApplyBucketDiffReply& reply,MessageSender& sender) +MergeHandler::handleApplyBucketDiffReply(api::ApplyBucketDiffReply& reply, + MessageSender& sender) { _env._metrics.applyBucketDiffReply.inc(); spi::Bucket bucket(reply.getBucket(), spi::PartitionId(_env._partition)); @@ -1356,7 +1396,8 @@ MergeHandler::handleApplyBucketDiffReply(api::ApplyBucketDiffReply& reply,Messag LOG(debug, "%s", reply.toString().c_str()); if (!_env._fileStorHandler.isMerging(bucket.getBucket())) { - LOG(warning, "Got ApplyBucketDiffReply for %s which we have no merge state for.", + LOG(warning, "Got ApplyBucketDiffReply for %s which we have no " + "merge state for.", bucket.toString().c_str()); return; } @@ -1374,19 +1415,25 @@ MergeHandler::handleApplyBucketDiffReply(api::ApplyBucketDiffReply& reply,Messag api::ReturnCode returnCode = reply.getResult(); try { if (reply.getResult().failed()) { - LOG(debug, "Got failed apply bucket diff reply %s", reply.toString().c_str()); + LOG(debug, "Got failed apply bucket diff reply %s", + reply.toString().c_str()); } else { assert(reply.getNodes().size() >= 2); uint8_t index = findOwnIndex(reply.getNodes(), _env._nodeIndex); if (applyDiffNeedLocalData(diff, index, false)) { framework::MilliSecTimer startTime(_env._component.getClock()); - fetchLocalData(bucket, reply.getLoadType(), diff, index, s.context); - _env._metrics.merge_handler_metrics.mergeDataReadLatency.addValue(startTime.getElapsedTimeAsDouble()); + fetchLocalData(bucket, reply.getLoadType(), diff, index, + s.context); + _env._metrics.merge_handler_metrics.mergeDataReadLatency.addValue( + startTime.getElapsedTimeAsDouble()); } if (applyDiffHasLocallyNeededData(diff, index)) { framework::MilliSecTimer startTime(_env._component.getClock()); - api::BucketInfo info(applyDiffLocally(bucket, reply.getLoadType(), diff, index, s.context)); - _env._metrics.merge_handler_metrics.mergeDataWriteLatency.addValue(startTime.getElapsedTimeAsDouble()); + api::BucketInfo info( + applyDiffLocally(bucket, reply.getLoadType(), diff, + index, s.context)); + _env._metrics.merge_handler_metrics.mergeDataWriteLatency.addValue( + startTime.getElapsedTimeAsDouble()); } else { LOG(spam, "Merge(%s): Didn't need fetched data on node %u (%u)", bucket.toString().c_str(), @@ -1417,7 +1464,8 @@ MergeHandler::handleApplyBucketDiffReply(api::ApplyBucketDiffReply& reply,Messag "Got reply indicating merge cycle did not fix any entries: %s", reply.toString(true).c_str()); LOG(warning, - "Merge state for which there was no progress across a full merge cycle: %s", + "Merge state for which there was no progress across a " + "full merge cycle: %s", s.toString().c_str()); } @@ -1431,7 +1479,8 @@ MergeHandler::handleApplyBucketDiffReply(api::ApplyBucketDiffReply& reply,Messag // We have sent something on and shouldn't reply now. clearState = false; } else { - _env._metrics.merge_handler_metrics.mergeLatencyTotal.addValue(s.startTime.getElapsedTimeAsDouble()); + _env._metrics.merge_handler_metrics.mergeLatencyTotal.addValue( + s.startTime.getElapsedTimeAsDouble()); } } } else { @@ -1443,7 +1492,8 @@ MergeHandler::handleApplyBucketDiffReply(api::ApplyBucketDiffReply& reply,Messag } catch (std::exception& e) { _env._fileStorHandler.clearMergeStatus( bucket.getBucket(), - api::ReturnCode(api::ReturnCode::INTERNAL_FAILURE, e.what())); + api::ReturnCode(api::ReturnCode::INTERNAL_FAILURE, + e.what())); throw; } diff --git a/storage/src/vespa/storage/persistence/persistencethread.cpp b/storage/src/vespa/storage/persistence/persistencethread.cpp index 2cdb6194b6d..53e455ea204 100644 --- a/storage/src/vespa/storage/persistence/persistencethread.cpp +++ b/storage/src/vespa/storage/persistence/persistencethread.cpp @@ -286,14 +286,14 @@ PersistenceThread::handleGet(api::GetCommand& cmd, MessageTracker::UP tracker) tracker->setMetric(metrics); metrics.request_size.addValue(cmd.getApproxByteSize()); - auto fieldSet = _env._component.getTypeRepo()->fieldSetRepo->getFieldSet(cmd.getFieldSet()); + document::FieldSet::UP fieldSet = document::FieldSetRepo::parse(*_env._component.getTypeRepo(), cmd.getFieldSet()); tracker->context().setReadConsistency(api_read_consistency_to_spi(cmd.internal_read_consistency())); spi::GetResult result = _spi.get(getBucket(cmd.getDocumentId(), cmd.getBucket()), *fieldSet, cmd.getDocumentId(), tracker->context()); if (tracker->checkForError(result)) { if (!result.hasDocument() && (document::FieldSet::Type::NONE != fieldSet->getType())) { - metrics.notFound.inc(); + _env._metrics.get[cmd.getLoadType()].notFound.inc(); } tracker->setReply(std::make_shared<api::GetReply>(cmd, result.getDocumentPtr(), result.getTimestamp(), false, result.is_tombstone())); @@ -455,11 +455,11 @@ MessageTracker::UP PersistenceThread::handleCreateIterator(CreateIteratorCommand& cmd, MessageTracker::UP tracker) { tracker->setMetric(_env._metrics.createIterator); - document::FieldSet::SP fieldSet = _env._component.getTypeRepo()->fieldSetRepo->getFieldSet(cmd.getFields()); + document::FieldSet::UP fieldSet = document::FieldSetRepo::parse(*_env._component.getTypeRepo(), cmd.getFields()); tracker->context().setReadConsistency(cmd.getReadConsistency()); spi::CreateIteratorResult result(_spi.createIterator( - spi::Bucket(cmd.getBucket(), spi::PartitionId(_env._partition)), - std::move(fieldSet), cmd.getSelection(), cmd.getIncludedVersions(), tracker->context())); + spi::Bucket(cmd.getBucket(), spi::PartitionId(_env._partition)), + *fieldSet, cmd.getSelection(), cmd.getIncludedVersions(), tracker->context())); if (tracker->checkForError(result)) { tracker->setReply(std::make_shared<CreateIteratorReply>(cmd, spi::IteratorId(result.getIteratorId()))); } diff --git a/storage/src/vespa/storage/persistence/persistenceutil.cpp b/storage/src/vespa/storage/persistence/persistenceutil.cpp index 63ac5405fab..6605e3f6363 100644 --- a/storage/src/vespa/storage/persistence/persistenceutil.cpp +++ b/storage/src/vespa/storage/persistence/persistenceutil.cpp @@ -162,7 +162,7 @@ PersistenceUtil::PersistenceUtil( _nodeIndex(_component.getIndex()), _metrics(metrics), _bucketFactory(_component.getBucketIdFactory()), - _repo(_component.getTypeRepo()->documentTypeRepo), + _repo(_component.getTypeRepo()), _spi(provider) { } diff --git a/storage/src/vespa/storage/persistence/provider_error_wrapper.cpp b/storage/src/vespa/storage/persistence/provider_error_wrapper.cpp index 0884d807eda..a5564282d17 100644 --- a/storage/src/vespa/storage/persistence/provider_error_wrapper.cpp +++ b/storage/src/vespa/storage/persistence/provider_error_wrapper.cpp @@ -111,8 +111,8 @@ ProviderErrorWrapper::get(const spi::Bucket& bucket, const document::FieldSet& f } spi::CreateIteratorResult -ProviderErrorWrapper::createIterator(const spi::Bucket &bucket, FieldSetSP fieldSet, const spi::Selection &selection, - spi::IncludedVersions versions, spi::Context &context) +ProviderErrorWrapper::createIterator(const spi::Bucket& bucket, const document::FieldSet& fieldSet, + const spi::Selection& selection, spi::IncludedVersions versions, spi::Context& context) { return checkResult(_impl.createIterator(bucket, fieldSet, selection, versions, context)); } diff --git a/storage/src/vespa/storage/persistence/provider_error_wrapper.h b/storage/src/vespa/storage/persistence/provider_error_wrapper.h index 54abf0e96fb..602877e0b02 100644 --- a/storage/src/vespa/storage/persistence/provider_error_wrapper.h +++ b/storage/src/vespa/storage/persistence/provider_error_wrapper.h @@ -52,9 +52,8 @@ public: spi::RemoveResult removeIfFound(const spi::Bucket&, spi::Timestamp, const document::DocumentId&, spi::Context&) override; spi::UpdateResult update(const spi::Bucket&, spi::Timestamp, spi::DocumentUpdateSP, spi::Context&) override; spi::GetResult get(const spi::Bucket&, const document::FieldSet&, const document::DocumentId&, spi::Context&) const override; - spi::CreateIteratorResult - createIterator(const spi::Bucket &bucket, FieldSetSP, const spi::Selection &, spi::IncludedVersions versions, - spi::Context &context) override; + spi::CreateIteratorResult createIterator(const spi::Bucket&, const document::FieldSet&, const spi::Selection&, + spi::IncludedVersions versions, spi::Context&) override; spi::IterateResult iterate(spi::IteratorId, uint64_t maxByteSize, spi::Context&) const override; spi::Result destroyIterator(spi::IteratorId, spi::Context&) override; spi::Result createBucket(const spi::Bucket&, spi::Context&) override; diff --git a/storage/src/vespa/storage/persistence/testandsethelper.cpp b/storage/src/vespa/storage/persistence/testandsethelper.cpp index 57586249817..9232abc5c8a 100644 --- a/storage/src/vespa/storage/persistence/testandsethelper.cpp +++ b/storage/src/vespa/storage/persistence/testandsethelper.cpp @@ -1,8 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // @author Vegard Sjonfjell -#include "fieldvisitor.h" -#include "testandsethelper.h" +#include <vespa/storage/persistence/fieldvisitor.h> +#include <vespa/storage/persistence/testandsethelper.h> #include <vespa/document/select/parser.h> #include <vespa/document/repo/documenttyperepo.h> #include <vespa/vespalib/util/stringfmt.h> @@ -11,19 +11,19 @@ using namespace std::string_literals; namespace storage { -void TestAndSetHelper::getDocumentType(const document::DocumentTypeRepo & documentTypeRepo) { +void TestAndSetHelper::getDocumentType() { if (!_docId.hasDocType()) { throw TestAndSetException(api::ReturnCode(api::ReturnCode::ILLEGAL_PARAMETERS, "Document id has no doctype")); } - _docTypePtr = documentTypeRepo.getDocumentType(_docId.getDocType()); + _docTypePtr = _component.getTypeRepo()->getDocumentType(_docId.getDocType()); if (_docTypePtr == nullptr) { throw TestAndSetException(api::ReturnCode(api::ReturnCode::ILLEGAL_PARAMETERS, "Document type does not exist")); } } -void TestAndSetHelper::parseDocumentSelection(const document::DocumentTypeRepo & documentTypeRepo) { - document::select::Parser parser(documentTypeRepo, _component.getBucketIdFactory()); +void TestAndSetHelper::parseDocumentSelection() { + document::select::Parser parser(*_component.getTypeRepo(), _component.getBucketIdFactory()); try { _docSelectionUp = parser.parse(_cmd.getCondition().getSelection()); @@ -49,9 +49,8 @@ TestAndSetHelper::TestAndSetHelper(PersistenceThread & thread, const api::TestAn _docTypePtr(nullptr), _missingDocumentImpliesMatch(missingDocumentImpliesMatch) { - auto docTypeRepo = _component.getTypeRepo()->documentTypeRepo; - getDocumentType(*docTypeRepo); - parseDocumentSelection(*docTypeRepo); + getDocumentType(); + parseDocumentSelection(); } TestAndSetHelper::~TestAndSetHelper() = default; diff --git a/storage/src/vespa/storage/persistence/testandsethelper.h b/storage/src/vespa/storage/persistence/testandsethelper.h index b528b5034f9..b5fa29d0106 100644 --- a/storage/src/vespa/storage/persistence/testandsethelper.h +++ b/storage/src/vespa/storage/persistence/testandsethelper.h @@ -28,8 +28,8 @@ class TestAndSetHelper { std::unique_ptr<document::select::Node> _docSelectionUp; bool _missingDocumentImpliesMatch; - void getDocumentType(const document::DocumentTypeRepo & documentTypeRepo); - void parseDocumentSelection(const document::DocumentTypeRepo & documentTypeRepo); + void getDocumentType(); + void parseDocumentSelection(); spi::GetResult retrieveDocument(const document::FieldSet & fieldSet, spi::Context & context); public: diff --git a/storage/src/vespa/storage/storageserver/communicationmanager.cpp b/storage/src/vespa/storage/storageserver/communicationmanager.cpp index b51394e2e64..c0adb01ad47 100644 --- a/storage/src/vespa/storage/storageserver/communicationmanager.cpp +++ b/storage/src/vespa/storage/storageserver/communicationmanager.cpp @@ -395,12 +395,10 @@ void CommunicationManager::configure(std::unique_ptr<CommunicationManagerConfig> // Configure messagebus here as we for legacy reasons have // config here. - auto documentTypeRepo = _component.getTypeRepo()->documentTypeRepo; - auto loadTypes = _component.getLoadTypes(); _mbus = std::make_unique<mbus::RPCMessageBus>( mbus::ProtocolSet() - .add(std::make_shared<documentapi::DocumentProtocol>(*loadTypes, documentTypeRepo)) - .add(std::make_shared<mbusprot::StorageProtocol>(documentTypeRepo, *loadTypes)), + .add(std::make_shared<documentapi::DocumentProtocol>(*_component.getLoadTypes(), _component.getTypeRepo())) + .add(std::make_shared<mbusprot::StorageProtocol>(_component.getTypeRepo(), *_component.getLoadTypes())), params, _configUri); diff --git a/storage/src/vespa/storage/visiting/visitorthread.cpp b/storage/src/vespa/storage/visiting/visitorthread.cpp index 73f4a70d80d..c6e75735690 100644 --- a/storage/src/vespa/storage/visiting/visitorthread.cpp +++ b/storage/src/vespa/storage/visiting/visitorthread.cpp @@ -31,7 +31,7 @@ VisitorThread::Event::Event(Event&& other) { } -VisitorThread::Event::~Event() = default; +VisitorThread::Event::~Event() {} VisitorThread::Event& VisitorThread::Event::operator= (Event&& other) @@ -44,7 +44,9 @@ VisitorThread::Event::operator= (Event&& other) return *this; } -VisitorThread::Event::Event(api::VisitorId visitor, const std::shared_ptr<api::StorageMessage>& msg) +VisitorThread::Event::Event( + api::VisitorId visitor, + const std::shared_ptr<api::StorageMessage>& msg) : _visitorId(visitor), _message(msg), _timer(), @@ -52,7 +54,9 @@ VisitorThread::Event::Event(api::VisitorId visitor, const std::shared_ptr<api::S { } -VisitorThread::Event::Event(api::VisitorId visitor, mbus::Reply::UP reply) +VisitorThread::Event::Event( + api::VisitorId visitor, + mbus::Reply::UP reply) : _visitorId(visitor), _mbusReply(std::move(reply)), _timer(), @@ -327,7 +331,7 @@ VisitorThread::handleNonExistingVisitorCall(const Event& entry, ReturnCode& code) { // Get current time. Set the time that is the oldest still recent. - framework::SecondTime currentTime(_component.getClock().getTimeInSeconds()); + framework::SecondTime currentTime(_component.getClock().getTimeInSeconds());; trimRecentlyCompletedList(currentTime); // Go through all recent visitors. Ignore request if recent @@ -431,7 +435,8 @@ VisitorThread::onCreateVisitor( do { // If no buckets are specified, fail command if (cmd->getBuckets().empty()) { - result = ReturnCode(ReturnCode::ILLEGAL_PARAMETERS, "No buckets specified"); + result = ReturnCode(ReturnCode::ILLEGAL_PARAMETERS, + "No buckets specified"); LOG(warning, "CreateVisitor(%s): No buckets specified. Aborting.", cmd->getInstanceId().c_str()); break; @@ -475,7 +480,7 @@ VisitorThread::onCreateVisitor( // Parse document selection try{ if (!cmd->getDocumentSelection().empty()) { - std::shared_ptr<const document::DocumentTypeRepo> repo(_component.getTypeRepo()->documentTypeRepo); + std::shared_ptr<const document::DocumentTypeRepo> repo(_component.getTypeRepo()); const document::BucketIdFactory& idFactory(_component.getBucketIdFactory()); document::select::Parser parser(*repo, idFactory); docSelection = parser.parse(cmd->getDocumentSelection()); diff --git a/storageserver/src/vespa/storageserver/app/dummyservicelayerprocess.cpp b/storageserver/src/vespa/storageserver/app/dummyservicelayerprocess.cpp index 338ae45714e..b642fa9dc8e 100644 --- a/storageserver/src/vespa/storageserver/app/dummyservicelayerprocess.cpp +++ b/storageserver/src/vespa/storageserver/app/dummyservicelayerprocess.cpp @@ -15,13 +15,13 @@ void DummyServiceLayerProcess::shutdown() { ServiceLayerProcess::shutdown(); - _provider.reset(); + _provider.reset(0); } void DummyServiceLayerProcess::setupProvider() { - _provider = std::make_unique<spi::dummy::DummyPersistence>(getTypeRepo()); + _provider.reset(new spi::dummy::DummyPersistence(getTypeRepo())); } } // storage diff --git a/storageserver/src/vespa/storageserver/app/dummyservicelayerprocess.h b/storageserver/src/vespa/storageserver/app/dummyservicelayerprocess.h index dc4e3286d0e..6eb774a131d 100644 --- a/storageserver/src/vespa/storageserver/app/dummyservicelayerprocess.h +++ b/storageserver/src/vespa/storageserver/app/dummyservicelayerprocess.h @@ -12,7 +12,7 @@ namespace storage { class DummyServiceLayerProcess : public ServiceLayerProcess { - std::unique_ptr<spi::PersistenceProvider> _provider; + spi::dummy::DummyPersistence::UP _provider; public: DummyServiceLayerProcess(const config::ConfigUri & configUri); diff --git a/streamingvisitors/src/tests/searchvisitor/searchvisitor_test.cpp b/streamingvisitors/src/tests/searchvisitor/searchvisitor_test.cpp index 216e02c5edd..18cd7fab2b8 100644 --- a/streamingvisitors/src/tests/searchvisitor/searchvisitor_test.cpp +++ b/streamingvisitors/src/tests/searchvisitor/searchvisitor_test.cpp @@ -35,7 +35,7 @@ private: public: SearchVisitorTest(); - ~SearchVisitorTest() override; + ~SearchVisitorTest(); int Main() override; }; @@ -46,9 +46,9 @@ SearchVisitorTest::SearchVisitorTest() : { _componentRegister.setNodeInfo("mycluster", lib::NodeType::STORAGE, 1); _componentRegister.setClock(_clock); - auto repo = std::make_shared<DocumentTypeRepo>(readDocumenttypesConfig(TEST_PATH("cfg/documenttypes.cfg"))); + StorageComponent::DocumentTypeRepoSP repo(new DocumentTypeRepo(readDocumenttypesConfig(TEST_PATH("cfg/documenttypes.cfg")))); _componentRegister.setDocumentTypeRepo(repo); - _component = std::make_unique<StorageComponent>(_componentRegister, "storage"); + _component.reset(new StorageComponent(_componentRegister, "storage")); } SearchVisitorTest::~SearchVisitorTest() = default; @@ -80,8 +80,8 @@ SearchVisitorTest::testCreateSearchVisitor(const vespalib::string & dir, const v void SearchVisitorTest::testSearchEnvironment() { - EXPECT_TRUE(_env.getVSMAdapter("simple") != nullptr); - EXPECT_TRUE(_env.getRankManager("simple") != nullptr); + EXPECT_TRUE(_env.getVSMAdapter("simple") != NULL); + EXPECT_TRUE(_env.getRankManager("simple") != NULL); } void diff --git a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp index cc28c76acce..76ef0f23dd2 100644 --- a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp +++ b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp @@ -677,7 +677,7 @@ SearchVisitor::setupScratchDocument(const StringFieldIdTMap & fieldsInQuery) } // Init based on default document type and mapping from field name to field id _docTypeMapping.init(_fieldSearchSpecMap.documentTypeMap().begin()->first, - _fieldsUnion, *_component.getTypeRepo()->documentTypeRepo); + _fieldsUnion, *_component.getTypeRepo()); _docTypeMapping.prepareBaseDoc(_fieldPathMap); } |