diff options
59 files changed, 464 insertions, 396 deletions
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java index 75b13a89e83..c50b9b7f842 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java @@ -36,7 +36,7 @@ public class LogserverContainerCluster extends ContainerCluster<LogserverContain protected boolean messageBusEnabled() { return false; } private void addLogHandler() { - Handler<?> logHandler = Handler.fromClassName(ContainerCluster.LOG_HANDLER_CLASS); + Handler logHandler = Handler.fromClassName(ContainerCluster.LOG_HANDLER_CLASS); logHandler.addServerBindings(SystemBindingPattern.fromHttpPath("/logs")); addComponent(logHandler); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java index 1da5e190c70..5d60cec0679 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java @@ -118,7 +118,7 @@ public class ClusterControllerContainer extends Container implements ZOOKEEPER_SERVER_BUNDLE); } - private void addHandler(Handler<?> h, String path) { + private void addHandler(Handler h, String path) { h.addServerBindings(SystemBindingPattern.fromHttpPath(path)); super.addHandler(h); } @@ -138,7 +138,7 @@ public class ClusterControllerContainer extends Container implements } private void addHandler(String id, String className, String path, ComponentSpecification bundle) { - addHandler(new Handler<>(createComponentModel(id, className, bundle)), path); + addHandler(new Handler(createComponentModel(id, className, bundle)), path); } private ReindexingContext reindexingContext() { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java index a29647b062a..680a4b97f86 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java @@ -119,12 +119,12 @@ public class MetricsProxyContainerCluster extends ContainerCluster<MetricsProxyC } private void addHttpHandler(Class<? extends ThreadedHttpRequestHandler> clazz, String bindingPath) { - Handler<AbstractConfigProducer<?>> metricsHandler = createMetricsHandler(clazz, bindingPath); + Handler metricsHandler = createMetricsHandler(clazz, bindingPath); addComponent(metricsHandler); } - static Handler<AbstractConfigProducer<?>> createMetricsHandler(Class<? extends ThreadedHttpRequestHandler> clazz, String bindingPath) { - Handler<AbstractConfigProducer<?>> metricsHandler = new Handler<>( + static Handler createMetricsHandler(Class<? extends ThreadedHttpRequestHandler> clazz, String bindingPath) { + Handler metricsHandler = new Handler( new ComponentModel(clazz.getName(), null, METRICS_PROXY_BUNDLE_NAME, null)); metricsHandler.addServerBindings( SystemBindingPattern.fromHttpPath(bindingPath), diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/UriBindingsValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/UriBindingsValidator.java index 4f322578b1c..718f1646126 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/UriBindingsValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/UriBindingsValidator.java @@ -24,7 +24,7 @@ class UriBindingsValidator extends Validator { @Override public void validate(VespaModel model, DeployState deployState) { for (ApplicationContainerCluster cluster : model.getContainerClusters().values()) { - for (Handler<?> handler : cluster.getHandlers()) { + for (Handler handler : cluster.getHandlers()) { for (BindingPattern binding : handler.getServerBindings()) { validateUserBinding(binding, model, deployState); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomClientProviderBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomClientProviderBuilder.java index 69613944e74..170e8940787 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomClientProviderBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomClientProviderBuilder.java @@ -5,7 +5,6 @@ import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.text.XML; import com.yahoo.vespa.model.container.ApplicationContainerCluster; -import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.container.component.Handler; import com.yahoo.vespa.model.container.component.UserBindingPattern; import org.w3c.dom.Element; @@ -22,7 +21,7 @@ public class DomClientProviderBuilder extends DomHandlerBuilder { @Override protected Handler doBuild(DeployState deployState, AbstractConfigProducer parent, Element clientElement) { - Handler<? super Component<?, ?>> client = createHandler(clientElement); + Handler client = createHandler(clientElement); for (Element binding : XML.getChildren(clientElement, "binding")) client.addClientBindings(UserBindingPattern.fromPattern(XML.getValue(binding))); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java index 025a6377b04..7bfe971981e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java @@ -9,15 +9,12 @@ import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.text.XML; import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.component.BindingPattern; -import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.container.component.Handler; import com.yahoo.vespa.model.container.component.UserBindingPattern; import com.yahoo.vespa.model.container.xml.BundleInstantiationSpecificationBuilder; import org.w3c.dom.Element; -import java.util.List; import java.util.Set; -import java.util.logging.Level; import static com.yahoo.vespa.model.container.ApplicationContainerCluster.METRICS_V2_HANDLER_BINDING_1; import static com.yahoo.vespa.model.container.ApplicationContainerCluster.METRICS_V2_HANDLER_BINDING_2; @@ -29,7 +26,7 @@ import static java.util.logging.Level.INFO; /** * @author gjoranv */ -public class DomHandlerBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Handler<?>> { +public class DomHandlerBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Handler> { private static final Set<BindingPattern> reservedBindings = Set.of(METRICS_V2_HANDLER_BINDING_1, @@ -45,8 +42,8 @@ public class DomHandlerBuilder extends VespaDomBuilder.DomConfigProducerBuilder< } @Override - protected Handler<?> doBuild(DeployState deployState, AbstractConfigProducer<?> parent, Element handlerElement) { - Handler<? super Component<?, ?>> handler = createHandler(handlerElement); + protected Handler doBuild(DeployState deployState, AbstractConfigProducer<?> parent, Element handlerElement) { + Handler handler = createHandler(handlerElement); for (Element binding : XML.getChildren(handlerElement, "binding")) addServerBinding(handler, UserBindingPattern.fromPattern(XML.getValue(binding)), deployState.getDeployLogger()); @@ -56,18 +53,18 @@ public class DomHandlerBuilder extends VespaDomBuilder.DomConfigProducerBuilder< return handler; } - Handler<? super Component<?, ?>> createHandler(Element handlerElement) { + Handler createHandler(Element handlerElement) { BundleInstantiationSpecification bundleSpec = BundleInstantiationSpecificationBuilder.build(handlerElement); - return new Handler<>(new ComponentModel(bundleSpec)); + return new Handler(new ComponentModel(bundleSpec)); } - private void addServerBinding(Handler<? super Component<?, ?>> handler, BindingPattern binding, DeployLogger log) { + private void addServerBinding(Handler handler, BindingPattern binding, DeployLogger log) { throwIfBindingIsReserved(binding, handler); handler.addServerBindings(binding); removeExistingServerBinding(binding, handler, log); } - private void throwIfBindingIsReserved(BindingPattern binding, Handler<?> newHandler) { + private void throwIfBindingIsReserved(BindingPattern binding, Handler newHandler) { for (var reserved : reservedBindings) { if (binding.hasSamePattern(reserved)) { throw new IllegalArgumentException("Binding '" + binding.patternString() + "' is a reserved Vespa binding and " + @@ -76,7 +73,7 @@ public class DomHandlerBuilder extends VespaDomBuilder.DomConfigProducerBuilder< } } - private void removeExistingServerBinding(BindingPattern binding, Handler<?> newHandler, DeployLogger log) { + private void removeExistingServerBinding(BindingPattern binding, Handler newHandler, DeployLogger log) { for (var handler : cluster.getHandlers()) { for (BindingPattern serverBinding : handler.getServerBindings()) { if (serverBinding.hasSamePattern(binding)) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java b/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java index 33c125dcecf..d5f41353616 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java @@ -1,8 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.clients; -import com.yahoo.config.model.producer.AbstractConfigProducer; -import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.container.handler.threadpool.ContainerThreadpoolConfig; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.vespa.model.container.ContainerCluster; @@ -41,31 +39,31 @@ public class ContainerDocumentApi { private static void addFeedHandler(ContainerCluster<?> cluster, HandlerOptions handlerOptions) { String bindingSuffix = ContainerCluster.RESERVED_URI_PREFIX + "/feedapi"; - var handler = newVespaClientHandler("com.yahoo.vespa.http.server.FeedHandler", bindingSuffix, handlerOptions); - cluster.addComponent(handler); var executor = new Threadpool("feedapi-handler", handlerOptions.feedApiThreadpoolOptions); - handler.inject(executor); - handler.addComponent(executor); + var handler = newVespaClientHandler("com.yahoo.vespa.http.server.FeedHandler", + bindingSuffix, handlerOptions, executor); + cluster.addComponent(handler); } private static void addRestApiHandler(ContainerCluster<?> cluster, HandlerOptions handlerOptions) { - var handler = newVespaClientHandler("com.yahoo.document.restapi.resource.DocumentV1ApiHandler", DOCUMENT_V1_PREFIX + "/*", handlerOptions); + var handler = newVespaClientHandler("com.yahoo.document.restapi.resource.DocumentV1ApiHandler", + DOCUMENT_V1_PREFIX + "/*", handlerOptions, null); cluster.addComponent(handler); // We need to include a dummy implementation of the previous restapi handler (using the same class name). // The internal legacy test framework requires that the name of the old handler is listed in /ApplicationStatus. - var oldHandlerDummy = handlerComponentSpecification("com.yahoo.document.restapi.resource.RestApi"); + var oldHandlerDummy = createHandler("com.yahoo.document.restapi.resource.RestApi", null); cluster.addComponent(oldHandlerDummy); } public boolean ignoreUndefinedFields() { return ignoreUndefinedFields; } - private static Handler<AbstractConfigProducer<?>> newVespaClientHandler( - String componentId, - String bindingSuffix, - HandlerOptions handlerOptions) { - Handler<AbstractConfigProducer<?>> handler = handlerComponentSpecification(componentId); + private static Handler newVespaClientHandler(String componentId, + String bindingSuffix, + HandlerOptions handlerOptions, + Threadpool executor) { + Handler handler = createHandler(componentId, executor); if (handlerOptions.bindings.isEmpty()) { handler.addServerBindings( SystemBindingPattern.fromHttpPath(bindingSuffix), @@ -81,9 +79,9 @@ public class ContainerDocumentApi { return handler; } - private static Handler<AbstractConfigProducer<?>> handlerComponentSpecification(String className) { - return new Handler<>(new ComponentModel( - BundleInstantiationSpecification.getFromStrings(className, null, "vespaclient-container-plugin"), "")); + private static Handler createHandler(String className, Threadpool executor) { + return new Handler(new ComponentModel(className, null, "vespaclient-container-plugin"), + executor); } public static final class HandlerOptions { @@ -104,12 +102,10 @@ public class ContainerDocumentApi { } @Override - public void getConfig(ContainerThreadpoolConfig.Builder builder) { - super.getConfig(builder); - - // User options overrides below configuration - if (hasUserOptions()) return; - builder.maxThreads(-4).minThreads(-4).queueSize(500); + protected void setDefaultConfigValues(ContainerThreadpoolConfig.Builder builder) { + builder.maxThreads(-4) + .minThreads(-4) + .queueSize(500); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index 937d7cf58d3..e316f826ad6 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -147,7 +147,7 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat } private void addMetricsHandler(String handlerClass, BindingPattern rootBinding, BindingPattern innerBinding) { - Handler<AbstractConfigProducer<?>> handler = new Handler<>( + Handler handler = new Handler( new ComponentModel(handlerClass, null, null, null)); handler.addServerBindings(rootBinding, innerBinding); addComponent(handler); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java index 63c323bfdaf..3d6332a9773 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java @@ -76,7 +76,7 @@ public abstract class Container extends AbstractService implements private final boolean dumpHeapOnShutdownTimeout; private final double shutdownTimeoutS; - private final ComponentGroup<Handler<?>> handlers = new ComponentGroup<>(this, "handler"); + private final ComponentGroup<Handler> handlers = new ComponentGroup<>(this, "handler"); private final ComponentGroup<Component<?, ?>> components = new ComponentGroup<>(this, "components"); private final JettyHttpServer defaultHttpServer; @@ -113,7 +113,7 @@ public abstract class Container extends AbstractService implements /** True if this container is retired (slated for removal) */ public boolean isRetired() { return retired; } - public ComponentGroup<Handler<?>> getHandlers() { + public ComponentGroup<Handler> getHandlers() { return handlers; } @@ -129,7 +129,7 @@ public abstract class Container extends AbstractService implements addComponent(new SimpleComponent(new ComponentModel(idSpec, classSpec, bundleSpec))); } - public final void addHandler(Handler<?> h) { + public final void addHandler(Handler h) { handlers.addComponent(h); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index 2385b5b3812..45fde57fcc7 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -142,6 +142,7 @@ public abstract class ContainerCluster<CONTAINER extends Container> private ContainerDocproc containerDocproc; private ContainerDocumentApi containerDocumentApi; private SecretStore secretStore; + private final ContainerThreadpool defaultHandlerThreadpool = new Handler.DefaultHandlerThreadpool(); private boolean rpcServerEnabled = true; private boolean httpServerEnabled = true; @@ -181,6 +182,7 @@ public abstract class ContainerCluster<CONTAINER extends Container> addCommonVespaBundles(); addSimpleComponent(AccessLog.class); addComponent(new DefaultThreadpoolProvider(this, defaultPoolNumThreads)); + addComponent(defaultHandlerThreadpool); addSimpleComponent(com.yahoo.concurrent.classlock.ClassLocking.class); addSimpleComponent("com.yahoo.container.jdisc.metric.MetricConsumerProviderProvider"); addSimpleComponent("com.yahoo.container.jdisc.metric.MetricProvider"); @@ -217,14 +219,14 @@ public abstract class ContainerCluster<CONTAINER extends Container> } public void addMetricStateHandler() { - Handler<AbstractConfigProducer<?>> stateHandler = new Handler<>( + Handler stateHandler = new Handler( new ComponentModel(STATE_HANDLER_CLASS, null, null, null)); stateHandler.addServerBindings(STATE_HANDLER_BINDING_1, STATE_HANDLER_BINDING_2); addComponent(stateHandler); } public void addDefaultRootHandler() { - Handler<AbstractConfigProducer<?>> handler = new Handler<>( + Handler handler = new Handler( new ComponentModel(BundleInstantiationSpecification.getFromStrings( BINDINGS_OVERVIEW_HANDLER_CLASS, null, null), null)); // null bundle, as the handler is in container-disc handler.addServerBindings(ROOT_HANDLER_BINDING); @@ -232,7 +234,7 @@ public abstract class ContainerCluster<CONTAINER extends Container> } public void addApplicationStatusHandler() { - Handler<AbstractConfigProducer<?>> statusHandler = new Handler<>( + Handler statusHandler = new Handler( new ComponentModel(BundleInstantiationSpecification.getFromStrings( APPLICATION_STATUS_HANDLER_CLASS, null, null), null)); // null bundle, as the handler is in container-disc statusHandler.addServerBindings(SystemBindingPattern.fromHttpPath("/ApplicationStatus")); @@ -240,13 +242,22 @@ public abstract class ContainerCluster<CONTAINER extends Container> } public void addVipHandler() { - Handler<?> vipHandler = Handler.fromClassName(FileStatusHandlerComponent.CLASS); + Handler vipHandler = Handler.fromClassName(FileStatusHandlerComponent.CLASS); vipHandler.addServerBindings(VIP_HANDLER_BINDING); addComponent(vipHandler); } public final void addComponent(Component<?, ?> component) { componentGroup.addComponent(component); + if (component instanceof Handler handler) { + ensureHandlerHasThreadpool(handler); + } + } + + private void ensureHandlerHasThreadpool(Handler handler) { + if (! handler.hasCustomThreadPool) { + handler.inject(defaultHandlerThreadpool); + } } public final void addSimpleComponent(String idSpec, String classSpec, String bundleSpec) { @@ -306,10 +317,8 @@ public abstract class ContainerCluster<CONTAINER extends Container> this.processingChains = processingChains; // Cannot use the class object for ProcessingHandler, because its superclass is not accessible - ProcessingHandler<?> processingHandler = new ProcessingHandler<>( - processingChains, - "com.yahoo.processing.handler.ProcessingHandler", - null); + ProcessingHandler<?> processingHandler = new ProcessingHandler<>(processingChains, + "com.yahoo.processing.handler.ProcessingHandler"); for (BindingPattern binding: serverBindings) processingHandler.addServerBindings(binding); @@ -368,9 +377,8 @@ public abstract class ContainerCluster<CONTAINER extends Container> return containerDocproc.getChains(); } - @SuppressWarnings("unchecked") - public Collection<Handler<?>> getHandlers() { - return (Collection<Handler<?>>)(Collection)componentGroup.getComponents(Handler.class); + public Collection<Handler> getHandlers() { + return componentGroup.getComponents(Handler.class); } public void setSecretStore(SecretStore secretStore) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java index cb8e6ba85ff..088465f56b1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java @@ -63,8 +63,8 @@ public class ContainerModelEvaluation implements rankProfileList.getConfig(builder); } - public static Handler<?> getHandler() { - Handler<?> handler = new Handler<>(new ComponentModel(REST_HANDLER_NAME, null, EVALUATION_BUNDLE_NAME)); + public static Handler getHandler() { + Handler handler = new Handler(new ComponentModel(REST_HANDLER_NAME, null, EVALUATION_BUNDLE_NAME)); handler.addServerBindings( SystemBindingPattern.fromHttpPath(REST_BINDING_PATH), SystemBindingPattern.fromHttpPath(REST_BINDING_PATH + "/*")); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerThreadpool.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerThreadpool.java index 489e4cc135a..fbd7bc9fe56 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerThreadpool.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerThreadpool.java @@ -17,7 +17,7 @@ import java.util.Optional; * * @author bjorncs */ -public class ContainerThreadpool extends SimpleComponent implements ContainerThreadpoolConfig.Producer { +public abstract class ContainerThreadpool extends SimpleComponent implements ContainerThreadpoolConfig.Producer { private final String name; private final UserOptions userOptions; @@ -32,8 +32,13 @@ public class ContainerThreadpool extends SimpleComponent implements ContainerThr this.userOptions = userOptions; } + // Must be implemented by subclasses to set values that may be overridden by user options. + protected abstract void setDefaultConfigValues(ContainerThreadpoolConfig.Builder builder); + @Override public void getConfig(ContainerThreadpoolConfig.Builder builder) { + setDefaultConfigValues(builder); + builder.name(this.name); if (userOptions != null) { builder.maxThreads(userOptions.maxThreads); @@ -42,9 +47,6 @@ public class ContainerThreadpool extends SimpleComponent implements ContainerThr } } - protected Optional<UserOptions> userOptions() { return Optional.ofNullable(userOptions); } - protected boolean hasUserOptions() { return userOptions().isPresent(); } - public static class UserOptions { private final int maxThreads; private final int minThreads; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/DiscBindingsConfigGenerator.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/DiscBindingsConfigGenerator.java index 76124b14209..e4a5c2cd440 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/DiscBindingsConfigGenerator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/DiscBindingsConfigGenerator.java @@ -14,16 +14,16 @@ import static java.util.stream.Collectors.toList; */ public class DiscBindingsConfigGenerator { - public static Map<String, Handlers.Builder> generate(Collection<? extends Handler<?>> handlers) { + public static Map<String, Handlers.Builder> generate(Collection<? extends Handler> handlers) { Map<String, Handlers.Builder> handlerBuilders = new LinkedHashMap<>(); - for (Handler<?> handler : handlers) { + for (Handler handler : handlers) { handlerBuilders.putAll(generate(handler)); } return handlerBuilders; } - public static <T extends Handler<?>> Map<String, Handlers.Builder> generate(T handler) { + public static <T extends Handler> Map<String, Handlers.Builder> generate(T handler) { if (handler.getServerBindings().isEmpty() && handler.getClientBindings().isEmpty()) return Collections.emptyMap(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java index 8ffdccae896..9f2bfe9251b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java @@ -1,8 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container.component; -import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.container.handler.threadpool.ContainerThreadpoolConfig; import com.yahoo.osgi.provider.model.ComponentModel; +import com.yahoo.vespa.model.container.ContainerThreadpool; import java.util.ArrayList; import java.util.Arrays; @@ -15,22 +16,35 @@ import java.util.Set; * Models a jdisc RequestHandler (including ClientProvider). * RequestHandlers always have at least one server binding, * while ClientProviders have at least one client binding. - * <p> - * Note that this is also used to model vespa handlers (which do not have any bindings) * * @author gjoranv */ -public class Handler<CHILD extends AbstractConfigProducer<?>> extends Component<CHILD, ComponentModel> { +public class Handler extends Component<Component<?, ?>, ComponentModel> { private final Set<BindingPattern> serverBindings = new LinkedHashSet<>(); private final List<BindingPattern> clientBindings = new ArrayList<>(); + public final boolean hasCustomThreadPool; + public Handler(ComponentModel model) { + this(model, null); + } + + public Handler(ComponentModel model, ContainerThreadpool threadpool) { super(model); + + // The default threadpool is always added to the cluster, so cannot be added here. + if (threadpool != null) { + hasCustomThreadPool = true; + addComponent(threadpool); + inject(threadpool); + } else { + hasCustomThreadPool = false; + } } - public static Handler<AbstractConfigProducer<?>> fromClassName(String className) { - return new Handler<>(new ComponentModel(className, null, null, null)); + public static Handler fromClassName(String className) { + return new Handler(new ComponentModel(className, null, null, null)); } public void addServerBindings(BindingPattern... bindings) { @@ -53,4 +67,24 @@ public class Handler<CHILD extends AbstractConfigProducer<?>> extends Component< return Collections.unmodifiableList(clientBindings); } + + /** + * The default threadpool for all handlers, except those that declare their own, e.g. SearchHandler. + */ + public static class DefaultHandlerThreadpool extends ContainerThreadpool { + + public DefaultHandlerThreadpool() { + super("default-handler-common", null); + } + + @Override + public void setDefaultConfigValues(ContainerThreadpoolConfig.Builder builder) { + builder.maxThreadExecutionTimeSeconds(190) + .keepAliveTime(5.0) + .maxThreads(-2) + .minThreads(-2) + .queueSize(-40); + } + } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/ProcessingHandler.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/ProcessingHandler.java index 3f68a0a2709..e0bbc2d755d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/ProcessingHandler.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/ProcessingHandler.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.model.container.component.chain; import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.container.core.ChainsConfig; import com.yahoo.osgi.provider.model.ComponentModel; -import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.model.container.ContainerThreadpool; import com.yahoo.vespa.model.container.component.Handler; @@ -14,21 +14,17 @@ import com.yahoo.vespa.model.container.component.Handler; * @author gjoranv */ public class ProcessingHandler<CHAINS extends Chains<?>> - extends Handler<AbstractConfigProducer<?>> + extends Handler implements ChainsConfig.Producer { protected final CHAINS chains; public ProcessingHandler(CHAINS chains, String handlerClass) { - this(chains, BundleInstantiationSpecification.getInternalProcessingSpecificationFromStrings(handlerClass, null)); + this(chains, BundleInstantiationSpecification.getInternalProcessingSpecificationFromStrings(handlerClass, null), null); } - public ProcessingHandler(CHAINS chains, String handlerClass, String bundle) { - this(chains, BundleInstantiationSpecification.getFromStrings(handlerClass, null, bundle)); - } - - private ProcessingHandler(CHAINS chains, BundleInstantiationSpecification spec) { - super(new ComponentModel(spec, null)); + public ProcessingHandler(CHAINS chains, BundleInstantiationSpecification spec, ContainerThreadpool threadpool) { + super(new ComponentModel(spec, null), threadpool); this.chains = chains; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/MbusClient.java b/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/MbusClient.java index 28a0748be26..0efcd8df37f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/MbusClient.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/MbusClient.java @@ -6,13 +6,12 @@ import com.yahoo.component.ComponentSpecification; import com.yahoo.container.jdisc.config.SessionConfig; import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.osgi.provider.model.ComponentModel; -import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.vespa.model.container.component.Handler; /** * @author Einar M R Rosenvinge */ -public class MbusClient extends Handler<AbstractConfigProducer<?>> implements SessionConfig.Producer { +public class MbusClient extends Handler implements SessionConfig.Producer { private static final ComponentSpecification CLASSNAME = ComponentSpecification.fromString("com.yahoo.container.jdisc.messagebus.MbusClientProvider"); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java index d85f00a5bb2..3ee0414bf32 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java @@ -51,7 +51,7 @@ public class AccessControl { private final String domain; private ClientAuthentication clientAuthentication = ClientAuthentication.need; private final Set<BindingPattern> excludeBindings = new LinkedHashSet<>(); - private Collection<Handler<?>> handlers = Collections.emptyList(); + private Collection<Handler> handlers = Collections.emptyList(); public Builder(String domain) { this.domain = domain; } @@ -79,12 +79,12 @@ public class AccessControl { public final String domain; public final ClientAuthentication clientAuthentication; private final Set<BindingPattern> excludedBindings; - private final Collection<Handler<?>> handlers; + private final Collection<Handler> handlers; private AccessControl(String domain, ClientAuthentication clientAuthentication, Set<BindingPattern> excludedBindings, - Collection<Handler<?>> handlers) { + Collection<Handler> handlers) { this.domain = domain; this.clientAuthentication = clientAuthentication; this.excludedBindings = Collections.unmodifiableSet(excludedBindings); @@ -119,7 +119,7 @@ public class AccessControl { public Set<BindingPattern> excludedBindings() { return excludedBindings; } /** all handlers (that are known by the access control components) **/ - public Collection<Handler<?>> handlers() { return handlers; } + public Collection<Handler> handlers() { return handlers; } public static boolean hasHandlerThatNeedsProtection(ApplicationContainerCluster cluster) { return cluster.getHandlers().stream() @@ -135,7 +135,7 @@ public class AccessControl { for (BindingPattern excludedBinding : excludedBindings) { http.getBindings().add(createAccessControlExcludedBinding(excludedBinding)); } - for (Handler<?> handler : handlers) { + for (Handler handler : handlers) { if (isExcludedHandler(handler)) { for (BindingPattern binding : handler.getServerBindings()) { http.getBindings().add(createAccessControlExcludedBinding(binding)); @@ -188,9 +188,9 @@ public class AccessControl { private static Chain<Filter> createChain(ComponentId id) { return new Chain<>(FilterChains.emptyChainSpec(id)); } - private static boolean isExcludedHandler(Handler<?> handler) { return EXCLUDED_HANDLERS.contains(handler.getClassId().getName()); } + private static boolean isExcludedHandler(Handler handler) { return EXCLUDED_HANDLERS.contains(handler.getClassId().getName()); } - private static boolean hasNonMbusBinding(Handler<?> handler) { + private static boolean hasNonMbusBinding(Handler handler) { return handler.getServerBindings().stream().anyMatch(binding -> ! binding.scheme().equals("mbus")); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/GUIHandler.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/GUIHandler.java index 7087cabafc1..c6b0eb28ed3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/GUIHandler.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/GUIHandler.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container.search; -import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.vespa.model.container.component.Handler; @@ -10,7 +9,7 @@ import com.yahoo.vespa.model.container.component.Handler; /** * @author Henrik Høiness */ -public class GUIHandler extends Handler<AbstractConfigProducer<?>> { +public class GUIHandler extends Handler { public static final String BUNDLE = "container-search-gui"; public static final String CLASS = "com.yahoo.search.query.gui.GUIHandler"; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 1c47f1d7c9c..956d7e5c366 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -222,7 +222,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { } if (deployState.zone().system().isPublic()) { BindingPattern bindingPattern = SystemBindingPattern.fromHttpPath("/validate-secret-store"); - Handler<AbstractConfigProducer<?>> handler = new Handler<>( + Handler handler = new Handler( new ComponentModel("com.yahoo.jdisc.cloud.aws.AwsParameterStoreValidationHandler", null, "jdisc-cloud-aws", null)); handler.addServerBindings(bindingPattern); cluster.addComponent(handler); @@ -897,7 +897,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { } private void addGUIHandler(ApplicationContainerCluster cluster) { - Handler<?> guiHandler = new GUIHandler(); + Handler guiHandler = new GUIHandler(); guiHandler.addServerBindings(SystemBindingPattern.fromHttpPath(GUIHandler.BINDING_PATH)); cluster.addComponent(guiHandler); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/SearchHandler.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/SearchHandler.java index 54d943f498a..6b0bf8a67b9 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/SearchHandler.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/SearchHandler.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container.xml; +import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.container.handler.threadpool.ContainerThreadpoolConfig; import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.ContainerThreadpool; @@ -11,6 +12,8 @@ import com.yahoo.vespa.model.container.search.searchchain.SearchChains; import java.util.List; +import static com.yahoo.container.bundle.BundleInstantiationSpecification.getInternalHandlerSpecificationFromStrings; + /** * Component definition for {@link com.yahoo.search.handler.SearchHandler} * @@ -19,38 +22,32 @@ import java.util.List; class SearchHandler extends ProcessingHandler<SearchChains> { static final String HANDLER_CLASS = com.yahoo.search.handler.SearchHandler.class.getName(); + static final BundleInstantiationSpecification HANDLER_SPEC = getInternalHandlerSpecificationFromStrings(HANDLER_CLASS, null); static final BindingPattern DEFAULT_BINDING = SystemBindingPattern.fromHttpPath("/search/*"); SearchHandler(ApplicationContainerCluster cluster, List<BindingPattern> bindings, ContainerThreadpool.UserOptions threadpoolOptions) { - super(cluster.getSearchChains(), HANDLER_CLASS); + super(cluster.getSearchChains(), HANDLER_SPEC, new Threadpool(threadpoolOptions)); bindings.forEach(this::addServerBindings); - Threadpool threadpool = new Threadpool(cluster, threadpoolOptions); - inject(threadpool); - addComponent(threadpool); } + private static class Threadpool extends ContainerThreadpool { - private final ApplicationContainerCluster cluster; - Threadpool(ApplicationContainerCluster cluster, UserOptions options) { + Threadpool(UserOptions options) { super("search-handler", options); - this.cluster = cluster; } @Override - public void getConfig(ContainerThreadpoolConfig.Builder builder) { - super.getConfig(builder); - - builder.maxThreadExecutionTimeSeconds(190); - builder.keepAliveTime(5.0); - - // User options overrides below configuration - if (hasUserOptions()) return; - builder.maxThreads(-2).minThreads(-2).queueSize(-40); + public void setDefaultConfigValues(ContainerThreadpoolConfig.Builder builder) { + builder.maxThreadExecutionTimeSeconds(190) + .keepAliveTime(5.0) + .maxThreads(-2) + .minThreads(-2) + .queueSize(-40); } - } + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java index 7bf08461df7..d50aedeafa0 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java @@ -70,7 +70,7 @@ public class MetricsProxyContainerClusterTest { @Test public void http_handlers_are_set_up() { VespaModel model = getModel(servicesWithAdminOnly(), self_hosted); - Collection<Handler<?>> handlers = model.getAdmin().getMetricsProxyCluster().getHandlers(); + Collection<Handler> handlers = model.getAdmin().getMetricsProxyCluster().getHandlers(); Collection<ComponentSpecification> handlerClasses = handlers.stream().map(Component::getClassId).collect(toList()); assertTrue(handlerClasses.contains(ComponentSpecification.fromString(MetricsV1Handler.class.getName()))); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java index 1a41d2689a2..072f88602a6 100755 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java @@ -36,16 +36,6 @@ public class DomAdminV2BuilderTest extends DomBuilderTest { root = new MockRoot("root"); } - // Supported for backwards compatibility - private Element servicesConfigserver() { - return XML.getDocument( - "<admin version=\"2.0\">" + - " <configserver hostalias=\"mockhost\"/>" + - " <adminserver hostalias=\"mockhost\"/>" + - "</admin>").getDocumentElement(); - - } - private Element servicesOverride() { return XML.getDocument( "<admin version=\"2.0\">" + @@ -131,15 +121,6 @@ public class DomAdminV2BuilderTest extends DomBuilderTest { } /** - * Tests that configserver works (deprecated, but allowed in admin 2.0) - */ - @Test - public void adminWithConfigserverElement() { - Admin admin = buildAdmin(servicesConfigserver()); - assertEquals(1, admin.getConfigservers().size()); - } - - /** * Tests that configservers/configserver works */ @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java index 81b6de8f2ee..d9659cf92d4 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java @@ -89,7 +89,7 @@ public class AccessLogTest extends ContainerModelBuilderTestBase { assertNotNull(getVespaAccessLog("default")); { // vespa - Component<?, ?> accessLogComponent = getContainerComponent("default", VespaAccessLog.class.getName()); + Component<?, ?> accessLogComponent = getComponent("default", VespaAccessLog.class.getName()); assertNotNull(accessLogComponent); assertEquals(VespaAccessLog.class.getName(), accessLogComponent.getClassId().getName(), VespaAccessLog.class.getName()); AccessLogConfig config = root.getConfig(AccessLogConfig.class, "default/component/com.yahoo.container.logging.VespaAccessLog"); @@ -101,7 +101,7 @@ public class AccessLogTest extends ContainerModelBuilderTestBase { } { // json - Component<?, ?> accessLogComponent = getContainerComponent("default", JSONAccessLog.class.getName()); + Component<?, ?> accessLogComponent = getComponent("default", JSONAccessLog.class.getName()); assertNotNull(accessLogComponent); assertEquals(JSONAccessLog.class.getName(), accessLogComponent.getClassId().getName(), JSONAccessLog.class.getName()); AccessLogConfig config = root.getConfig(AccessLogConfig.class, "default/component/com.yahoo.container.logging.JSONAccessLog"); @@ -124,7 +124,7 @@ public class AccessLogTest extends ContainerModelBuilderTestBase { nodesXml, "</container>" ); createModel(root, clusterElem); - Component<?, ?> connectionLogComponent = getContainerComponent("default", FileConnectionLog.class.getName()); + Component<?, ?> connectionLogComponent = getComponent("default", FileConnectionLog.class.getName()); assertNotNull(connectionLogComponent); ConnectionLogConfig config = root.getConfig(ConnectionLogConfig.class, "default/component/com.yahoo.container.logging.FileConnectionLog"); assertEquals("default", config.cluster()); @@ -140,7 +140,7 @@ public class AccessLogTest extends ContainerModelBuilderTestBase { nodesXml, "</container>" ); createModel(root, clusterElem); - Component<?, ?> fileConnectionLogComponent = getContainerComponent("default", FileConnectionLog.class.getName()); + Component<?, ?> fileConnectionLogComponent = getComponent("default", FileConnectionLog.class.getName()); assertNull(fileConnectionLogComponent); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerDocumentApiBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerDocumentApiBuilderTest.java index ca0b4681e51..3cab01fd038 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerDocumentApiBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerDocumentApiBuilderTest.java @@ -30,11 +30,11 @@ import static org.junit.Assert.assertTrue; */ public class ContainerDocumentApiBuilderTest extends ContainerModelBuilderTestBase { - private Map<String, Handler<?>> getHandlers(String clusterName) { + private Map<String, Handler> getHandlers(String clusterName) { ContainerCluster<?> cluster = (ContainerCluster<?>) root.getChildren().get(clusterName); - Map<String, Handler<?>> handlerMap = new HashMap<>(); - Collection<Handler<?>> handlers = cluster.getHandlers(); - for (Handler<?> handler : handlers) { + Map<String, Handler> handlerMap = new HashMap<>(); + Collection<Handler> handlers = cluster.getHandlers(); + for (Handler handler : handlers) { assertFalse(handlerMap.containsKey(handler.getComponentId().toString())); //die on overwrites handlerMap.put(handler.getComponentId().toString(), handler); } @@ -56,7 +56,7 @@ public class ContainerDocumentApiBuilderTest extends ContainerModelBuilderTestBa } private void verifyCustomBindings(String id) { - Handler<?> handler = getHandlers("cluster1").get(id); + Handler handler = getHandlers("cluster1").get(id); assertTrue(handler.getServerBindings().contains(UserBindingPattern.fromHttpPath("/document-api/reserved-for-internal-use/feedapi"))); assertTrue(handler.getServerBindings().contains(UserBindingPattern.fromHttpPath("/document-api/reserved-for-internal-use/feedapi/"))); @@ -73,7 +73,7 @@ public class ContainerDocumentApiBuilderTest extends ContainerModelBuilderTestBa "</container>"); createModel(root, elem); - Map<String, Handler<?>> handlerMap = getHandlers("cluster1"); + Map<String, Handler> handlerMap = getHandlers("cluster1"); assertNotNull(handlerMap.get("com.yahoo.container.handler.VipStatusHandler")); assertNotNull(handlerMap.get("com.yahoo.container.handler.observability.ApplicationStatusHandler")); @@ -110,8 +110,8 @@ public class ContainerDocumentApiBuilderTest extends ContainerModelBuilderTestBa "</container>"); root = new MockRoot("root", new MockApplicationPackage.Builder().build()); createModel(root, elem); - Map<String, Handler<?>> handlers = getHandlers("cluster1"); - Handler<?> feedApiHandler = handlers.get("com.yahoo.vespa.http.server.FeedHandler"); + Map<String, Handler> handlers = getHandlers("cluster1"); + Handler feedApiHandler = handlers.get("com.yahoo.vespa.http.server.FeedHandler"); Set<String> injectedComponentIds = feedApiHandler.getInjectedComponentIds(); assertTrue(injectedComponentIds.contains("threadpool@feedapi-handler")); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java index 3a241d9607f..2ab81a7a4bb 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java @@ -21,7 +21,6 @@ import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.Zone; import com.yahoo.config.provisioning.FlavorsConfig; -import com.yahoo.container.ComponentsConfig; import com.yahoo.container.QrConfig; import com.yahoo.container.core.ChainsConfig; import com.yahoo.container.core.VipStatusConfig; @@ -42,6 +41,7 @@ import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.ContainerModelEvaluation; import com.yahoo.vespa.model.container.component.Component; +import com.yahoo.vespa.model.container.component.Handler; import com.yahoo.vespa.model.content.utils.ContentClusterUtils; import com.yahoo.vespa.model.test.VespaModelTester; import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithFilePkg; @@ -60,19 +60,14 @@ import java.util.stream.Collectors; import static com.yahoo.config.model.test.TestUtil.joinLines; import static com.yahoo.test.LinePatternMatcher.containsLineWithPattern; import static com.yahoo.vespa.defaults.Defaults.getDefaults; -import static com.yahoo.vespa.model.container.ContainerCluster.ROOT_HANDLER_BINDING; -import static com.yahoo.vespa.model.container.ContainerCluster.STATE_HANDLER_BINDING_1; -import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.not; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.hasItem; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -198,6 +193,17 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { } @Test + public void builtin_handlers_get_default_threadpool() { + createBasicContainerModel(); + + Handler h1 = getHandler("default", ApplicationStatusHandler.class.getName()); + assertTrue(h1.getInjectedComponentIds().contains("threadpool@default-handler-common")); + + Handler h2 = getHandler("default", BindingsOverviewHandler.class.getName()); + assertTrue(h2.getInjectedComponentIds().contains("threadpool@default-handler-common")); + } + + @Test public void verify_bindings_for_builtin_handlers() { createBasicContainerModel(); JdiscBindingsConfig config = root.getConfig(JdiscBindingsConfig.class, "default/container.0"); @@ -216,68 +222,6 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { } @Test - public void default_root_handler_binding_can_be_stolen_by_user_configured_handler() { - Element clusterElem = DomBuilderTest.parse( - "<container id='default' version='1.0'>" + - " <handler id='userRootHandler'>" + - " <binding>" + ROOT_HANDLER_BINDING.patternString() + "</binding>" + - " </handler>" + - "</container>"); - createModel(root, clusterElem); - - // The handler is still set up. - ComponentsConfig.Components userRootHandler = getComponent(componentsConfig(), BindingsOverviewHandler.class.getName()); - assertNotNull(userRootHandler); - - // .. but it has no bindings - var discBindingsConfig = root.getConfig(JdiscBindingsConfig.class, "default"); - assertNull(discBindingsConfig.handlers(BindingsOverviewHandler.class.getName())); - } - - @Test - public void reserved_binding_cannot_be_stolen_by_user_configured_handler() { - Element clusterElem = DomBuilderTest.parse( - "<container id='default' version='1.0'>" + - " <handler id='userHandler'>" + - " <binding>" + STATE_HANDLER_BINDING_1.patternString() + "</binding>" + - " </handler>" + - "</container>"); - try { - createModel(root, clusterElem); - fail("Expected exception when stealing a reserved binding."); - } catch (IllegalArgumentException e) { - assertThat(e.getMessage(), is("Binding 'http://*/state/v1' is a reserved Vespa binding " + - "and cannot be used by handler: userHandler")); - } - } - - @Test - public void handler_bindings_are_included_in_discBindings_config() { - createClusterWithJDiscHandler(); - String discBindingsConfig = root.getConfig(JdiscBindingsConfig.class, "default").toString(); - assertThat(discBindingsConfig, containsString(".serverBindings[0] \"http://*/binding0\"")); - assertThat(discBindingsConfig, containsString(".serverBindings[1] \"http://*/binding1\"")); - } - - @Test - public void handlers_are_included_in_components_config() { - createClusterWithJDiscHandler(); - assertThat(componentsConfig().toString(), containsString(".id \"discHandler\"")); - } - - private void createClusterWithJDiscHandler() { - Element clusterElem = DomBuilderTest.parse( - "<container id='default' version='1.0'>", - " <handler id='discHandler'>", - " <binding>http://*/binding0</binding>", - " <binding>http://*/binding1</binding>", - " </handler>", - "</container>"); - - createModel(root, clusterElem); - } - - @Test public void processing_handler_bindings_can_be_overridden() { Element clusterElem = DomBuilderTest.parse( "<container id='default' version='1.0'>", @@ -375,20 +319,6 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { } @Test - public void nested_components_are_injected_to_handlers() { - Element clusterElem = DomBuilderTest.parse( - "<container id='default' version='1.0'>", - " <handler id='myHandler'>", - " <component id='injected' />", - " </handler>", - "</container>"); - - createModel(root, clusterElem); - Component<?,?> handler = getContainerComponent("default", "myHandler"); - assertThat(handler.getInjectedComponentIds(), hasItem("injected@myHandler")); - } - - @Test public void component_includes_are_added() { VespaModelCreatorWithFilePkg creator = new VespaModelCreatorWithFilePkg("src/test/cfg/application/include_dirs"); VespaModel model = creator.create(true); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTestBase.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTestBase.java index 3d7b17d37e0..dc831ef863d 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTestBase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTestBase.java @@ -13,6 +13,7 @@ import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.ContainerModel; import com.yahoo.vespa.model.container.component.Component; +import com.yahoo.vespa.model.container.component.Handler; import com.yahoo.vespa.model.container.search.ContainerSearch; import org.junit.Before; import org.w3c.dom.Element; @@ -94,7 +95,7 @@ public abstract class ContainerModelBuilderTestBase { return root.getConfig(ComponentsConfig.class, "default"); } - protected ComponentsConfig.Components getComponent(ComponentsConfig componentsConfig, String id) { + protected ComponentsConfig.Components getComponentInConfig(ComponentsConfig componentsConfig, String id) { for (ComponentsConfig.Components component : componentsConfig.components()) { if (component.id().equals(id)) return component; @@ -106,11 +107,18 @@ public abstract class ContainerModelBuilderTestBase { return (ApplicationContainerCluster) root.getChildren().get(clusterId); } - public Component<?, ?> getContainerComponent(String clusterId, String componentId) { + public Component<?, ?> getComponent(String clusterId, String componentId) { return getContainerCluster(clusterId).getComponentsMap().get( ComponentId.fromString(componentId)); } + public Handler getHandler(String clusterId, String componentId) { + Component<?,?> component = getComponent(clusterId, componentId); + if (! (component instanceof Handler)) + throw new RuntimeException("Component is not a handler: " + componentId); + return (Handler) component; + } + void assertComponentConfigured(ApplicationContainerCluster cluster, String componentId) { Component<?, ?> component = cluster.getComponentsMap().get(ComponentId.fromString(componentId)); assertNotNull(component); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/HandlerBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/HandlerBuilderTest.java new file mode 100644 index 00000000000..42cda0d8034 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/HandlerBuilderTest.java @@ -0,0 +1,118 @@ +package com.yahoo.vespa.model.container.xml; + +import com.yahoo.config.model.builder.xml.test.DomBuilderTest; +import com.yahoo.container.ComponentsConfig; +import com.yahoo.container.jdisc.JdiscBindingsConfig; +import com.yahoo.container.usability.BindingsOverviewHandler; +import com.yahoo.vespa.model.container.ApplicationContainerCluster; +import com.yahoo.vespa.model.container.component.Component; +import com.yahoo.vespa.model.container.component.Handler; +import org.junit.Test; +import org.w3c.dom.Element; + +import static com.yahoo.vespa.model.container.ContainerCluster.ROOT_HANDLER_BINDING; +import static com.yahoo.vespa.model.container.ContainerCluster.STATE_HANDLER_BINDING_1; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.hasItem; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * Tests for container model building with custom handlers. + * + * @author gjoranv + */ +public class HandlerBuilderTest extends ContainerModelBuilderTestBase { + + @Test + public void handlers_are_included_in_components_config() { + createClusterWithJDiscHandler(); + assertThat(componentsConfig().toString(), containsString(".id \"discHandler\"")); + } + + @Test + public void handler_bindings_are_included_in_discBindings_config() { + createClusterWithJDiscHandler(); + String discBindingsConfig = root.getConfig(JdiscBindingsConfig.class, "default").toString(); + assertThat(discBindingsConfig, containsString(".serverBindings[0] \"http://*/binding0\"")); + assertThat(discBindingsConfig, containsString(".serverBindings[1] \"http://*/binding1\"")); + } + + @Test + public void nested_components_are_injected_to_handlers() { + Element clusterElem = DomBuilderTest.parse( + "<container id='default' version='1.0'>", + " <handler id='myHandler'>", + " <component id='injected' />", + " </handler>", + "</container>"); + + createModel(root, clusterElem); + Component<?,?> handler = getComponent("default", "myHandler"); + assertThat(handler.getInjectedComponentIds(), hasItem("injected@myHandler")); + } + + @Test + public void default_root_handler_binding_can_be_stolen_by_user_configured_handler() { + Element clusterElem = DomBuilderTest.parse( + "<container id='default' version='1.0'>" + + " <handler id='userRootHandler'>" + + " <binding>" + ROOT_HANDLER_BINDING.patternString() + "</binding>" + + " </handler>" + + "</container>"); + createModel(root, clusterElem); + + // The handler is still set up. + ComponentsConfig.Components userRootHandler = getComponentInConfig(componentsConfig(), BindingsOverviewHandler.class.getName()); + assertNotNull(userRootHandler); + + // .. but it has no bindings + var discBindingsConfig = root.getConfig(JdiscBindingsConfig.class, "default"); + assertNull(discBindingsConfig.handlers(BindingsOverviewHandler.class.getName())); + } + + @Test + public void reserved_binding_cannot_be_stolen_by_user_configured_handler() { + Element clusterElem = DomBuilderTest.parse( + "<container id='default' version='1.0'>" + + " <handler id='userHandler'>" + + " <binding>" + STATE_HANDLER_BINDING_1.patternString() + "</binding>" + + " </handler>" + + "</container>"); + try { + createModel(root, clusterElem); + fail("Expected exception when stealing a reserved binding."); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), is("Binding 'http://*/state/v1' is a reserved Vespa binding " + + "and cannot be used by handler: userHandler")); + } + } + + @Test + public void custom_handler_gets_default_threadpool() { + createClusterWithJDiscHandler(); + ApplicationContainerCluster cluster = (ApplicationContainerCluster)root.getChildren().get("default"); + Handler handler = cluster.getHandlers().stream() + .filter(h -> h.getComponentId().toString().equals("discHandler")) + .findAny().orElseThrow(); + + assertTrue(handler.getInjectedComponentIds().contains("threadpool@default-handler-common")); + } + + private void createClusterWithJDiscHandler() { + Element clusterElem = DomBuilderTest.parse( + "<container id='default' version='1.0'>", + " <handler id='discHandler'>", + " <binding>http://*/binding0</binding>", + " <binding>http://*/binding1</binding>", + " </handler>", + "</container>"); + + createModel(root, clusterElem); + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ApplicationBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/SearchBuilderTest.java index ca59e053a89..c41373b9f85 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ApplicationBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/SearchBuilderTest.java @@ -31,7 +31,7 @@ import static org.junit.Assert.fail; /** * @author gjoranv */ -public class ApplicationBuilderTest extends ContainerModelBuilderTestBase { +public class SearchBuilderTest extends ContainerModelBuilderTestBase { private ChainsConfig chainsConfig() { return root.getConfig(ChainsConfig.class, "default/component/com.yahoo.search.handler.SearchHandler"); @@ -53,7 +53,7 @@ public class ApplicationBuilderTest extends ContainerModelBuilderTestBase { ApplicationContainerCluster cluster = (ApplicationContainerCluster)root.getChildren().get("default"); GUIHandler guiHandler = null; - for (Handler<?> handler : cluster.getHandlers()) { + for (Handler handler : cluster.getHandlers()) { if (handler instanceof GUIHandler) { guiHandler = (GUIHandler) handler; } @@ -230,12 +230,7 @@ public class ApplicationBuilderTest extends ContainerModelBuilderTestBase { createModel(root, clusterElem); - ApplicationContainerCluster cluster = (ApplicationContainerCluster)root.getChildren().get("default"); - Handler<?> searchHandler = cluster.getHandlers().stream() - .filter(h -> h.getComponentId().toString().equals(SearchHandler.HANDLER_CLASS)) - .findAny() - .get(); - + Handler searchHandler = getHandler("default", SearchHandler.HANDLER_CLASS); assertTrue(searchHandler.getInjectedComponentIds().contains("threadpool@search-handler")); ContainerThreadpoolConfig config = root.getConfig( diff --git a/config-model/src/test/schema-test-files/services.xml b/config-model/src/test/schema-test-files/services.xml index 4fb81963152..df87d2b9f3d 100644 --- a/config-model/src/test/schema-test-files/services.xml +++ b/config-model/src/test/schema-test-files/services.xml @@ -193,6 +193,9 @@ <handler id="jdisc-handler"> <binding>http://*:*/HelloWorld</binding> <binding>http://*:*/Status</binding> + <component id="injected-to-handler"> + <config name="foo"/> + </component> </handler> <server id="server-provider"> diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index ca56a200c2c..7949c2633e0 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -911,14 +911,10 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye sessionsPerTenant.keySet().forEach(tenant -> tenant.getSessionRepository().deleteExpiredSessions(activeSessions)); } - public int deleteExpiredRemoteSessions(Duration expiryTime) { - return deleteExpiredRemoteSessions(clock, expiryTime); - } - - public int deleteExpiredRemoteSessions(Clock clock, Duration expiryTime) { + public int deleteExpiredRemoteSessions(Clock clock) { return tenantRepository.getAllTenants() .stream() - .map(tenant -> tenant.getSessionRepository().deleteExpiredRemoteSessions(clock, expiryTime)) + .map(tenant -> tenant.getSessionRepository().deleteExpiredRemoteSessions(clock)) .mapToInt(i -> i) .sum(); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/SessionsMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/SessionsMaintainer.java index d980fb079e7..38ac41d0eb9 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/SessionsMaintainer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/SessionsMaintainer.java @@ -16,22 +16,17 @@ import java.util.logging.Level; * @author hmusum */ public class SessionsMaintainer extends ConfigServerMaintainer { - private final boolean hostedVespa; SessionsMaintainer(ApplicationRepository applicationRepository, Curator curator, Duration interval, FlagSource flagSource) { super(applicationRepository, curator, flagSource, applicationRepository.clock().instant(), interval, true); - this.hostedVespa = applicationRepository.configserverConfig().hostedVespa(); } @Override protected double maintain() { applicationRepository.deleteExpiredLocalSessions(); - if (hostedVespa) { - Duration expiryTime = Duration.ofMinutes(90); - int deleted = applicationRepository.deleteExpiredRemoteSessions(expiryTime); - log.log(Level.FINE, () -> "Deleted " + deleted + " expired remote sessions older than " + expiryTime); - } + int deleted = applicationRepository.deleteExpiredRemoteSessions(applicationRepository.clock()); + log.log(Level.FINE, () -> "Deleted " + deleted + " expired remote sessions"); return 1.0; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java index d803488cb0a..bf596ddf985 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java @@ -344,13 +344,17 @@ public class SessionRepository { public RemoteSession createRemoteSession(long sessionId) { SessionZooKeeperClient sessionZKClient = createSessionZooKeeperClient(sessionId); RemoteSession session = new RemoteSession(tenantName, sessionId, sessionZKClient); - RemoteSession newSession = loadSessionIfActive(session).orElse(session); - remoteSessionCache.put(sessionId, newSession); - updateSessionStateWatcher(sessionId, newSession); - return newSession; + loadSessionIfActive(session); + remoteSessionCache.put(sessionId, session); + updateSessionStateWatcher(sessionId); + return session; } - public int deleteExpiredRemoteSessions(Clock clock, Duration expiryTime) { + public int deleteExpiredRemoteSessions(Clock clock) { + Duration expiryTime = configserverConfig.hostedVespa() + ? sessionLifetime.multipliedBy(2) + : sessionLifetime.multipliedBy(24); // TODO: Remove when tested more (Sep. 2022 at the latest) + List<Long> remoteSessionsFromZooKeeper = getRemoteSessionsFromZooKeeper(); log.log(Level.FINE, () -> "Remote sessions for tenant " + tenantName + ": " + remoteSessionsFromZooKeeper); @@ -375,9 +379,11 @@ public class SessionRepository { return deleted; } - public void deactivateAndUpdateCache(RemoteSession remoteSession) { - RemoteSession session = remoteSession.deactivated(); - remoteSessionCache.put(session.getSessionId(), session); + public void deactivateAndUpdateCache(long sessionId) { + var s = remoteSessionCache.get(sessionId); + if (s == null) return; + + remoteSessionCache.put(sessionId, s.deactivated()); } public void deleteRemoteSessionFromZooKeeper(Session session) { @@ -388,7 +394,7 @@ public class SessionRepository { } private boolean sessionHasExpired(Instant created, Duration expiryTime, Clock clock) { - return (created.plus(expiryTime).isBefore(clock.instant())); + return created.plus(expiryTime).isBefore(clock.instant()); } private List<Long> getSessionListFromDirectoryCache(List<ChildData> children) { @@ -439,8 +445,11 @@ public class SessionRepository { return session.getStatus() == Session.Status.DELETE; } - void activate(RemoteSession session) { - long sessionId = session.getSessionId(); + void activate(long sessionId) { + createLocalSessionFromDistributedApplicationPackage(sessionId); + RemoteSession session = remoteSessionCache.get(sessionId); + if (session == null) return; + CompletionWaiter waiter = createSessionZooKeeperClient(sessionId).getActiveWaiter(); log.log(Level.FINE, () -> session.logPre() + "Activating " + sessionId); applicationRepo.activateApplication(ensureApplicationLoaded(session), sessionId); @@ -449,21 +458,25 @@ public class SessionRepository { log.log(Level.INFO, session.logPre() + "Session activated: " + sessionId); } - private Optional<RemoteSession> loadSessionIfActive(RemoteSession session) { + private void loadSessionIfActive(RemoteSession session) { for (ApplicationId applicationId : applicationRepo.activeApplications()) { Optional<Long> activeSession = applicationRepo.activeSessionOf(applicationId); if (activeSession.isPresent() && activeSession.get() == session.getSessionId()) { log.log(Level.FINE, () -> "Found active application for session " + session.getSessionId() + " , loading it"); applicationRepo.activateApplication(ensureApplicationLoaded(session), session.getSessionId()); log.log(Level.INFO, session.logPre() + "Application activated successfully: " + applicationId + " (generation " + session.getSessionId() + ")"); - return Optional.ofNullable(remoteSessionCache.get(session.getSessionId())); + return; } } - return Optional.empty(); } - void prepareRemoteSession(RemoteSession session) { - SessionZooKeeperClient sessionZooKeeperClient = createSessionZooKeeperClient(session.getSessionId()); + void prepareRemoteSession(long sessionId) { + // Might need to create local session first + createLocalSessionFromDistributedApplicationPackage(sessionId); + RemoteSession session = remoteSessionCache.get(sessionId); + if (session == null) return; + + SessionZooKeeperClient sessionZooKeeperClient = createSessionZooKeeperClient(sessionId); CompletionWaiter waiter = sessionZooKeeperClient.getPrepareWaiter(); ensureApplicationLoaded(session); notifyCompletion(waiter); @@ -480,7 +493,7 @@ public class SessionRepository { RemoteSession activated = session.activated(applicationSet); long sessionId = activated.getSessionId(); remoteSessionCache.put(sessionId, activated); - updateSessionStateWatcher(sessionId, activated); + updateSessionStateWatcher(sessionId); return applicationSet; } @@ -817,8 +830,9 @@ public class SessionRepository { } /** - * Returns a new local session for the given session id if it does not already exist. - * Will also add the session to the local session cache if necessary + * Create a new local session for the given session id if it does not already exist. + * Will also add the session to the local session cache if necessary. If there is no + * remote session matching the session it will also be created. */ public void createLocalSessionFromDistributedApplicationPackage(long sessionId) { if (applicationRepo.sessionExistsInFileSystem(sessionId)) { @@ -886,15 +900,12 @@ public class SessionRepository { return new TenantFileSystemDirs(configServerDB, tenantName).getUserApplicationDir(sessionId); } - private void updateSessionStateWatcher(long sessionId, RemoteSession remoteSession) { - SessionStateWatcher sessionStateWatcher = sessionStateWatchers.get(sessionId); - if (sessionStateWatcher == null) { - Curator.FileCache fileCache = curator.createFileCache(getSessionStatePath(sessionId).getAbsolute(), false); + private void updateSessionStateWatcher(long sessionId) { + sessionStateWatchers.computeIfAbsent(sessionId, (id) -> { + Curator.FileCache fileCache = curator.createFileCache(getSessionStatePath(id).getAbsolute(), false); fileCache.addListener(this::nodeChanged); - sessionStateWatchers.put(sessionId, new SessionStateWatcher(fileCache, remoteSession, metricUpdater, zkWatcherExecutor, this)); - } else { - sessionStateWatcher.updateRemoteSession(remoteSession); - } + return new SessionStateWatcher(fileCache, id, metricUpdater, zkWatcherExecutor, this); + }); } @Override diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionStateWatcher.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionStateWatcher.java index 6c6f60426fe..abd2266fcf3 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionStateWatcher.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionStateWatcher.java @@ -13,7 +13,7 @@ import java.util.logging.Logger; import static com.yahoo.vespa.config.server.session.Session.Status; /** - * Watches one particular session (/config/v2/tenants/<tenantName>/sessions/<n>/sessionState in ZooKeeper) + * Watches session state for a session (/config/v2/tenants/<tenantName>/sessions/<n>/sessionState in ZooKeeper) * The session must be in the session repo. * * @author Vegard Havdal @@ -24,18 +24,18 @@ public class SessionStateWatcher { private static final Logger log = Logger.getLogger(SessionStateWatcher.class.getName()); private final Curator.FileCache fileCache; - private volatile RemoteSession session; + private final long sessionId; private final MetricUpdater metrics; private final Executor zkWatcherExecutor; private final SessionRepository sessionRepository; SessionStateWatcher(Curator.FileCache fileCache, - RemoteSession session, + long sessionId, MetricUpdater metrics, Executor zkWatcherExecutor, SessionRepository sessionRepository) { this.fileCache = fileCache; - this.session = session; + this.sessionId = sessionId; this.metrics = metrics; this.fileCache.addListener(this::nodeChanged); this.fileCache.start(); @@ -44,37 +44,27 @@ public class SessionStateWatcher { } private synchronized void sessionStatusChanged(Status newStatus) { - long sessionId = session.getSessionId(); - switch (newStatus) { case NEW: case UNKNOWN: break; case DELETE: - sessionRepository.deactivateAndUpdateCache(session); + case DEACTIVATE: + sessionRepository.deactivateAndUpdateCache(sessionId); break; case PREPARE: - createLocalSession(sessionId); - sessionRepository.prepareRemoteSession(session); + sessionRepository.prepareRemoteSession(sessionId); break; case ACTIVATE: - createLocalSession(sessionId); - sessionRepository.activate(session); - break; - case DEACTIVATE: - sessionRepository.deactivateAndUpdateCache(session); + sessionRepository.activate(sessionId); break; default: throw new IllegalStateException("Unknown status " + newStatus); } } - private void createLocalSession(long sessionId) { - sessionRepository.createLocalSessionFromDistributedApplicationPackage(sessionId); - } - public long getSessionId() { - return session.getSessionId(); + return sessionId; } public void close() { @@ -93,10 +83,9 @@ public class SessionStateWatcher { if (node != null) { newStatus = Status.parse(Utf8.toString(node.getData())); - String debugMessage = log.isLoggable(Level.FINE) ? - session.logPre() + "Session " + session.getSessionId() - + " changed status to " + newStatus.name() : - null; + String debugMessage = log.isLoggable(Level.FINE) + ? "Session " + sessionId + " changed status to " + newStatus.name() + : null; if (debugMessage != null) log.fine(debugMessage); sessionStatusChanged(newStatus); @@ -104,15 +93,10 @@ public class SessionStateWatcher { if (debugMessage != null) log.fine(debugMessage + ": Done"); } } catch (Exception e) { - log.log(Level.WARNING, session.logPre() + "Error handling session change to " + - newStatus.name() + " for session " + getSessionId(), e); + log.log(Level.WARNING, "Error handling session change to " + newStatus.name() + " for session " + getSessionId(), e); metrics.incSessionChangeErrors(); } }); } - public synchronized void updateRemoteSession(RemoteSession session) { - this.session = session; - } - } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java index 46cf84e19c1..fe188e08036 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java @@ -446,7 +446,7 @@ public class ApplicationRepositoryTest { assertEquals(3, localSession.getSessionId()); // All sessions except 3 should be removed after the call to deleteExpiredRemoteSessions - assertEquals(2, tester.applicationRepository().deleteExpiredRemoteSessions(clock, Duration.ofSeconds(0))); + assertEquals(2, tester.applicationRepository().deleteExpiredRemoteSessions(clock)); ArrayList<Long> remoteSessions = new ArrayList<>(sessionRepository.getRemoteSessionsFromZooKeeper()); Session remoteSession = sessionRepository.getRemoteSession(remoteSessions.get(0)); assertEquals(3, remoteSession.getSessionId()); diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java index 11355dee24f..fb7ba1ab39e 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java @@ -50,7 +50,7 @@ public abstract class ThreadedHttpRequestHandler extends ThreadedRequestHandler this(executor, metric, false); } - // TODO: move Inject annotation here! + // TODO: deprecate this and the Context class. The context component set up in the model does not get a dedicated thread pool. public ThreadedHttpRequestHandler(Context context) { this(context.executor, context.metric); } diff --git a/container-core/src/test/java/com/yahoo/container/di/componentgraph/core/ComponentGraphTest.java b/container-core/src/test/java/com/yahoo/container/di/componentgraph/core/ComponentGraphTest.java index f8f567e1890..5abb96527bd 100644 --- a/container-core/src/test/java/com/yahoo/container/di/componentgraph/core/ComponentGraphTest.java +++ b/container-core/src/test/java/com/yahoo/container/di/componentgraph/core/ComponentGraphTest.java @@ -102,7 +102,7 @@ public class ComponentGraphTest { } @Test - public void component_can_be_injected_into_another_component() { + public void component_can_be_explicitly_injected_into_another_component() { Node injectedComponent = mockComponentNode(SimpleComponent.class); Node targetComponent = mockComponentNode(ComponentTakingComponent.class); targetComponent.inject(injectedComponent); @@ -117,6 +117,22 @@ public class ComponentGraphTest { ComponentTakingComponent instance = componentGraph.getInstance(ComponentTakingComponent.class); assertNotNull(instance); + assertSame(injectedComponent.instance.get(), instance.injectedComponent); + } + + @Test + public void explicitly_injected_components_may_be_unused() { + Node notUsingInjected = mockComponentNode(SimpleComponent.class); + Node injectedComponent = mockComponentNode(SimpleComponent2.class); + notUsingInjected.inject(injectedComponent); + + ComponentGraph componentGraph = new ComponentGraph(); + componentGraph.add(injectedComponent); + componentGraph.add(notUsingInjected); + componentGraph.complete(); + + SimpleComponent instanceNotUsingInjected = componentGraph.getInstance(SimpleComponent.class); + assertNotNull(instanceNotUsingInjected); } @Test @@ -514,7 +530,7 @@ public class ComponentGraphTest { } public static class ComponentTakingComponent extends AbstractComponent { - private final SimpleComponent injectedComponent; + final SimpleComponent injectedComponent; public ComponentTakingComponent(SimpleComponent injectedComponent) { assertNotNull(injectedComponent); diff --git a/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java b/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java index 384ac4fad33..b5a292a554d 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java @@ -114,7 +114,7 @@ public class PermanentFlags { ZONE_ID, APPLICATION_ID); public static final UnboundStringFlag ZOOKEEPER_SERVER_VERSION = defineStringFlag( - "zookeeper-server-version", "3.7.1", + "zookeeper-server-version", "3.7.1", // Note: Nodes running Vespa 7 have 3.7.1 as the only available version "ZooKeeper server version, a jar file zookeeper-server-<ZOOKEEPER_SERVER_VERSION>-jar-with-dependencies.jar must exist", "Takes effect on restart of Docker container", NODE_TYPE, APPLICATION_ID, HOSTNAME); diff --git a/fnet/src/vespa/fnet/connection.h b/fnet/src/vespa/fnet/connection.h index 4d66f22ce2b..10cf74e79de 100644 --- a/fnet/src/vespa/fnet/connection.h +++ b/fnet/src/vespa/fnet/connection.h @@ -240,10 +240,6 @@ private: bool writePendingAfterConnect(); - /** - * @return address spec of socket peer. Only makes sense to call on non-listening sockets. - */ - vespalib::string GetPeerSpec() const; public: FNET_Connection(const FNET_Connection &) = delete; FNET_Connection &operator=(const FNET_Connection &) = delete; @@ -309,6 +305,10 @@ public: return ((_currentID & 0x01) != (chid & 0x01)); } + /** + * @return address spec of socket peer. Only makes sense to call on non-listening sockets. + */ + vespalib::string GetPeerSpec() const; /** * Does this connection have the ability to accept incoming channels ? diff --git a/fnet/src/vespa/fnet/frt/require_capabilities.cpp b/fnet/src/vespa/fnet/frt/require_capabilities.cpp index c74e9ad648a..fc64621717f 100644 --- a/fnet/src/vespa/fnet/frt/require_capabilities.cpp +++ b/fnet/src/vespa/fnet/frt/require_capabilities.cpp @@ -5,9 +5,25 @@ #include <vespa/fnet/connection.h> #include <vespa/vespalib/net/connection_auth_context.h> +#include <vespa/log/bufferedlogger.h> +LOG_SETUP(".fnet.frt.require_capabilities"); + +using namespace vespalib::net::tls; + bool FRT_RequireCapabilities::allow(FRT_RPCRequest& req) const noexcept { const auto& auth_ctx = req.GetConnection()->auth_context(); - return auth_ctx.capabilities().contains_all(_required_capabilities); + const bool is_authorized = auth_ctx.capabilities().contains_all(_required_capabilities); + if (!is_authorized) { + auto peer_spec = req.GetConnection()->GetPeerSpec(); + std::string method_name(req.GetMethodName(), req.GetMethodNameLen()); + LOGBT(warning, peer_spec, "Permission denied for RPC method '%s'. " + "Peer at %s with %s. Call requires %s, but peer has %s", + method_name.c_str(), peer_spec.c_str(), + to_string(auth_ctx.peer_credentials()).c_str(), + _required_capabilities.to_string().c_str(), + auth_ctx.capabilities().to_string().c_str()); + } + return is_authorized; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java index af62579b942..3c5b20da4d0 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java @@ -12,7 +12,6 @@ import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeMutex; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Agent; -import com.yahoo.vespa.hosted.provision.node.Allocation; import com.yahoo.vespa.hosted.provision.node.History; import com.yahoo.vespa.orchestrator.ApplicationIdNotFoundException; import com.yahoo.vespa.orchestrator.status.ApplicationInstanceStatus; @@ -20,11 +19,9 @@ import com.yahoo.yolean.Exceptions; import java.time.Duration; import java.time.Instant; -import java.util.ArrayList; import java.util.Collection; import java.util.HashSet; import java.util.List; -import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.logging.Level; @@ -109,6 +106,26 @@ public class NodeFailer extends NodeRepositoryMaintainer { failActive(failing); } + // Active hosts + NodeList activeNodes = nodeRepository().nodes().list(Node.State.active); + for (Node host : activeNodes.hosts().failing()) { + if ( ! activeNodes.childrenOf(host).isEmpty()) continue; + Optional<NodeMutex> locked = Optional.empty(); + try { + attempts++; + locked = nodeRepository().nodes().lockAndGet(host); + if (locked.isEmpty()) continue; + nodeRepository().nodes().fail(List.of(locked.get().node()), Agent.NodeFailer, + "Host should be failed and have no tenant nodes"); + } + catch (Exception e) { + failures++; + } + finally { + locked.ifPresent(NodeMutex::close); + } + } + int throttlingActive = Math.min(1, throttledHostFailures + throttledNodeFailures); metric.set(throttlingActiveMetric, throttlingActive, null); metric.set(throttledHostFailuresMetric, throttledHostFailures, null); @@ -136,9 +153,6 @@ public class NodeFailer extends NodeRepositoryMaintainer { Set<FailingNode> failingNodes = new HashSet<>(); NodeList activeNodes = nodeRepository().nodes().list(Node.State.active); - for (Node host : activeNodes.hosts().failing()) - failingNodes.add(new FailingNode(host, "Host should be failed and have no tenant nodes")); - for (Node node : activeNodes) { Instant graceTimeStart = clock().instant().minus(nodeRepository().nodes().suspended(node) ? suspendedDownTimeLimit : downTimeLimit); if (node.isDown() && node.history().hasEventBefore(History.Event.Type.down, graceTimeStart) && !applicationSuspended(node)) { @@ -227,63 +241,42 @@ public class NodeFailer extends NodeRepositoryMaintainer { deployer.deployFromLocalActive(failing.node().allocation().get().owner(), Duration.ofMinutes(30)); if (deployment.isEmpty()) return false; - // If the active node that we are trying to fail is of type host, we need to successfully fail all - // the children nodes running on it before we fail the host. Failing a child node in a dynamically - // provisioned zone may require provisioning new hosts that require the host application lock to be held, - // so we must release ours before failing the children. - List<FailingNode> activeChildrenToFail = new ArrayList<>(); - try (NodeMutex lock = nodeRepository().nodes().lockAndGetRequired(failing.node())) { - // Now that we have gotten the node object under the proper lock, sanity-check it still makes sense to fail - if (!Objects.equals(failing.node().allocation().map(Allocation::owner), lock.node().allocation().map(Allocation::owner))) - return false; - if (lock.node().state() == Node.State.failed) - return true; - if (!Objects.equals(failing.node().state(), lock.node().state())) - return false; - failing = new FailingNode(lock.node(), failing.reason); - + try (Mutex lock = nodeRepository().nodes().lock(failing.node().allocation().get().owner())) { + // If the active node that we are trying to fail is of type host, we need to successfully fail all + // the children nodes running on it before we fail the host + boolean allTenantNodesFailedOutSuccessfully = true; String reasonForChildFailure = "Failing due to parent host " + failing.node().hostname() + " failure: " + failing.reason(); for (Node failingTenantNode : nodeRepository().nodes().list().childrenOf(failing.node())) { if (failingTenantNode.state() == Node.State.active) { - activeChildrenToFail.add(new FailingNode(failingTenantNode, reasonForChildFailure)); - } else if (failingTenantNode.state() != Node.State.failed) { + allTenantNodesFailedOutSuccessfully &= failActive(new FailingNode(failingTenantNode, reasonForChildFailure)); + } else { nodeRepository().nodes().fail(failingTenantNode.hostname(), Agent.NodeFailer, reasonForChildFailure); } } - // A parent with children gets wantToFail to avoid getting more nodes allocated to it. + if (! allTenantNodesFailedOutSuccessfully) return false; wantToFail(failing.node(), true, lock); - - if (activeChildrenToFail.isEmpty()) { - try { - deployment.get().activate(); - return true; - } catch (TransientException e) { - log.log(Level.INFO, "Failed to redeploy " + failing.node().allocation().get().owner() + - " with a transient error, will be retried by application maintainer: " + - Exceptions.toMessageString(e)); - return true; - } catch (RuntimeException e) { - // Reset want to fail: We'll retry failing unless it heals in the meantime - nodeRepository().nodes().node(failing.node().hostname()) - .ifPresent(n -> wantToFail(n, false, lock)); - log.log(Level.WARNING, "Could not fail " + failing.node() + " for " + failing.node().allocation().get().owner() + - " for " + failing.reason() + ": " + Exceptions.toMessageString(e)); - return false; - } + try { + deployment.get().activate(); + return true; + } catch (TransientException e) { + log.log(Level.INFO, "Failed to redeploy " + failing.node().allocation().get().owner() + + " with a transient error, will be retried by application maintainer: " + + Exceptions.toMessageString(e)); + return true; + } catch (RuntimeException e) { + // Reset want to fail: We'll retry failing unless it heals in the meantime + nodeRepository().nodes().node(failing.node().hostname()) + .ifPresent(n -> wantToFail(n, false, lock)); + log.log(Level.WARNING, "Could not fail " + failing.node() + " for " + failing.node().allocation().get().owner() + + " for " + failing.reason() + ": " + Exceptions.toMessageString(e)); + return false; } } - - // In a dynamically provisioned zone the failing of the first child may require a new host to be provisioned, - // so failActive() may take a long time to complete, but the remaining children should be fast. - activeChildrenToFail.forEach(this::failActive); - - return false; } private void wantToFail(Node node, boolean wantToFail, Mutex lock) { - if (!node.status().wantToFail()) - nodeRepository().nodes().write(node.withWantToFail(wantToFail, Agent.NodeFailer, clock().instant()), lock); + nodeRepository().nodes().write(node.withWantToFail(wantToFail, Agent.NodeFailer, clock().instant()), lock); } /** Returns true if node failing should be throttled */ @@ -291,10 +284,9 @@ public class NodeFailer extends NodeRepositoryMaintainer { if (throttlePolicy == ThrottlePolicy.disabled) return false; Instant startOfThrottleWindow = clock().instant().minus(throttlePolicy.throttleWindow); NodeList allNodes = nodeRepository().nodes().list(); - NodeList recentlyFailedNodes = allNodes - .matching(n -> n.status().wantToFail() || - (n.state() == Node.State.failed && - n.history().hasEventAfter(History.Event.Type.failed, startOfThrottleWindow))); + NodeList recentlyFailedNodes = allNodes.state(Node.State.failed) + .matching(n -> n.history().hasEventAfter(History.Event.Type.failed, + startOfThrottleWindow)); // Allow failing any node within policy if (recentlyFailedNodes.size() < throttlePolicy.allowedToFailOf(allNodes.size())) return false; diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailerTest.java index 3ba536ee4d7..f7d29a116ed 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailerTest.java @@ -457,16 +457,6 @@ public class NodeFailerTest { tester.allNodesMakeAConfigRequestExcept(); tester.runMaintainers(); - assertEquals(2, tester.deployer.redeployments); - assertEquals(3, tester.nodeRepository.nodes().list(Node.State.failed).nodeType(NodeType.tenant).size()); - assertEquals(8, tester.nodeRepository.nodes().list(Node.State.active).nodeType(NodeType.tenant).size()); - assertEquals(10, tester.nodeRepository.nodes().list(Node.State.ready).nodeType(NodeType.tenant).size()); - assertEquals(7, tester.nodeRepository.nodes().list(Node.State.active).nodeType(NodeType.host).size()); - assertEquals(0, tester.nodeRepository.nodes().list(Node.State.failed).nodeType(NodeType.host).size()); - - // The failing of the host is deferred to the next maintain - tester.runMaintainers(); - assertEquals(2 + 1, tester.deployer.redeployments); assertEquals(3, tester.nodeRepository.nodes().list(Node.State.failed).nodeType(NodeType.tenant).size()); assertEquals(8, tester.nodeRepository.nodes().list(Node.State.active).nodeType(NodeType.tenant).size()); @@ -505,7 +495,6 @@ public class NodeFailerTest { tester.clock.advance(Duration.ofMinutes(90)); tester.allNodesMakeAConfigRequestExcept(); tester.runMaintainers(); - tester.runMaintainers(); // The host is failed in the 2. maintain() assertEquals(5 + 2, tester.deployer.redeployments); assertEquals(7, tester.nodeRepository.nodes().list(Node.State.failed).nodeType(NodeType.tenant).size()); @@ -598,7 +587,7 @@ public class NodeFailerTest { @Test public void node_failing_throttle() { - // Throttles based on an absolute number in small zone + // Throttles based on a absolute number in small zone { // 10 hosts with 3 tenant nodes each, total 40 nodes NodeFailTester tester = NodeFailTester.withTwoApplications(10); @@ -606,12 +595,13 @@ public class NodeFailerTest { // 3 hosts fail. 2 of them and all of their children are allowed to fail List<Node> failedHosts = hosts.asList().subList(0, 3); - failedHosts.forEach(host -> tester.serviceMonitor.setHostDown(host.hostname())); + failedHosts.forEach(host -> { + tester.serviceMonitor.setHostDown(host.hostname()); + }); tester.runMaintainers(); tester.clock.advance(Duration.ofMinutes(61)); tester.runMaintainers(); - tester.runMaintainers(); // hosts are typically failed in the 2. maintain() assertEquals(2 + /* hosts */ (2 * 3) /* containers per host */, tester.nodeRepository.nodes().list(Node.State.failed).size()); @@ -630,7 +620,6 @@ public class NodeFailerTest { // The final host and its containers are failed out tester.clock.advance(Duration.ofMinutes(30)); tester.runMaintainers(); - tester.runMaintainers(); // hosts are failed in the 2. maintain() assertEquals(12, tester.nodeRepository.nodes().list(Node.State.failed).size()); assertEquals("Throttling is not indicated by the metric, as no throttled attempt is made", 0, tester.metric.values.get(NodeFailer.throttlingActiveMetric)); assertEquals("No throttled node failures", 0, tester.metric.values.get(NodeFailer.throttledNodeFailuresMetric)); @@ -735,8 +724,7 @@ public class NodeFailerTest { .map(Map.Entry::getKey) .flatMap(parentHost -> Stream.of(parentHost.get())) .filter(node -> ! exceptSet.contains(node)) - .findFirst() - .orElseThrow(); + .findFirst().get(); } } diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp index 5bd92fd788f..155de714c8c 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp @@ -101,7 +101,9 @@ SummarySetup(const vespalib::string & baseDir, const DocTypeName & docTypeName, docsum_blob_entry_filter.add_skip(RES_FLOAT); docsum_blob_entry_filter.add_skip(RES_DOUBLE); docsum_blob_entry_filter.add_skip(RES_INT64); + docsum_blob_entry_filter.add_skip(RES_JSONSTRING); docsum_blob_entry_filter.add_skip(RES_TENSOR); + docsum_blob_entry_filter.add_skip(RES_FEATUREDATA); auto resultConfig = std::make_unique<ResultConfig>(docsum_blob_entry_filter); if (!resultConfig->ReadConfig(summaryCfg, make_string("SummaryManager(%s)", baseDir.c_str()).c_str())) { std::ostringstream oss; diff --git a/storage/src/vespa/storage/storageserver/rpc/storage_api_rpc_service.cpp b/storage/src/vespa/storage/storageserver/rpc/storage_api_rpc_service.cpp index 006193f0c37..41dcafc055a 100644 --- a/storage/src/vespa/storage/storageserver/rpc/storage_api_rpc_service.cpp +++ b/storage/src/vespa/storage/storageserver/rpc/storage_api_rpc_service.cpp @@ -5,6 +5,7 @@ #include "rpc_envelope_proto.h" #include "shared_rpc_resources.h" #include "storage_api_rpc_service.h" +#include <vespa/fnet/frt/require_capabilities.h> #include <vespa/fnet/frt/supervisor.h> #include <vespa/fnet/frt/target.h> #include <vespa/slobrok/sbmirror.h> @@ -54,6 +55,9 @@ StorageApiRpcService::Params::~Params() = default; void StorageApiRpcService::register_server_methods(SharedRpcResources& rpc_resources) { FRT_ReflectionBuilder rb(&rpc_resources.supervisor()); rb.DefineMethod(rpc_v1_method_name(), "bixbix", "bixbix", FRT_METHOD(StorageApiRpcService::RPC_rpc_v1_send), this); + rb.RequestAccessFilter(std::make_unique<FRT_RequireCapabilities>(vespalib::net::tls::CapabilitySet::of({ + vespalib::net::tls::Capability::content_storage_api() + }))); rb.MethodDesc("V1 of StorageAPI direct RPC protocol"); rb.ParamDesc("header_encoding", "0=raw, 6=lz4"); rb.ParamDesc("header_decoded_size", "Uncompressed header blob size"); diff --git a/streamingvisitors/src/vespa/vsm/vsm/vsm-adapter.cpp b/streamingvisitors/src/vespa/vsm/vsm/vsm-adapter.cpp index add52effac2..0a75780d9f7 100644 --- a/streamingvisitors/src/vespa/vsm/vsm/vsm-adapter.cpp +++ b/streamingvisitors/src/vespa/vsm/vsm/vsm-adapter.cpp @@ -139,11 +139,13 @@ VSMAdapter::configure(const VSMConfigSnapshot & snapshot) DocsumBlobEntryFilter docsum_blob_entry_filter; docsum_blob_entry_filter.add_skip(search::docsummary::RES_INT); docsum_blob_entry_filter.add_skip(search::docsummary::RES_SHORT); + docsum_blob_entry_filter.add_skip(search::docsummary::RES_BOOL); docsum_blob_entry_filter.add_skip(search::docsummary::RES_BYTE); docsum_blob_entry_filter.add_skip(search::docsummary::RES_FLOAT); docsum_blob_entry_filter.add_skip(search::docsummary::RES_DOUBLE); docsum_blob_entry_filter.add_skip(search::docsummary::RES_INT64); docsum_blob_entry_filter.add_skip(search::docsummary::RES_TENSOR); + docsum_blob_entry_filter.add_skip(search::docsummary::RES_FEATUREDATA); std::unique_ptr<ResultConfig> resCfg(new ResultConfig(docsum_blob_entry_filter)); if ( ! resCfg->ReadConfig(*summary.get(), _configId.c_str())) { throw std::runtime_error("(re-)configuration of VSM (docsum tools) failed due to bad summary config"); diff --git a/zookeeper-server/CMakeLists.txt b/zookeeper-server/CMakeLists.txt index 42ecff1cc96..b5b1641c54e 100644 --- a/zookeeper-server/CMakeLists.txt +++ b/zookeeper-server/CMakeLists.txt @@ -1,4 +1,4 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. add_subdirectory(zookeeper-server-common) add_subdirectory(zookeeper-server) -add_subdirectory(zookeeper-server-3.7.1) +add_subdirectory(zookeeper-server-3.8.0) diff --git a/zookeeper-server/README b/zookeeper-server/README index 2f4845dad29..e8218ca2149 100644 --- a/zookeeper-server/README +++ b/zookeeper-server/README @@ -2,7 +2,7 @@ Vespa ZooKeeper server interface Subdirectory zookeeper-server has code for the zookeeper server version we use by default, testing out new versions can be done by -copying zookeeper-server dir to zookeeper-server-<version> and +copying zookeeper-server dir to zookeeper-server-<version>, updating CMakeLists.txt and pom.xml in that directory and pom.xml and CMakeLists.txt in this directory. Selecting zookeeper server version -is done with feature flag ZOOKEEPER_SERVER_VERSION. +that is used runtime is done with feature flag ZOOKEEPER_SERVER_VERSION. diff --git a/zookeeper-server/pom.xml b/zookeeper-server/pom.xml index b9cc4478bc2..cf19f15e71f 100644 --- a/zookeeper-server/pom.xml +++ b/zookeeper-server/pom.xml @@ -14,7 +14,7 @@ <modules> <module>zookeeper-server-common</module> <module>zookeeper-server</module> - <module>zookeeper-server-3.7.1</module> + <module>zookeeper-server-3.8.0</module> </modules> <dependencies> <dependency> diff --git a/zookeeper-server/zookeeper-server-3.7.1/CMakeLists.txt b/zookeeper-server/zookeeper-server-3.8.0/CMakeLists.txt index 6afd812ef07..e0fcc05c90a 100644 --- a/zookeeper-server/zookeeper-server-3.7.1/CMakeLists.txt +++ b/zookeeper-server/zookeeper-server-3.8.0/CMakeLists.txt @@ -1,2 +1,2 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_jar(zookeeper-server-3.7.1-jar-with-dependencies.jar) +install_jar(zookeeper-server-3.8.0-jar-with-dependencies.jar) diff --git a/zookeeper-server/zookeeper-server-3.7.1/pom.xml b/zookeeper-server/zookeeper-server-3.8.0/pom.xml index ea0e61075f1..f6c8952849c 100644 --- a/zookeeper-server/zookeeper-server-3.7.1/pom.xml +++ b/zookeeper-server/zookeeper-server-3.8.0/pom.xml @@ -8,11 +8,11 @@ <version>8-SNAPSHOT</version> <relativePath>../pom.xml</relativePath> </parent> - <artifactId>zookeeper-server-3.7.1</artifactId> + <artifactId>zookeeper-server-3.8.0</artifactId> <packaging>container-plugin</packaging> <version>8-SNAPSHOT</version> <properties> - <zookeeper.version>3.7.1</zookeeper.version> + <zookeeper.version>3.8.0</zookeeper.version> </properties> <dependencies> <dependency> diff --git a/zookeeper-server/zookeeper-server-3.7.1/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java b/zookeeper-server/zookeeper-server-3.8.0/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java index e94110af2fb..e94110af2fb 100644 --- a/zookeeper-server/zookeeper-server-3.7.1/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java +++ b/zookeeper-server/zookeeper-server-3.8.0/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java diff --git a/zookeeper-server/zookeeper-server-3.7.1/src/main/java/com/yahoo/vespa/zookeeper/VespaMtlsAuthenticationProvider.java b/zookeeper-server/zookeeper-server-3.8.0/src/main/java/com/yahoo/vespa/zookeeper/VespaMtlsAuthenticationProvider.java index 66742b0e05b..66742b0e05b 100644 --- a/zookeeper-server/zookeeper-server-3.7.1/src/main/java/com/yahoo/vespa/zookeeper/VespaMtlsAuthenticationProvider.java +++ b/zookeeper-server/zookeeper-server-3.8.0/src/main/java/com/yahoo/vespa/zookeeper/VespaMtlsAuthenticationProvider.java diff --git a/zookeeper-server/zookeeper-server-3.7.1/src/main/java/com/yahoo/vespa/zookeeper/VespaQuorumPeer.java b/zookeeper-server/zookeeper-server-3.8.0/src/main/java/com/yahoo/vespa/zookeeper/VespaQuorumPeer.java index 47ec03367c1..47ec03367c1 100644 --- a/zookeeper-server/zookeeper-server-3.7.1/src/main/java/com/yahoo/vespa/zookeeper/VespaQuorumPeer.java +++ b/zookeeper-server/zookeeper-server-3.8.0/src/main/java/com/yahoo/vespa/zookeeper/VespaQuorumPeer.java diff --git a/zookeeper-server/zookeeper-server-3.7.1/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperAdminImpl.java b/zookeeper-server/zookeeper-server-3.8.0/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperAdminImpl.java index ae7bf8d84f5..ae7bf8d84f5 100644 --- a/zookeeper-server/zookeeper-server-3.7.1/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperAdminImpl.java +++ b/zookeeper-server/zookeeper-server-3.8.0/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperAdminImpl.java diff --git a/zookeeper-server/zookeeper-server-3.7.1/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java b/zookeeper-server/zookeeper-server-3.8.0/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java index 48f95d28910..48f95d28910 100644 --- a/zookeeper-server/zookeeper-server-3.7.1/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java +++ b/zookeeper-server/zookeeper-server-3.8.0/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java diff --git a/zookeeper-server/zookeeper-server-3.7.1/src/main/java/org/apache/zookeeper/common/NetUtils.java b/zookeeper-server/zookeeper-server-3.8.0/src/main/java/org/apache/zookeeper/common/NetUtils.java index 33ec9b1303a..33ec9b1303a 100644 --- a/zookeeper-server/zookeeper-server-3.7.1/src/main/java/org/apache/zookeeper/common/NetUtils.java +++ b/zookeeper-server/zookeeper-server-3.8.0/src/main/java/org/apache/zookeeper/common/NetUtils.java diff --git a/zookeeper-server/zookeeper-server-3.7.1/src/main/java/org/apache/zookeeper/server/VespaNettyServerCnxnFactory.java b/zookeeper-server/zookeeper-server-3.8.0/src/main/java/org/apache/zookeeper/server/VespaNettyServerCnxnFactory.java index fdfe0fe8467..fdfe0fe8467 100644 --- a/zookeeper-server/zookeeper-server-3.7.1/src/main/java/org/apache/zookeeper/server/VespaNettyServerCnxnFactory.java +++ b/zookeeper-server/zookeeper-server-3.8.0/src/main/java/org/apache/zookeeper/server/VespaNettyServerCnxnFactory.java diff --git a/zookeeper-server/zookeeper-server-3.7.1/src/test/java/com/yahoo/vespa/zookeper/VespaZooKeeperTest.java b/zookeeper-server/zookeeper-server-3.8.0/src/test/java/com/yahoo/vespa/zookeper/VespaZooKeeperTest.java index db643d76e0d..db643d76e0d 100644 --- a/zookeeper-server/zookeeper-server-3.7.1/src/test/java/com/yahoo/vespa/zookeper/VespaZooKeeperTest.java +++ b/zookeeper-server/zookeeper-server-3.8.0/src/test/java/com/yahoo/vespa/zookeper/VespaZooKeeperTest.java diff --git a/zookeeper-server/zookeeper-server/CMakeLists.txt b/zookeeper-server/zookeeper-server/CMakeLists.txt index 98cb495e4b6..c99916fafad 100644 --- a/zookeeper-server/zookeeper-server/CMakeLists.txt +++ b/zookeeper-server/zookeeper-server/CMakeLists.txt @@ -1,4 +1,4 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_jar(zookeeper-server-3.7.0-jar-with-dependencies.jar) +install_jar(zookeeper-server-3.7.1-jar-with-dependencies.jar) # Make symlink so that we have a default version, should be done only in zookeeper-server module -install_symlink(lib/jars/zookeeper-server-3.7.0-jar-with-dependencies.jar lib/jars/zookeeper-server-jar-with-dependencies.jar) +install_symlink(lib/jars/zookeeper-server-3.7.1-jar-with-dependencies.jar lib/jars/zookeeper-server-jar-with-dependencies.jar) diff --git a/zookeeper-server/zookeeper-server/pom.xml b/zookeeper-server/zookeeper-server/pom.xml index 72cb21b2362..ea0e61075f1 100644 --- a/zookeeper-server/zookeeper-server/pom.xml +++ b/zookeeper-server/zookeeper-server/pom.xml @@ -8,11 +8,11 @@ <version>8-SNAPSHOT</version> <relativePath>../pom.xml</relativePath> </parent> - <artifactId>zookeeper-server-3.7.0</artifactId> + <artifactId>zookeeper-server-3.7.1</artifactId> <packaging>container-plugin</packaging> <version>8-SNAPSHOT</version> <properties> - <zookeeper.version>3.7.0</zookeeper.version> + <zookeeper.version>3.7.1</zookeeper.version> </properties> <dependencies> <dependency> |