diff options
83 files changed, 797 insertions, 369 deletions
diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java index abd7ba3be9f..c88c1cdfc48 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java @@ -1,22 +1,22 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.model.test; +import com.yahoo.component.Version; +import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.application.api.ApplicationMetaData; +import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.ComponentInfo; import com.yahoo.config.application.api.UnparsedConfigDefinition; -import com.yahoo.config.application.api.ApplicationFile; -import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.TenantName; import com.yahoo.io.IOUtils; -import com.yahoo.path.Path; import com.yahoo.io.reader.NamedReader; +import com.yahoo.path.Path; import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.search.query.profile.config.QueryProfileXMLReader; import com.yahoo.vespa.config.ConfigDefinitionKey; -import com.yahoo.config.application.api.ApplicationPackage; import java.io.BufferedInputStream; import java.io.File; @@ -165,7 +165,7 @@ public class MockApplicationPackage implements ApplicationPackage { @Override public ApplicationFile getFile(Path file) { if (files.containsKey(file)) return files.get(file); - return new MockApplicationFile(file, Path.fromString(root.toString())); + return new MockApplicationFile(file, root); } @Override @@ -288,7 +288,7 @@ public class MockApplicationPackage implements ApplicationPackage { Map<Path, MockApplicationFile> mockFiles = new HashMap<>(); for (var file : files.entrySet()) mockFiles.put(file.getKey(), new MockApplicationFile(file.getKey(), - Path.fromString(root.toString()), file.getValue())); + root, file.getValue())); this.files = mockFiles; return this; } @@ -374,8 +374,8 @@ public class MockApplicationPackage implements ApplicationPackage { public static class MockApplicationFile extends ApplicationFile { - /** The path to the application package root */ - private final Path root; + /** The application package root */ + private final File root; /** The File pointing to the actual file represented by this */ private final File file; @@ -383,14 +383,14 @@ public class MockApplicationPackage implements ApplicationPackage { /** The content of this file, or null to read it from the file system. */ private final String content; - public MockApplicationFile(Path filePath, Path applicationPackagePath) { - this(filePath, applicationPackagePath, null); + public MockApplicationFile(Path relativeFile, File root) { + this(relativeFile, root, null); } - private MockApplicationFile(Path filePath, Path applicationPackagePath, String content) { - super(filePath); - this.root = applicationPackagePath; - file = applicationPackagePath.append(filePath).toFile(); + private MockApplicationFile(Path relativeFile, File root, String content) { + super(relativeFile); + this.root = root; + this.file = root.toPath().resolve(relativeFile.toString()).toFile(); this.content = content; } @@ -491,7 +491,7 @@ public class MockApplicationPackage implements ApplicationPackage { Iterator<String> pathIterator = path.iterator(); // Skip the path elements this shares with the root - for (Iterator<String> rootIterator = root.iterator(); rootIterator.hasNext(); ) { + for (Iterator<String> rootIterator = Path.fromString(root.toString()).iterator(); rootIterator.hasNext(); ) { String rootElement = rootIterator.next(); String pathElement = pathIterator.next(); if ( ! rootElement.equals(pathElement)) throw new RuntimeException("Assumption broken"); diff --git a/config-model/src/main/java/com/yahoo/schema/processing/RankingExpressionTypeResolver.java b/config-model/src/main/java/com/yahoo/schema/processing/RankingExpressionTypeResolver.java index 07f79f16334..d985089b2cb 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/RankingExpressionTypeResolver.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/RankingExpressionTypeResolver.java @@ -93,9 +93,9 @@ public class RankingExpressionTypeResolver extends Processor { throw new IllegalArgumentException(profile + " is strict but is missing a query profile type " + "declaration of features " + context.queryFeaturesNotDeclared()); else - deployLogger.logApplicationPackage(Level.WARNING, "The following query features used in " + profile + - " are not declared in query profile " + - "types and will be interpreted as scalars, not tensors: " + + deployLogger.logApplicationPackage(Level.WARNING, "The following query features used in " + + profile + " are not declared " + + "and will be interpreted as scalars, not tensors: " + context.queryFeaturesNotDeclared()); warnedAbout.addAll(context.queryFeaturesNotDeclared()); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java index 75b13a89e83..c50b9b7f842 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java @@ -36,7 +36,7 @@ public class LogserverContainerCluster extends ContainerCluster<LogserverContain protected boolean messageBusEnabled() { return false; } private void addLogHandler() { - Handler<?> logHandler = Handler.fromClassName(ContainerCluster.LOG_HANDLER_CLASS); + Handler logHandler = Handler.fromClassName(ContainerCluster.LOG_HANDLER_CLASS); logHandler.addServerBindings(SystemBindingPattern.fromHttpPath("/logs")); addComponent(logHandler); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java index 1da5e190c70..5d60cec0679 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java @@ -118,7 +118,7 @@ public class ClusterControllerContainer extends Container implements ZOOKEEPER_SERVER_BUNDLE); } - private void addHandler(Handler<?> h, String path) { + private void addHandler(Handler h, String path) { h.addServerBindings(SystemBindingPattern.fromHttpPath(path)); super.addHandler(h); } @@ -138,7 +138,7 @@ public class ClusterControllerContainer extends Container implements } private void addHandler(String id, String className, String path, ComponentSpecification bundle) { - addHandler(new Handler<>(createComponentModel(id, className, bundle)), path); + addHandler(new Handler(createComponentModel(id, className, bundle)), path); } private ReindexingContext reindexingContext() { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java index a29647b062a..680a4b97f86 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java @@ -119,12 +119,12 @@ public class MetricsProxyContainerCluster extends ContainerCluster<MetricsProxyC } private void addHttpHandler(Class<? extends ThreadedHttpRequestHandler> clazz, String bindingPath) { - Handler<AbstractConfigProducer<?>> metricsHandler = createMetricsHandler(clazz, bindingPath); + Handler metricsHandler = createMetricsHandler(clazz, bindingPath); addComponent(metricsHandler); } - static Handler<AbstractConfigProducer<?>> createMetricsHandler(Class<? extends ThreadedHttpRequestHandler> clazz, String bindingPath) { - Handler<AbstractConfigProducer<?>> metricsHandler = new Handler<>( + static Handler createMetricsHandler(Class<? extends ThreadedHttpRequestHandler> clazz, String bindingPath) { + Handler metricsHandler = new Handler( new ComponentModel(clazz.getName(), null, METRICS_PROXY_BUNDLE_NAME, null)); metricsHandler.addServerBindings( SystemBindingPattern.fromHttpPath(bindingPath), diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/UriBindingsValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/UriBindingsValidator.java index 4f322578b1c..718f1646126 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/UriBindingsValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/UriBindingsValidator.java @@ -24,7 +24,7 @@ class UriBindingsValidator extends Validator { @Override public void validate(VespaModel model, DeployState deployState) { for (ApplicationContainerCluster cluster : model.getContainerClusters().values()) { - for (Handler<?> handler : cluster.getHandlers()) { + for (Handler handler : cluster.getHandlers()) { for (BindingPattern binding : handler.getServerBindings()) { validateUserBinding(binding, model, deployState); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomClientProviderBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomClientProviderBuilder.java index 69613944e74..170e8940787 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomClientProviderBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomClientProviderBuilder.java @@ -5,7 +5,6 @@ import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.text.XML; import com.yahoo.vespa.model.container.ApplicationContainerCluster; -import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.container.component.Handler; import com.yahoo.vespa.model.container.component.UserBindingPattern; import org.w3c.dom.Element; @@ -22,7 +21,7 @@ public class DomClientProviderBuilder extends DomHandlerBuilder { @Override protected Handler doBuild(DeployState deployState, AbstractConfigProducer parent, Element clientElement) { - Handler<? super Component<?, ?>> client = createHandler(clientElement); + Handler client = createHandler(clientElement); for (Element binding : XML.getChildren(clientElement, "binding")) client.addClientBindings(UserBindingPattern.fromPattern(XML.getValue(binding))); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java index 025a6377b04..7bfe971981e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java @@ -9,15 +9,12 @@ import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.text.XML; import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.component.BindingPattern; -import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.container.component.Handler; import com.yahoo.vespa.model.container.component.UserBindingPattern; import com.yahoo.vespa.model.container.xml.BundleInstantiationSpecificationBuilder; import org.w3c.dom.Element; -import java.util.List; import java.util.Set; -import java.util.logging.Level; import static com.yahoo.vespa.model.container.ApplicationContainerCluster.METRICS_V2_HANDLER_BINDING_1; import static com.yahoo.vespa.model.container.ApplicationContainerCluster.METRICS_V2_HANDLER_BINDING_2; @@ -29,7 +26,7 @@ import static java.util.logging.Level.INFO; /** * @author gjoranv */ -public class DomHandlerBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Handler<?>> { +public class DomHandlerBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Handler> { private static final Set<BindingPattern> reservedBindings = Set.of(METRICS_V2_HANDLER_BINDING_1, @@ -45,8 +42,8 @@ public class DomHandlerBuilder extends VespaDomBuilder.DomConfigProducerBuilder< } @Override - protected Handler<?> doBuild(DeployState deployState, AbstractConfigProducer<?> parent, Element handlerElement) { - Handler<? super Component<?, ?>> handler = createHandler(handlerElement); + protected Handler doBuild(DeployState deployState, AbstractConfigProducer<?> parent, Element handlerElement) { + Handler handler = createHandler(handlerElement); for (Element binding : XML.getChildren(handlerElement, "binding")) addServerBinding(handler, UserBindingPattern.fromPattern(XML.getValue(binding)), deployState.getDeployLogger()); @@ -56,18 +53,18 @@ public class DomHandlerBuilder extends VespaDomBuilder.DomConfigProducerBuilder< return handler; } - Handler<? super Component<?, ?>> createHandler(Element handlerElement) { + Handler createHandler(Element handlerElement) { BundleInstantiationSpecification bundleSpec = BundleInstantiationSpecificationBuilder.build(handlerElement); - return new Handler<>(new ComponentModel(bundleSpec)); + return new Handler(new ComponentModel(bundleSpec)); } - private void addServerBinding(Handler<? super Component<?, ?>> handler, BindingPattern binding, DeployLogger log) { + private void addServerBinding(Handler handler, BindingPattern binding, DeployLogger log) { throwIfBindingIsReserved(binding, handler); handler.addServerBindings(binding); removeExistingServerBinding(binding, handler, log); } - private void throwIfBindingIsReserved(BindingPattern binding, Handler<?> newHandler) { + private void throwIfBindingIsReserved(BindingPattern binding, Handler newHandler) { for (var reserved : reservedBindings) { if (binding.hasSamePattern(reserved)) { throw new IllegalArgumentException("Binding '" + binding.patternString() + "' is a reserved Vespa binding and " + @@ -76,7 +73,7 @@ public class DomHandlerBuilder extends VespaDomBuilder.DomConfigProducerBuilder< } } - private void removeExistingServerBinding(BindingPattern binding, Handler<?> newHandler, DeployLogger log) { + private void removeExistingServerBinding(BindingPattern binding, Handler newHandler, DeployLogger log) { for (var handler : cluster.getHandlers()) { for (BindingPattern serverBinding : handler.getServerBindings()) { if (serverBinding.hasSamePattern(binding)) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java b/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java index 33c125dcecf..f23f27c0d8e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java @@ -1,8 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.clients; -import com.yahoo.config.model.producer.AbstractConfigProducer; -import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.container.handler.threadpool.ContainerThreadpoolConfig; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.vespa.model.container.ContainerCluster; @@ -41,31 +39,31 @@ public class ContainerDocumentApi { private static void addFeedHandler(ContainerCluster<?> cluster, HandlerOptions handlerOptions) { String bindingSuffix = ContainerCluster.RESERVED_URI_PREFIX + "/feedapi"; - var handler = newVespaClientHandler("com.yahoo.vespa.http.server.FeedHandler", bindingSuffix, handlerOptions); - cluster.addComponent(handler); var executor = new Threadpool("feedapi-handler", handlerOptions.feedApiThreadpoolOptions); - handler.inject(executor); - handler.addComponent(executor); + var handler = newVespaClientHandler("com.yahoo.vespa.http.server.FeedHandler", + bindingSuffix, handlerOptions, executor); + cluster.addComponent(handler); } private static void addRestApiHandler(ContainerCluster<?> cluster, HandlerOptions handlerOptions) { - var handler = newVespaClientHandler("com.yahoo.document.restapi.resource.DocumentV1ApiHandler", DOCUMENT_V1_PREFIX + "/*", handlerOptions); + var handler = newVespaClientHandler("com.yahoo.document.restapi.resource.DocumentV1ApiHandler", + DOCUMENT_V1_PREFIX + "/*", handlerOptions, null); cluster.addComponent(handler); // We need to include a dummy implementation of the previous restapi handler (using the same class name). // The internal legacy test framework requires that the name of the old handler is listed in /ApplicationStatus. - var oldHandlerDummy = handlerComponentSpecification("com.yahoo.document.restapi.resource.RestApi"); + var oldHandlerDummy = createHandler("com.yahoo.document.restapi.resource.RestApi", null); cluster.addComponent(oldHandlerDummy); } public boolean ignoreUndefinedFields() { return ignoreUndefinedFields; } - private static Handler<AbstractConfigProducer<?>> newVespaClientHandler( - String componentId, - String bindingSuffix, - HandlerOptions handlerOptions) { - Handler<AbstractConfigProducer<?>> handler = handlerComponentSpecification(componentId); + private static Handler newVespaClientHandler(String componentId, + String bindingSuffix, + HandlerOptions handlerOptions, + Threadpool executor) { + Handler handler = createHandler(componentId, executor); if (handlerOptions.bindings.isEmpty()) { handler.addServerBindings( SystemBindingPattern.fromHttpPath(bindingSuffix), @@ -81,9 +79,9 @@ public class ContainerDocumentApi { return handler; } - private static Handler<AbstractConfigProducer<?>> handlerComponentSpecification(String className) { - return new Handler<>(new ComponentModel( - BundleInstantiationSpecification.getFromStrings(className, null, "vespaclient-container-plugin"), "")); + private static Handler createHandler(String className, Threadpool executor) { + return new Handler(new ComponentModel(className, null, "vespaclient-container-plugin"), + executor); } public static final class HandlerOptions { @@ -104,12 +102,10 @@ public class ContainerDocumentApi { } @Override - public void getConfig(ContainerThreadpoolConfig.Builder builder) { - super.getConfig(builder); - - // User options overrides below configuration - if (hasUserOptions()) return; - builder.maxThreads(-4).minThreads(-4).queueSize(500); + protected void setDefaultConfigValues(ContainerThreadpoolConfig.Builder builder) { + builder.maxThreads(-4) + .minThreads(-4) + .queueSize(500); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index 937d7cf58d3..e316f826ad6 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -147,7 +147,7 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat } private void addMetricsHandler(String handlerClass, BindingPattern rootBinding, BindingPattern innerBinding) { - Handler<AbstractConfigProducer<?>> handler = new Handler<>( + Handler handler = new Handler( new ComponentModel(handlerClass, null, null, null)); handler.addServerBindings(rootBinding, innerBinding); addComponent(handler); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java index 63c323bfdaf..3d6332a9773 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java @@ -76,7 +76,7 @@ public abstract class Container extends AbstractService implements private final boolean dumpHeapOnShutdownTimeout; private final double shutdownTimeoutS; - private final ComponentGroup<Handler<?>> handlers = new ComponentGroup<>(this, "handler"); + private final ComponentGroup<Handler> handlers = new ComponentGroup<>(this, "handler"); private final ComponentGroup<Component<?, ?>> components = new ComponentGroup<>(this, "components"); private final JettyHttpServer defaultHttpServer; @@ -113,7 +113,7 @@ public abstract class Container extends AbstractService implements /** True if this container is retired (slated for removal) */ public boolean isRetired() { return retired; } - public ComponentGroup<Handler<?>> getHandlers() { + public ComponentGroup<Handler> getHandlers() { return handlers; } @@ -129,7 +129,7 @@ public abstract class Container extends AbstractService implements addComponent(new SimpleComponent(new ComponentModel(idSpec, classSpec, bundleSpec))); } - public final void addHandler(Handler<?> h) { + public final void addHandler(Handler h) { handlers.addComponent(h); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index 2385b5b3812..e69872ca1e7 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -73,6 +73,8 @@ import java.util.TreeSet; import java.util.stream.Collectors; import java.util.stream.Stream; +import static com.yahoo.vespa.model.container.component.chain.ProcessingHandler.PROCESSING_HANDLER_CLASS; + /** * Parent class for all container cluster types. * @@ -129,7 +131,7 @@ public abstract class ContainerCluster<CONTAINER extends Container> public static final BindingPattern VIP_HANDLER_BINDING = SystemBindingPattern.fromHttpPath("/status.html"); public static final Set<Path> SEARCH_AND_DOCPROC_BUNDLES = Stream.of( - PlatformBundles.searchAndDocprocBundle, "container-search-gui", "docprocs", "linguistics-components") + PlatformBundles.SEARCH_AND_DOCPROC_BUNDLE, "container-search-gui", "docprocs", "linguistics-components") .map(PlatformBundles::absoluteBundlePath).collect(Collectors.toSet()); private final String name; @@ -142,6 +144,7 @@ public abstract class ContainerCluster<CONTAINER extends Container> private ContainerDocproc containerDocproc; private ContainerDocumentApi containerDocumentApi; private SecretStore secretStore; + private final ContainerThreadpool defaultHandlerThreadpool = new Handler.DefaultHandlerThreadpool(); private boolean rpcServerEnabled = true; private boolean httpServerEnabled = true; @@ -181,6 +184,7 @@ public abstract class ContainerCluster<CONTAINER extends Container> addCommonVespaBundles(); addSimpleComponent(AccessLog.class); addComponent(new DefaultThreadpoolProvider(this, defaultPoolNumThreads)); + addComponent(defaultHandlerThreadpool); addSimpleComponent(com.yahoo.concurrent.classlock.ClassLocking.class); addSimpleComponent("com.yahoo.container.jdisc.metric.MetricConsumerProviderProvider"); addSimpleComponent("com.yahoo.container.jdisc.metric.MetricProvider"); @@ -217,36 +221,45 @@ public abstract class ContainerCluster<CONTAINER extends Container> } public void addMetricStateHandler() { - Handler<AbstractConfigProducer<?>> stateHandler = new Handler<>( + Handler stateHandler = new Handler( new ComponentModel(STATE_HANDLER_CLASS, null, null, null)); stateHandler.addServerBindings(STATE_HANDLER_BINDING_1, STATE_HANDLER_BINDING_2); addComponent(stateHandler); } public void addDefaultRootHandler() { - Handler<AbstractConfigProducer<?>> handler = new Handler<>( - new ComponentModel(BundleInstantiationSpecification.getFromStrings( + Handler handler = new Handler( + new ComponentModel(BundleInstantiationSpecification.fromStrings( BINDINGS_OVERVIEW_HANDLER_CLASS, null, null), null)); // null bundle, as the handler is in container-disc handler.addServerBindings(ROOT_HANDLER_BINDING); addComponent(handler); } public void addApplicationStatusHandler() { - Handler<AbstractConfigProducer<?>> statusHandler = new Handler<>( - new ComponentModel(BundleInstantiationSpecification.getFromStrings( + Handler statusHandler = new Handler( + new ComponentModel(BundleInstantiationSpecification.fromStrings( APPLICATION_STATUS_HANDLER_CLASS, null, null), null)); // null bundle, as the handler is in container-disc statusHandler.addServerBindings(SystemBindingPattern.fromHttpPath("/ApplicationStatus")); addComponent(statusHandler); } public void addVipHandler() { - Handler<?> vipHandler = Handler.fromClassName(FileStatusHandlerComponent.CLASS); + Handler vipHandler = Handler.fromClassName(FileStatusHandlerComponent.CLASS); vipHandler.addServerBindings(VIP_HANDLER_BINDING); addComponent(vipHandler); } public final void addComponent(Component<?, ?> component) { componentGroup.addComponent(component); + if (component instanceof Handler handler) { + ensureHandlerHasThreadpool(handler); + } + } + + private void ensureHandlerHasThreadpool(Handler handler) { + if (! handler.hasCustomThreadPool) { + handler.inject(defaultHandlerThreadpool); + } } public final void addSimpleComponent(String idSpec, String classSpec, String bundleSpec) { @@ -305,11 +318,9 @@ public abstract class ContainerCluster<CONTAINER extends Container> this.processingChains = processingChains; - // Cannot use the class object for ProcessingHandler, because its superclass is not accessible ProcessingHandler<?> processingHandler = new ProcessingHandler<>( processingChains, - "com.yahoo.processing.handler.ProcessingHandler", - null); + BundleInstantiationSpecification.fromStrings(PROCESSING_HANDLER_CLASS, null, null)); for (BindingPattern binding: serverBindings) processingHandler.addServerBindings(binding); @@ -368,9 +379,8 @@ public abstract class ContainerCluster<CONTAINER extends Container> return containerDocproc.getChains(); } - @SuppressWarnings("unchecked") - public Collection<Handler<?>> getHandlers() { - return (Collection<Handler<?>>)(Collection)componentGroup.getComponents(Handler.class); + public Collection<Handler> getHandlers() { + return componentGroup.getComponents(Handler.class); } public void setSecretStore(SecretStore secretStore) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java index cb8e6ba85ff..088465f56b1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java @@ -63,8 +63,8 @@ public class ContainerModelEvaluation implements rankProfileList.getConfig(builder); } - public static Handler<?> getHandler() { - Handler<?> handler = new Handler<>(new ComponentModel(REST_HANDLER_NAME, null, EVALUATION_BUNDLE_NAME)); + public static Handler getHandler() { + Handler handler = new Handler(new ComponentModel(REST_HANDLER_NAME, null, EVALUATION_BUNDLE_NAME)); handler.addServerBindings( SystemBindingPattern.fromHttpPath(REST_BINDING_PATH), SystemBindingPattern.fromHttpPath(REST_BINDING_PATH + "/*")); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerThreadpool.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerThreadpool.java index 489e4cc135a..fb4e62f5cd1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerThreadpool.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerThreadpool.java @@ -17,14 +17,14 @@ import java.util.Optional; * * @author bjorncs */ -public class ContainerThreadpool extends SimpleComponent implements ContainerThreadpoolConfig.Producer { +public abstract class ContainerThreadpool extends SimpleComponent implements ContainerThreadpoolConfig.Producer { private final String name; private final UserOptions userOptions; public ContainerThreadpool(String name, UserOptions userOptions) { super(new ComponentModel( - BundleInstantiationSpecification.getFromStrings( + BundleInstantiationSpecification.fromStrings( "threadpool@" + name, ContainerThreadpoolImpl.class.getName(), null))); @@ -32,8 +32,13 @@ public class ContainerThreadpool extends SimpleComponent implements ContainerThr this.userOptions = userOptions; } + // Must be implemented by subclasses to set values that may be overridden by user options. + protected abstract void setDefaultConfigValues(ContainerThreadpoolConfig.Builder builder); + @Override public void getConfig(ContainerThreadpoolConfig.Builder builder) { + setDefaultConfigValues(builder); + builder.name(this.name); if (userOptions != null) { builder.maxThreads(userOptions.maxThreads); @@ -42,9 +47,6 @@ public class ContainerThreadpool extends SimpleComponent implements ContainerThr } } - protected Optional<UserOptions> userOptions() { return Optional.ofNullable(userOptions); } - protected boolean hasUserOptions() { return userOptions().isPresent(); } - public static class UserOptions { private final int maxThreads; private final int minThreads; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/DefaultThreadpoolProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/container/DefaultThreadpoolProvider.java index 0b37abaded9..0fdd36b8811 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/DefaultThreadpoolProvider.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/DefaultThreadpoolProvider.java @@ -19,7 +19,7 @@ class DefaultThreadpoolProvider extends SimpleComponent implements ThreadpoolCon DefaultThreadpoolProvider(ContainerCluster<?> cluster, int defaultWorkerThreads) { super(new ComponentModel( - BundleInstantiationSpecification.getFromStrings( + BundleInstantiationSpecification.fromStrings( "default-threadpool", ThreadPoolProvider.class.getName(), null))); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/IdentityProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/container/IdentityProvider.java index 7ce2425179b..5e8bb85c29d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/IdentityProvider.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/IdentityProvider.java @@ -33,7 +33,7 @@ public class IdentityProvider extends SimpleComponent implements IdentityConfig. URI ztsUrl, String athenzDnsSuffix, Zone zone) { - super(new ComponentModel(BundleInstantiationSpecification.getFromStrings(CLASS, CLASS, BUNDLE))); + super(new ComponentModel(BundleInstantiationSpecification.fromStrings(CLASS, CLASS, BUNDLE))); this.domain = domain; this.service = service; this.loadBalancerName = loadBalancerName; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/PlatformBundles.java b/config-model/src/main/java/com/yahoo/vespa/model/container/PlatformBundles.java index e5125fe7e1d..6a1e647e9be 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/PlatformBundles.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/PlatformBundles.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container; +import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.vespa.defaults.Defaults; import java.nio.file.Path; @@ -28,7 +29,7 @@ public class PlatformBundles { } public static final Path LIBRARY_PATH = Paths.get(Defaults.getDefaults().underVespaHome("lib/jars")); - public static final String searchAndDocprocBundle = "container-search-and-docproc"; + public static final String SEARCH_AND_DOCPROC_BUNDLE = BundleInstantiationSpecification.CONTAINER_SEARCH_AND_DOCPROC; public static Set<Path> commonVespaBundles() { var bundles = new LinkedHashSet<Path>(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/DiscBindingsConfigGenerator.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/DiscBindingsConfigGenerator.java index 76124b14209..e4a5c2cd440 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/DiscBindingsConfigGenerator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/DiscBindingsConfigGenerator.java @@ -14,16 +14,16 @@ import static java.util.stream.Collectors.toList; */ public class DiscBindingsConfigGenerator { - public static Map<String, Handlers.Builder> generate(Collection<? extends Handler<?>> handlers) { + public static Map<String, Handlers.Builder> generate(Collection<? extends Handler> handlers) { Map<String, Handlers.Builder> handlerBuilders = new LinkedHashMap<>(); - for (Handler<?> handler : handlers) { + for (Handler handler : handlers) { handlerBuilders.putAll(generate(handler)); } return handlerBuilders; } - public static <T extends Handler<?>> Map<String, Handlers.Builder> generate(T handler) { + public static <T extends Handler> Map<String, Handlers.Builder> generate(T handler) { if (handler.getServerBindings().isEmpty() && handler.getClientBindings().isEmpty()) return Collections.emptyMap(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java index 8ffdccae896..9f2bfe9251b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java @@ -1,8 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container.component; -import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.container.handler.threadpool.ContainerThreadpoolConfig; import com.yahoo.osgi.provider.model.ComponentModel; +import com.yahoo.vespa.model.container.ContainerThreadpool; import java.util.ArrayList; import java.util.Arrays; @@ -15,22 +16,35 @@ import java.util.Set; * Models a jdisc RequestHandler (including ClientProvider). * RequestHandlers always have at least one server binding, * while ClientProviders have at least one client binding. - * <p> - * Note that this is also used to model vespa handlers (which do not have any bindings) * * @author gjoranv */ -public class Handler<CHILD extends AbstractConfigProducer<?>> extends Component<CHILD, ComponentModel> { +public class Handler extends Component<Component<?, ?>, ComponentModel> { private final Set<BindingPattern> serverBindings = new LinkedHashSet<>(); private final List<BindingPattern> clientBindings = new ArrayList<>(); + public final boolean hasCustomThreadPool; + public Handler(ComponentModel model) { + this(model, null); + } + + public Handler(ComponentModel model, ContainerThreadpool threadpool) { super(model); + + // The default threadpool is always added to the cluster, so cannot be added here. + if (threadpool != null) { + hasCustomThreadPool = true; + addComponent(threadpool); + inject(threadpool); + } else { + hasCustomThreadPool = false; + } } - public static Handler<AbstractConfigProducer<?>> fromClassName(String className) { - return new Handler<>(new ComponentModel(className, null, null, null)); + public static Handler fromClassName(String className) { + return new Handler(new ComponentModel(className, null, null, null)); } public void addServerBindings(BindingPattern... bindings) { @@ -53,4 +67,24 @@ public class Handler<CHILD extends AbstractConfigProducer<?>> extends Component< return Collections.unmodifiableList(clientBindings); } + + /** + * The default threadpool for all handlers, except those that declare their own, e.g. SearchHandler. + */ + public static class DefaultHandlerThreadpool extends ContainerThreadpool { + + public DefaultHandlerThreadpool() { + super("default-handler-common", null); + } + + @Override + public void setDefaultConfigValues(ContainerThreadpoolConfig.Builder builder) { + builder.maxThreadExecutionTimeSeconds(190) + .keepAliveTime(5.0) + .maxThreads(-2) + .minThreads(-2) + .queueSize(-40); + } + } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/SimpleComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/SimpleComponent.java index 2c4dd9a8dda..23915afacdd 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/SimpleComponent.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/SimpleComponent.java @@ -17,7 +17,7 @@ public class SimpleComponent extends Component<AbstractConfigProducer<?>, Compon } public SimpleComponent(String className) { - this(new ComponentModel(BundleInstantiationSpecification.getFromStrings(className, null, null))); + this(new ComponentModel(BundleInstantiationSpecification.fromStrings(className, null, null))); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/ProcessingHandler.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/ProcessingHandler.java index 3f68a0a2709..897a1f22f30 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/ProcessingHandler.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/ProcessingHandler.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.model.container.component.chain; import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.container.core.ChainsConfig; import com.yahoo.osgi.provider.model.ComponentModel; -import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.model.container.ContainerThreadpool; import com.yahoo.vespa.model.container.component.Handler; @@ -14,21 +14,21 @@ import com.yahoo.vespa.model.container.component.Handler; * @author gjoranv */ public class ProcessingHandler<CHAINS extends Chains<?>> - extends Handler<AbstractConfigProducer<?>> + extends Handler implements ChainsConfig.Producer { - protected final CHAINS chains; + // Cannot use the class object for ProcessingHandler, because its superclass is not accessible + public static final String PROCESSING_HANDLER_CLASS = "com.yahoo.processing.handler.ProcessingHandler"; - public ProcessingHandler(CHAINS chains, String handlerClass) { - this(chains, BundleInstantiationSpecification.getInternalProcessingSpecificationFromStrings(handlerClass, null)); - } + protected final CHAINS chains; - public ProcessingHandler(CHAINS chains, String handlerClass, String bundle) { - this(chains, BundleInstantiationSpecification.getFromStrings(handlerClass, null, bundle)); + // Create a handler that uses the default threadpool for handlers + public ProcessingHandler(CHAINS chains, BundleInstantiationSpecification spec) { + this(chains, spec, null); } - private ProcessingHandler(CHAINS chains, BundleInstantiationSpecification spec) { - super(new ComponentModel(spec, null)); + public ProcessingHandler(CHAINS chains, BundleInstantiationSpecification spec, ContainerThreadpool threadpool) { + super(new ComponentModel(spec), threadpool); this.chains = chains; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/DocprocChains.java b/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/DocprocChains.java index 31ba195775b..109ab3e806e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/DocprocChains.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/DocprocChains.java @@ -9,6 +9,7 @@ import com.yahoo.docproc.jdisc.observability.DocprocsStatusExtension; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.ContainerCluster; +import com.yahoo.vespa.model.container.PlatformBundles; import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.container.component.SimpleComponent; import com.yahoo.vespa.model.container.component.SystemBindingPattern; @@ -26,13 +27,12 @@ public class DocprocChains extends Chains<DocprocChain> { public DocprocChains(AbstractConfigProducer<?> parent, String subId) { super(parent, subId); - docprocHandler = new ProcessingHandler<>(this, "com.yahoo.docproc.jdisc.DocumentProcessingHandler"); + docprocHandler = new ProcessingHandler<>( + this, + BundleInstantiationSpecification.fromSearchAndDocproc("com.yahoo.docproc.jdisc.DocumentProcessingHandler")); addComponent(docprocHandler); - addComponent( - new SimpleComponent( - new ComponentModel( - BundleInstantiationSpecification.getInternalProcessingSpecificationFromStrings( - DocprocsStatusExtension.class.getName(), null), null))); + addComponent(new SimpleComponent( + new ComponentModel(DocprocsStatusExtension.class.getName(), null, PlatformBundles.SEARCH_AND_DOCPROC_BUNDLE))); if (! (getParent() instanceof ApplicationContainerCluster)) { // All application containers already have a DocumentTypeManager, diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/MbusClient.java b/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/MbusClient.java index 28a0748be26..0efcd8df37f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/MbusClient.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/MbusClient.java @@ -6,13 +6,12 @@ import com.yahoo.component.ComponentSpecification; import com.yahoo.container.jdisc.config.SessionConfig; import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.osgi.provider.model.ComponentModel; -import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.vespa.model.container.component.Handler; /** * @author Einar M R Rosenvinge */ -public class MbusClient extends Handler<AbstractConfigProducer<?>> implements SessionConfig.Producer { +public class MbusClient extends Handler implements SessionConfig.Producer { private static final ComponentSpecification CLASSNAME = ComponentSpecification.fromString("com.yahoo.container.jdisc.messagebus.MbusClientProvider"); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java index d85f00a5bb2..3ee0414bf32 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java @@ -51,7 +51,7 @@ public class AccessControl { private final String domain; private ClientAuthentication clientAuthentication = ClientAuthentication.need; private final Set<BindingPattern> excludeBindings = new LinkedHashSet<>(); - private Collection<Handler<?>> handlers = Collections.emptyList(); + private Collection<Handler> handlers = Collections.emptyList(); public Builder(String domain) { this.domain = domain; } @@ -79,12 +79,12 @@ public class AccessControl { public final String domain; public final ClientAuthentication clientAuthentication; private final Set<BindingPattern> excludedBindings; - private final Collection<Handler<?>> handlers; + private final Collection<Handler> handlers; private AccessControl(String domain, ClientAuthentication clientAuthentication, Set<BindingPattern> excludedBindings, - Collection<Handler<?>> handlers) { + Collection<Handler> handlers) { this.domain = domain; this.clientAuthentication = clientAuthentication; this.excludedBindings = Collections.unmodifiableSet(excludedBindings); @@ -119,7 +119,7 @@ public class AccessControl { public Set<BindingPattern> excludedBindings() { return excludedBindings; } /** all handlers (that are known by the access control components) **/ - public Collection<Handler<?>> handlers() { return handlers; } + public Collection<Handler> handlers() { return handlers; } public static boolean hasHandlerThatNeedsProtection(ApplicationContainerCluster cluster) { return cluster.getHandlers().stream() @@ -135,7 +135,7 @@ public class AccessControl { for (BindingPattern excludedBinding : excludedBindings) { http.getBindings().add(createAccessControlExcludedBinding(excludedBinding)); } - for (Handler<?> handler : handlers) { + for (Handler handler : handlers) { if (isExcludedHandler(handler)) { for (BindingPattern binding : handler.getServerBindings()) { http.getBindings().add(createAccessControlExcludedBinding(binding)); @@ -188,9 +188,9 @@ public class AccessControl { private static Chain<Filter> createChain(ComponentId id) { return new Chain<>(FilterChains.emptyChainSpec(id)); } - private static boolean isExcludedHandler(Handler<?> handler) { return EXCLUDED_HANDLERS.contains(handler.getClassId().getName()); } + private static boolean isExcludedHandler(Handler handler) { return EXCLUDED_HANDLERS.contains(handler.getClassId().getName()); } - private static boolean hasNonMbusBinding(Handler<?> handler) { + private static boolean hasNonMbusBinding(Handler handler) { return handler.getServerBindings().stream().anyMatch(binding -> ! binding.scheme().equals("mbus")); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java index 3ac12381a1f..7a0b6c8e023 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java @@ -25,7 +25,7 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; -import static com.yahoo.vespa.model.container.PlatformBundles.searchAndDocprocBundle; +import static com.yahoo.vespa.model.container.PlatformBundles.SEARCH_AND_DOCPROC_BUNDLE; /** * @author gjoranv @@ -56,9 +56,9 @@ public class ContainerSearch extends ContainerSubsystem<SearchChains> this.owningCluster = cluster; this.options = options; - owningCluster.addComponent(Component.fromClassAndBundle(QUERY_PROFILE_REGISTRY_CLASS, searchAndDocprocBundle)); - owningCluster.addComponent(Component.fromClassAndBundle(com.yahoo.search.schema.SchemaInfo.class.getName(), searchAndDocprocBundle)); - owningCluster.addComponent(Component.fromClassAndBundle(SearchStatusExtension.class.getName(), searchAndDocprocBundle)); + owningCluster.addComponent(Component.fromClassAndBundle(QUERY_PROFILE_REGISTRY_CLASS, SEARCH_AND_DOCPROC_BUNDLE)); + owningCluster.addComponent(Component.fromClassAndBundle(com.yahoo.search.schema.SchemaInfo.class.getName(), SEARCH_AND_DOCPROC_BUNDLE)); + owningCluster.addComponent(Component.fromClassAndBundle(SearchStatusExtension.class.getName(), SEARCH_AND_DOCPROC_BUNDLE)); cluster.addSearchAndDocprocBundles(); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/DispatcherComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/DispatcherComponent.java index 6edcd8901de..44c60cf0619 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/DispatcherComponent.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/DispatcherComponent.java @@ -32,7 +32,7 @@ public class DispatcherComponent extends Component<AbstractConfigProducer<?>, Co String dispatcherComponentId = "dispatcher." + indexedSearchCluster.getClusterName(); // used by ClusterSearcher return new ComponentModel(dispatcherComponentId, com.yahoo.search.dispatch.Dispatcher.class.getName(), - PlatformBundles.searchAndDocprocBundle); + PlatformBundles.SEARCH_AND_DOCPROC_BUNDLE); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/GUIHandler.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/GUIHandler.java index 7087cabafc1..b00faa80715 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/GUIHandler.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/GUIHandler.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container.search; -import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.vespa.model.container.component.Handler; @@ -10,7 +9,7 @@ import com.yahoo.vespa.model.container.component.Handler; /** * @author Henrik Høiness */ -public class GUIHandler extends Handler<AbstractConfigProducer<?>> { +public class GUIHandler extends Handler { public static final String BUNDLE = "container-search-gui"; public static final String CLASS = "com.yahoo.search.query.gui.GUIHandler"; @@ -21,7 +20,7 @@ public class GUIHandler extends Handler<AbstractConfigProducer<?>> { } public static BundleInstantiationSpecification bundleSpec(String className, String bundle) { - return BundleInstantiationSpecification.getFromStrings(className, className, bundle); + return BundleInstantiationSpecification.fromStrings(className, className, bundle); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/RpcResourcePoolComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/RpcResourcePoolComponent.java index 956d551f6b3..b4b2af3c808 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/RpcResourcePoolComponent.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/RpcResourcePoolComponent.java @@ -13,6 +13,6 @@ public class RpcResourcePoolComponent extends Component<RpcResourcePoolComponent private static ComponentModel toComponentModel(String clusterName) { String componentId = "rpcresourcepool." + clusterName; - return new ComponentModel(componentId, com.yahoo.search.dispatch.rpc.RpcResourcePool.class.getName(), PlatformBundles.searchAndDocprocBundle); + return new ComponentModel(componentId, com.yahoo.search.dispatch.rpc.RpcResourcePool.class.getName(), PlatformBundles.SEARCH_AND_DOCPROC_BUNDLE); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilder.java index fbdefd0afb0..cdea63cfb70 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilder.java @@ -10,6 +10,8 @@ import org.w3c.dom.Element; import java.util.Arrays; import java.util.List; +import static com.yahoo.vespa.model.container.component.chain.ProcessingHandler.PROCESSING_HANDLER_CLASS; + /** * This object builds a bundle instantiation spec from an XML element. * @@ -30,7 +32,7 @@ public class BundleInstantiationSpecificationBuilder { private static BundleInstantiationSpecification setBundleForSearchAndDocprocComponents(BundleInstantiationSpecification spec) { if (PlatformBundles.isSearchAndDocprocClass(spec.getClassName())) - return spec.inBundle(PlatformBundles.searchAndDocprocBundle); + return spec.inBundle(PlatformBundles.SEARCH_AND_DOCPROC_BUNDLE); else return spec; } @@ -39,7 +41,7 @@ public class BundleInstantiationSpecificationBuilder { private static void validate(BundleInstantiationSpecification instSpec) { List<String> forbiddenClasses = Arrays.asList( SearchHandler.HANDLER_CLASS, - "com.yahoo.processing.handler.ProcessingHandler"); + PROCESSING_HANDLER_CLASS); for (String forbiddenClass: forbiddenClasses) { if (forbiddenClass.equals(instSpec.getClassName())) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/CloudSecretStore.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/CloudSecretStore.java index 18916f1e09b..373eb9714d3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/CloudSecretStore.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/CloudSecretStore.java @@ -21,7 +21,7 @@ public class CloudSecretStore extends SimpleComponent implements SecretStoreConf private final List<StoreConfig> configList; public CloudSecretStore() { - super(new ComponentModel(BundleInstantiationSpecification.getFromStrings(CLASS, CLASS, BUNDLE))); + super(new ComponentModel(BundleInstantiationSpecification.fromStrings(CLASS, CLASS, BUNDLE))); configList = new ArrayList<>(); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 95b5d804e7b..a8ce0219434 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -30,6 +30,7 @@ import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.Zone; +import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.container.logging.FileConnectionLog; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.schema.OnnxModel; @@ -222,7 +223,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { } if (deployState.zone().system().isPublic()) { BindingPattern bindingPattern = SystemBindingPattern.fromHttpPath("/validate-secret-store"); - Handler<AbstractConfigProducer<?>> handler = new Handler<>( + Handler handler = new Handler( new ComponentModel("com.yahoo.jdisc.cloud.aws.AwsParameterStoreValidationHandler", null, "jdisc-cloud-aws", null)); handler.addServerBindings(bindingPattern); cluster.addComponent(handler); @@ -886,8 +887,9 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { private void addSearchHandler(ApplicationContainerCluster cluster, Element searchElement) { // Magic spell is needed to receive the chains config :-| - cluster.addComponent(new ProcessingHandler<>(cluster.getSearch().getChains(), - "com.yahoo.search.searchchain.ExecutionFactory")); + cluster.addComponent(new ProcessingHandler<>( + cluster.getSearch().getChains(), + BundleInstantiationSpecification.fromSearchAndDocproc("com.yahoo.search.searchchain.ExecutionFactory"))); cluster.addComponent( new SearchHandler( @@ -897,7 +899,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { } private void addGUIHandler(ApplicationContainerCluster cluster) { - Handler<?> guiHandler = new GUIHandler(); + Handler guiHandler = new GUIHandler(); guiHandler.addServerBindings(SystemBindingPattern.fromHttpPath(GUIHandler.BINDING_PATH)); cluster.addComponent(guiHandler); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/SearchHandler.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/SearchHandler.java index 54d943f498a..7e1b3be9240 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/SearchHandler.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/SearchHandler.java @@ -1,9 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container.xml; +import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.container.handler.threadpool.ContainerThreadpoolConfig; import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.ContainerThreadpool; +import com.yahoo.vespa.model.container.PlatformBundles; import com.yahoo.vespa.model.container.component.BindingPattern; import com.yahoo.vespa.model.container.component.SystemBindingPattern; import com.yahoo.vespa.model.container.component.chain.ProcessingHandler; @@ -11,6 +13,8 @@ import com.yahoo.vespa.model.container.search.searchchain.SearchChains; import java.util.List; +import static com.yahoo.container.bundle.BundleInstantiationSpecification.fromSearchAndDocproc; + /** * Component definition for {@link com.yahoo.search.handler.SearchHandler} * @@ -19,38 +23,32 @@ import java.util.List; class SearchHandler extends ProcessingHandler<SearchChains> { static final String HANDLER_CLASS = com.yahoo.search.handler.SearchHandler.class.getName(); + static final BundleInstantiationSpecification HANDLER_SPEC = fromSearchAndDocproc(HANDLER_CLASS); static final BindingPattern DEFAULT_BINDING = SystemBindingPattern.fromHttpPath("/search/*"); SearchHandler(ApplicationContainerCluster cluster, List<BindingPattern> bindings, ContainerThreadpool.UserOptions threadpoolOptions) { - super(cluster.getSearchChains(), HANDLER_CLASS); + super(cluster.getSearchChains(), HANDLER_SPEC, new Threadpool(threadpoolOptions)); bindings.forEach(this::addServerBindings); - Threadpool threadpool = new Threadpool(cluster, threadpoolOptions); - inject(threadpool); - addComponent(threadpool); } + private static class Threadpool extends ContainerThreadpool { - private final ApplicationContainerCluster cluster; - Threadpool(ApplicationContainerCluster cluster, UserOptions options) { + Threadpool(UserOptions options) { super("search-handler", options); - this.cluster = cluster; } @Override - public void getConfig(ContainerThreadpoolConfig.Builder builder) { - super.getConfig(builder); - - builder.maxThreadExecutionTimeSeconds(190); - builder.keepAliveTime(5.0); - - // User options overrides below configuration - if (hasUserOptions()) return; - builder.maxThreads(-2).minThreads(-2).queueSize(-40); + public void setDefaultConfigValues(ContainerThreadpoolConfig.Builder builder) { + builder.maxThreadExecutionTimeSeconds(190) + .keepAliveTime(5.0) + .maxThreads(-2) + .minThreads(-2) + .queueSize(-40); } - } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/document/DocumentFactoryBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/document/DocumentFactoryBuilder.java index 5df862647a0..7f3d7981bb6 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/document/DocumentFactoryBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/document/DocumentFactoryBuilder.java @@ -31,7 +31,7 @@ public class DocumentFactoryBuilder { String concDocFactory=pkg+"."+CONCRETE_DOC_FACTORY_CLASS; String bundle = e.getAttribute("bundle"); Component<AbstractConfigProducer<?>, ComponentModel> component = new Component<>( - new ComponentModel(BundleInstantiationSpecification.getFromStrings(concDocFactory, concDocFactory, bundle))); + new ComponentModel(BundleInstantiationSpecification.fromStrings(concDocFactory, concDocFactory, bundle))); if (!cluster.getComponentsMap().containsKey(component.getComponentId())) cluster.addComponent(component); types.put(type, concDocFactory); } diff --git a/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionTypeResolverTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionTypeResolverTestCase.java index 4b6a22fc81a..f2f2a82b97c 100644 --- a/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionTypeResolverTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionTypeResolverTestCase.java @@ -456,7 +456,7 @@ public class RankingExpressionTypeResolverTestCase { builder.build(true); String message = logger.findMessage("The following query features"); assertNotNull(message); - assertEquals("WARNING: The following query features used in rank profile 'my_rank_profile' are not declared in query profile types and " + + assertEquals("WARNING: The following query features used in rank profile 'my_rank_profile' are not declared and " + "will be interpreted as scalars, not tensors: [query(bar), query(baz), query(foo)]", message); } diff --git a/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithOnnxTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithOnnxTestCase.java index 94a51d25717..bfd0520c62a 100644 --- a/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithOnnxTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithOnnxTestCase.java @@ -391,7 +391,7 @@ public class RankingExpressionWithOnnxTestCase { @Override public ApplicationFile getFile(Path file) { - return new MockApplicationFile(file, Path.fromString(root().toString())); + return new MockApplicationFile(file, root()); } @Override diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java index 7bf08461df7..d50aedeafa0 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java @@ -70,7 +70,7 @@ public class MetricsProxyContainerClusterTest { @Test public void http_handlers_are_set_up() { VespaModel model = getModel(servicesWithAdminOnly(), self_hosted); - Collection<Handler<?>> handlers = model.getAdmin().getMetricsProxyCluster().getHandlers(); + Collection<Handler> handlers = model.getAdmin().getMetricsProxyCluster().getHandlers(); Collection<ComponentSpecification> handlerClasses = handlers.stream().map(Component::getClassId).collect(toList()); assertTrue(handlerClasses.contains(ComponentSpecification.fromString(MetricsV1Handler.class.getName()))); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java index 81b6de8f2ee..d9659cf92d4 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java @@ -89,7 +89,7 @@ public class AccessLogTest extends ContainerModelBuilderTestBase { assertNotNull(getVespaAccessLog("default")); { // vespa - Component<?, ?> accessLogComponent = getContainerComponent("default", VespaAccessLog.class.getName()); + Component<?, ?> accessLogComponent = getComponent("default", VespaAccessLog.class.getName()); assertNotNull(accessLogComponent); assertEquals(VespaAccessLog.class.getName(), accessLogComponent.getClassId().getName(), VespaAccessLog.class.getName()); AccessLogConfig config = root.getConfig(AccessLogConfig.class, "default/component/com.yahoo.container.logging.VespaAccessLog"); @@ -101,7 +101,7 @@ public class AccessLogTest extends ContainerModelBuilderTestBase { } { // json - Component<?, ?> accessLogComponent = getContainerComponent("default", JSONAccessLog.class.getName()); + Component<?, ?> accessLogComponent = getComponent("default", JSONAccessLog.class.getName()); assertNotNull(accessLogComponent); assertEquals(JSONAccessLog.class.getName(), accessLogComponent.getClassId().getName(), JSONAccessLog.class.getName()); AccessLogConfig config = root.getConfig(AccessLogConfig.class, "default/component/com.yahoo.container.logging.JSONAccessLog"); @@ -124,7 +124,7 @@ public class AccessLogTest extends ContainerModelBuilderTestBase { nodesXml, "</container>" ); createModel(root, clusterElem); - Component<?, ?> connectionLogComponent = getContainerComponent("default", FileConnectionLog.class.getName()); + Component<?, ?> connectionLogComponent = getComponent("default", FileConnectionLog.class.getName()); assertNotNull(connectionLogComponent); ConnectionLogConfig config = root.getConfig(ConnectionLogConfig.class, "default/component/com.yahoo.container.logging.FileConnectionLog"); assertEquals("default", config.cluster()); @@ -140,7 +140,7 @@ public class AccessLogTest extends ContainerModelBuilderTestBase { nodesXml, "</container>" ); createModel(root, clusterElem); - Component<?, ?> fileConnectionLogComponent = getContainerComponent("default", FileConnectionLog.class.getName()); + Component<?, ?> fileConnectionLogComponent = getComponent("default", FileConnectionLog.class.getName()); assertNull(fileConnectionLogComponent); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilderTest.java index 686f7bbd1f1..125f1823fec 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilderTest.java @@ -29,7 +29,7 @@ public class BundleInstantiationSpecificationBuilderTest { @Test public void bundle_is_replaced_for_internal_class() { String internalClass = GroupingValidator.class.getName(); - verifyExpectedBundle(internalClass, null, PlatformBundles.searchAndDocprocBundle); + verifyExpectedBundle(internalClass, null, PlatformBundles.SEARCH_AND_DOCPROC_BUNDLE); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerDocumentApiBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerDocumentApiBuilderTest.java index 61e0a9e15a4..4a9d44ea2f8 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerDocumentApiBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerDocumentApiBuilderTest.java @@ -30,11 +30,11 @@ import static org.junit.Assert.assertTrue; */ public class ContainerDocumentApiBuilderTest extends ContainerModelBuilderTestBase { - private Map<String, Handler<?>> getHandlers(String clusterName) { + private Map<String, Handler> getHandlers(String clusterName) { ContainerCluster<?> cluster = (ContainerCluster<?>) root.getChildren().get(clusterName); - Map<String, Handler<?>> handlerMap = new HashMap<>(); - Collection<Handler<?>> handlers = cluster.getHandlers(); - for (Handler<?> handler : handlers) { + Map<String, Handler> handlerMap = new HashMap<>(); + Collection<Handler> handlers = cluster.getHandlers(); + for (Handler handler : handlers) { assertFalse(handlerMap.containsKey(handler.getComponentId().toString())); //die on overwrites handlerMap.put(handler.getComponentId().toString(), handler); } @@ -56,7 +56,7 @@ public class ContainerDocumentApiBuilderTest extends ContainerModelBuilderTestBa } private void verifyCustomBindings(String id) { - Handler<?> handler = getHandlers("cluster1").get(id); + Handler handler = getHandlers("cluster1").get(id); assertTrue(handler.getServerBindings().contains(UserBindingPattern.fromHttpPath("/document-api/reserved-for-internal-use/feedapi"))); assertTrue(handler.getServerBindings().contains(UserBindingPattern.fromHttpPath("/document-api/reserved-for-internal-use/feedapi/"))); @@ -73,7 +73,7 @@ public class ContainerDocumentApiBuilderTest extends ContainerModelBuilderTestBa "</container>"); createModel(root, elem); - Map<String, Handler<?>> handlerMap = getHandlers("cluster1"); + Map<String, Handler> handlerMap = getHandlers("cluster1"); assertNotNull(handlerMap.get("com.yahoo.container.handler.VipStatusHandler")); assertNotNull(handlerMap.get("com.yahoo.container.handler.observability.ApplicationStatusHandler")); @@ -112,8 +112,8 @@ public class ContainerDocumentApiBuilderTest extends ContainerModelBuilderTestBa "</container>"); root = new MockRoot("root", new MockApplicationPackage.Builder().build()); createModel(root, elem); - Map<String, Handler<?>> handlers = getHandlers("cluster1"); - Handler<?> feedApiHandler = handlers.get("com.yahoo.vespa.http.server.FeedHandler"); + Map<String, Handler> handlers = getHandlers("cluster1"); + Handler feedApiHandler = handlers.get("com.yahoo.vespa.http.server.FeedHandler"); Set<String> injectedComponentIds = feedApiHandler.getInjectedComponentIds(); assertTrue(injectedComponentIds.contains("threadpool@feedapi-handler")); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java index 3a241d9607f..1d43c031a1b 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java @@ -42,6 +42,7 @@ import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.ContainerModelEvaluation; import com.yahoo.vespa.model.container.component.Component; +import com.yahoo.vespa.model.container.component.Handler; import com.yahoo.vespa.model.content.utils.ContentClusterUtils; import com.yahoo.vespa.model.test.VespaModelTester; import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithFilePkg; @@ -60,19 +61,15 @@ import java.util.stream.Collectors; import static com.yahoo.config.model.test.TestUtil.joinLines; import static com.yahoo.test.LinePatternMatcher.containsLineWithPattern; import static com.yahoo.vespa.defaults.Defaults.getDefaults; -import static com.yahoo.vespa.model.container.ContainerCluster.ROOT_HANDLER_BINDING; -import static com.yahoo.vespa.model.container.ContainerCluster.STATE_HANDLER_BINDING_1; -import static org.hamcrest.CoreMatchers.is; +import static com.yahoo.vespa.model.container.component.chain.ProcessingHandler.PROCESSING_HANDLER_CLASS; import static org.hamcrest.CoreMatchers.not; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.hasItem; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -198,6 +195,17 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { } @Test + public void builtin_handlers_get_default_threadpool() { + createBasicContainerModel(); + + Handler h1 = getHandler("default", ApplicationStatusHandler.class.getName()); + assertTrue(h1.getInjectedComponentIds().contains("threadpool@default-handler-common")); + + Handler h2 = getHandler("default", BindingsOverviewHandler.class.getName()); + assertTrue(h2.getInjectedComponentIds().contains("threadpool@default-handler-common")); + } + + @Test public void verify_bindings_for_builtin_handlers() { createBasicContainerModel(); JdiscBindingsConfig config = root.getConfig(JdiscBindingsConfig.class, "default/container.0"); @@ -216,68 +224,6 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { } @Test - public void default_root_handler_binding_can_be_stolen_by_user_configured_handler() { - Element clusterElem = DomBuilderTest.parse( - "<container id='default' version='1.0'>" + - " <handler id='userRootHandler'>" + - " <binding>" + ROOT_HANDLER_BINDING.patternString() + "</binding>" + - " </handler>" + - "</container>"); - createModel(root, clusterElem); - - // The handler is still set up. - ComponentsConfig.Components userRootHandler = getComponent(componentsConfig(), BindingsOverviewHandler.class.getName()); - assertNotNull(userRootHandler); - - // .. but it has no bindings - var discBindingsConfig = root.getConfig(JdiscBindingsConfig.class, "default"); - assertNull(discBindingsConfig.handlers(BindingsOverviewHandler.class.getName())); - } - - @Test - public void reserved_binding_cannot_be_stolen_by_user_configured_handler() { - Element clusterElem = DomBuilderTest.parse( - "<container id='default' version='1.0'>" + - " <handler id='userHandler'>" + - " <binding>" + STATE_HANDLER_BINDING_1.patternString() + "</binding>" + - " </handler>" + - "</container>"); - try { - createModel(root, clusterElem); - fail("Expected exception when stealing a reserved binding."); - } catch (IllegalArgumentException e) { - assertThat(e.getMessage(), is("Binding 'http://*/state/v1' is a reserved Vespa binding " + - "and cannot be used by handler: userHandler")); - } - } - - @Test - public void handler_bindings_are_included_in_discBindings_config() { - createClusterWithJDiscHandler(); - String discBindingsConfig = root.getConfig(JdiscBindingsConfig.class, "default").toString(); - assertThat(discBindingsConfig, containsString(".serverBindings[0] \"http://*/binding0\"")); - assertThat(discBindingsConfig, containsString(".serverBindings[1] \"http://*/binding1\"")); - } - - @Test - public void handlers_are_included_in_components_config() { - createClusterWithJDiscHandler(); - assertThat(componentsConfig().toString(), containsString(".id \"discHandler\"")); - } - - private void createClusterWithJDiscHandler() { - Element clusterElem = DomBuilderTest.parse( - "<container id='default' version='1.0'>", - " <handler id='discHandler'>", - " <binding>http://*/binding0</binding>", - " <binding>http://*/binding1</binding>", - " </handler>", - "</container>"); - - createModel(root, clusterElem); - } - - @Test public void processing_handler_bindings_can_be_overridden() { Element clusterElem = DomBuilderTest.parse( "<container id='default' version='1.0'>", @@ -324,12 +270,19 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { @Test public void processingHandler_gets_only_processing_chains_in_chains_config() { createClusterWithProcessingAndSearchChains(); - String processingHandlerConfigId = "default/component/com.yahoo.processing.handler.ProcessingHandler"; + String processingHandlerConfigId = "default/component/" + PROCESSING_HANDLER_CLASS; String chainsConfig = getChainsConfig(processingHandlerConfigId); assertThat(chainsConfig, containsLineWithPattern(".*\\.id \"testProcessor@default\"$")); assertThat(chainsConfig, not(containsLineWithPattern(".*\\.id \"testSearcher@default\"$"))); } + @Test + public void processingHandler_is_instantiated_from_the_default_bundle() { + createClusterWithProcessingAndSearchChains(); + ComponentsConfig.Components config = getComponentInConfig(componentsConfig(), PROCESSING_HANDLER_CLASS); + assertEquals(PROCESSING_HANDLER_CLASS, config.bundle()); + } + private void createClusterWithProcessingAndSearchChains() { Element clusterElem = DomBuilderTest.parse( "<container id='default' version='1.0'>" + @@ -375,20 +328,6 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { } @Test - public void nested_components_are_injected_to_handlers() { - Element clusterElem = DomBuilderTest.parse( - "<container id='default' version='1.0'>", - " <handler id='myHandler'>", - " <component id='injected' />", - " </handler>", - "</container>"); - - createModel(root, clusterElem); - Component<?,?> handler = getContainerComponent("default", "myHandler"); - assertThat(handler.getInjectedComponentIds(), hasItem("injected@myHandler")); - } - - @Test public void component_includes_are_added() { VespaModelCreatorWithFilePkg creator = new VespaModelCreatorWithFilePkg("src/test/cfg/application/include_dirs"); VespaModel model = creator.create(true); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTestBase.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTestBase.java index 3d7b17d37e0..dc831ef863d 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTestBase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTestBase.java @@ -13,6 +13,7 @@ import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.ContainerModel; import com.yahoo.vespa.model.container.component.Component; +import com.yahoo.vespa.model.container.component.Handler; import com.yahoo.vespa.model.container.search.ContainerSearch; import org.junit.Before; import org.w3c.dom.Element; @@ -94,7 +95,7 @@ public abstract class ContainerModelBuilderTestBase { return root.getConfig(ComponentsConfig.class, "default"); } - protected ComponentsConfig.Components getComponent(ComponentsConfig componentsConfig, String id) { + protected ComponentsConfig.Components getComponentInConfig(ComponentsConfig componentsConfig, String id) { for (ComponentsConfig.Components component : componentsConfig.components()) { if (component.id().equals(id)) return component; @@ -106,11 +107,18 @@ public abstract class ContainerModelBuilderTestBase { return (ApplicationContainerCluster) root.getChildren().get(clusterId); } - public Component<?, ?> getContainerComponent(String clusterId, String componentId) { + public Component<?, ?> getComponent(String clusterId, String componentId) { return getContainerCluster(clusterId).getComponentsMap().get( ComponentId.fromString(componentId)); } + public Handler getHandler(String clusterId, String componentId) { + Component<?,?> component = getComponent(clusterId, componentId); + if (! (component instanceof Handler)) + throw new RuntimeException("Component is not a handler: " + componentId); + return (Handler) component; + } + void assertComponentConfigured(ApplicationContainerCluster cluster, String componentId) { Component<?, ?> component = cluster.getComponentsMap().get(ComponentId.fromString(componentId)); assertNotNull(component); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/HandlerBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/HandlerBuilderTest.java new file mode 100644 index 00000000000..42cda0d8034 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/HandlerBuilderTest.java @@ -0,0 +1,118 @@ +package com.yahoo.vespa.model.container.xml; + +import com.yahoo.config.model.builder.xml.test.DomBuilderTest; +import com.yahoo.container.ComponentsConfig; +import com.yahoo.container.jdisc.JdiscBindingsConfig; +import com.yahoo.container.usability.BindingsOverviewHandler; +import com.yahoo.vespa.model.container.ApplicationContainerCluster; +import com.yahoo.vespa.model.container.component.Component; +import com.yahoo.vespa.model.container.component.Handler; +import org.junit.Test; +import org.w3c.dom.Element; + +import static com.yahoo.vespa.model.container.ContainerCluster.ROOT_HANDLER_BINDING; +import static com.yahoo.vespa.model.container.ContainerCluster.STATE_HANDLER_BINDING_1; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.hasItem; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * Tests for container model building with custom handlers. + * + * @author gjoranv + */ +public class HandlerBuilderTest extends ContainerModelBuilderTestBase { + + @Test + public void handlers_are_included_in_components_config() { + createClusterWithJDiscHandler(); + assertThat(componentsConfig().toString(), containsString(".id \"discHandler\"")); + } + + @Test + public void handler_bindings_are_included_in_discBindings_config() { + createClusterWithJDiscHandler(); + String discBindingsConfig = root.getConfig(JdiscBindingsConfig.class, "default").toString(); + assertThat(discBindingsConfig, containsString(".serverBindings[0] \"http://*/binding0\"")); + assertThat(discBindingsConfig, containsString(".serverBindings[1] \"http://*/binding1\"")); + } + + @Test + public void nested_components_are_injected_to_handlers() { + Element clusterElem = DomBuilderTest.parse( + "<container id='default' version='1.0'>", + " <handler id='myHandler'>", + " <component id='injected' />", + " </handler>", + "</container>"); + + createModel(root, clusterElem); + Component<?,?> handler = getComponent("default", "myHandler"); + assertThat(handler.getInjectedComponentIds(), hasItem("injected@myHandler")); + } + + @Test + public void default_root_handler_binding_can_be_stolen_by_user_configured_handler() { + Element clusterElem = DomBuilderTest.parse( + "<container id='default' version='1.0'>" + + " <handler id='userRootHandler'>" + + " <binding>" + ROOT_HANDLER_BINDING.patternString() + "</binding>" + + " </handler>" + + "</container>"); + createModel(root, clusterElem); + + // The handler is still set up. + ComponentsConfig.Components userRootHandler = getComponentInConfig(componentsConfig(), BindingsOverviewHandler.class.getName()); + assertNotNull(userRootHandler); + + // .. but it has no bindings + var discBindingsConfig = root.getConfig(JdiscBindingsConfig.class, "default"); + assertNull(discBindingsConfig.handlers(BindingsOverviewHandler.class.getName())); + } + + @Test + public void reserved_binding_cannot_be_stolen_by_user_configured_handler() { + Element clusterElem = DomBuilderTest.parse( + "<container id='default' version='1.0'>" + + " <handler id='userHandler'>" + + " <binding>" + STATE_HANDLER_BINDING_1.patternString() + "</binding>" + + " </handler>" + + "</container>"); + try { + createModel(root, clusterElem); + fail("Expected exception when stealing a reserved binding."); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), is("Binding 'http://*/state/v1' is a reserved Vespa binding " + + "and cannot be used by handler: userHandler")); + } + } + + @Test + public void custom_handler_gets_default_threadpool() { + createClusterWithJDiscHandler(); + ApplicationContainerCluster cluster = (ApplicationContainerCluster)root.getChildren().get("default"); + Handler handler = cluster.getHandlers().stream() + .filter(h -> h.getComponentId().toString().equals("discHandler")) + .findAny().orElseThrow(); + + assertTrue(handler.getInjectedComponentIds().contains("threadpool@default-handler-common")); + } + + private void createClusterWithJDiscHandler() { + Element clusterElem = DomBuilderTest.parse( + "<container id='default' version='1.0'>", + " <handler id='discHandler'>", + " <binding>http://*/binding0</binding>", + " <binding>http://*/binding1</binding>", + " </handler>", + "</container>"); + + createModel(root, clusterElem); + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ApplicationBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/SearchBuilderTest.java index ca59e053a89..c41373b9f85 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ApplicationBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/SearchBuilderTest.java @@ -31,7 +31,7 @@ import static org.junit.Assert.fail; /** * @author gjoranv */ -public class ApplicationBuilderTest extends ContainerModelBuilderTestBase { +public class SearchBuilderTest extends ContainerModelBuilderTestBase { private ChainsConfig chainsConfig() { return root.getConfig(ChainsConfig.class, "default/component/com.yahoo.search.handler.SearchHandler"); @@ -53,7 +53,7 @@ public class ApplicationBuilderTest extends ContainerModelBuilderTestBase { ApplicationContainerCluster cluster = (ApplicationContainerCluster)root.getChildren().get("default"); GUIHandler guiHandler = null; - for (Handler<?> handler : cluster.getHandlers()) { + for (Handler handler : cluster.getHandlers()) { if (handler instanceof GUIHandler) { guiHandler = (GUIHandler) handler; } @@ -230,12 +230,7 @@ public class ApplicationBuilderTest extends ContainerModelBuilderTestBase { createModel(root, clusterElem); - ApplicationContainerCluster cluster = (ApplicationContainerCluster)root.getChildren().get("default"); - Handler<?> searchHandler = cluster.getHandlers().stream() - .filter(h -> h.getComponentId().toString().equals(SearchHandler.HANDLER_CLASS)) - .findAny() - .get(); - + Handler searchHandler = getHandler("default", SearchHandler.HANDLER_CLASS); assertTrue(searchHandler.getInjectedComponentIds().contains("threadpool@search-handler")); ContainerThreadpoolConfig config = root.getConfig( diff --git a/config-model/src/test/schema-test-files/services.xml b/config-model/src/test/schema-test-files/services.xml index 4fb81963152..df87d2b9f3d 100644 --- a/config-model/src/test/schema-test-files/services.xml +++ b/config-model/src/test/schema-test-files/services.xml @@ -193,6 +193,9 @@ <handler id="jdisc-handler"> <binding>http://*:*/HelloWorld</binding> <binding>http://*:*/Status</binding> + <component id="injected-to-handler"> + <config name="foo"/> + </component> </handler> <server id="server-provider"> diff --git a/configdefinitions/src/vespa/configserver.def b/configdefinitions/src/vespa/configserver.def index b13c9ba1932..9d502b8bc35 100644 --- a/configdefinitions/src/vespa/configserver.def +++ b/configdefinitions/src/vespa/configserver.def @@ -47,7 +47,7 @@ zoneDnsSuffixes[] string # RPC protocol maxgetconfigclients int default=1000000 maxoutputbuffersize int default=65536 -useVespaVersionInRequest bool default=false +useVespaVersionInRequest bool default=true payloadCompressionType enum { UNCOMPRESSED, LZ4 } default=LZ4 # Athenz config diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 7949c2633e0..ffb6c038b70 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -96,6 +96,7 @@ import java.time.Clock; import java.time.Duration; import java.time.Instant; import java.util.Collection; +import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -631,15 +632,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye .stream() .filter(fileReference -> ! fileReferencesInUse.contains(fileReference)) .filter(fileReference -> isLastFileAccessBefore(new File(fileReferencesPath, fileReference), instant)) - .sorted((a, b) -> { - if (a.equals(b)) - return 0; - else if (lastAccessed(new File(fileReferencesPath, a)) - .isBefore(lastAccessed(new File(fileReferencesPath, b)))) - return -1; - else - return 1; - }) + .sorted(Comparator.comparing(a -> lastAccessed(new File(fileReferencesPath, a)))) .collect(Collectors.toList()); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java index c31015b533a..3cc1993d09e 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java @@ -6,13 +6,11 @@ import com.yahoo.collections.Pair; import com.yahoo.component.Version; import com.yahoo.config.provision.TenantName; import com.yahoo.container.di.config.ApplicationBundlesConfig; -import com.yahoo.net.HostName; -import com.yahoo.vespa.config.PayloadChecksum; -import com.yahoo.vespa.config.PayloadChecksum.Type; -import com.yahoo.vespa.config.PayloadChecksums; import com.yahoo.jrt.Request; +import com.yahoo.net.HostName; import com.yahoo.vespa.config.ConfigPayload; import com.yahoo.vespa.config.ErrorCode; +import com.yahoo.vespa.config.PayloadChecksums; import com.yahoo.vespa.config.UnknownConfigIdException; import com.yahoo.vespa.config.protocol.ConfigResponse; import com.yahoo.vespa.config.protocol.JRTServerConfigRequest; @@ -22,11 +20,13 @@ import com.yahoo.vespa.config.protocol.VespaVersion; import com.yahoo.vespa.config.server.GetConfigContext; import com.yahoo.vespa.config.server.UnknownConfigDefinitionException; import com.yahoo.vespa.config.server.tenant.TenantRepository; - import java.util.Optional; +import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; +import static com.yahoo.vespa.config.ErrorCode.APPLICATION_NOT_LOADED; +import static com.yahoo.vespa.config.ErrorCode.UNKNOWN_VESPA_VERSION; import static com.yahoo.vespa.config.protocol.SlimeConfigResponse.fromConfigPayload; /** @@ -56,7 +56,7 @@ class GetConfigProcessor implements Runnable { private void respond(JRTServerConfigRequest request) { Request req = request.getRequest(); if (req.isError()) { - Level logLevel = (req.errorCode() == ErrorCode.APPLICATION_NOT_LOADED) ? Level.FINE : Level.INFO; + Level logLevel = Set.of(APPLICATION_NOT_LOADED, UNKNOWN_VESPA_VERSION).contains(req.errorCode()) ? Level.FINE : Level.INFO; log.log(logLevel, () -> logPre + req.errorMessage()); } rpcServer.respond(request); @@ -98,9 +98,10 @@ class GetConfigProcessor implements Runnable { GetConfigContext context = rpcServer.createGetConfigContext(tenant, request, trace); if (context == null || ! context.requestHandler().hasApplication(context.applicationId(), Optional.empty())) { - handleError(request, ErrorCode.APPLICATION_NOT_LOADED, "No application exists"); + handleError(request, APPLICATION_NOT_LOADED, "No application exists"); return null; } + logPre = TenantRepository.logPre(context.applicationId()); Optional<Version> vespaVersion = rpcServer.useRequestVersion() ? request.getVespaVersion().map(VespaVersion::toString).map(Version::fromString) : @@ -114,7 +115,6 @@ class GetConfigProcessor implements Runnable { return null; } - this.logPre = TenantRepository.logPre(context.applicationId()); ConfigResponse config; try { config = rpcServer.resolveConfig(request, context, vespaVersion); @@ -177,7 +177,7 @@ class GetConfigProcessor implements Runnable { } private static String printableVespaVersion(Optional<Version> vespaVersion) { - return (vespaVersion.isPresent() ? vespaVersion.get().toFullString() : "LATEST"); + return vespaVersion.map(Version::toFullString).orElse("LATEST"); } private void returnEmpty(JRTServerConfigRequest request) { diff --git a/container-core/src/main/java/com/yahoo/container/bundle/BundleInstantiationSpecification.java b/container-core/src/main/java/com/yahoo/container/bundle/BundleInstantiationSpecification.java index d61df4310f0..926c67df0ba 100644 --- a/container-core/src/main/java/com/yahoo/container/bundle/BundleInstantiationSpecification.java +++ b/container-core/src/main/java/com/yahoo/container/bundle/BundleInstantiationSpecification.java @@ -14,6 +14,8 @@ import com.yahoo.component.ComponentSpecification; */ public final class BundleInstantiationSpecification { + public static final String CONTAINER_SEARCH_AND_DOCPROC = "container-search-and-docproc"; + public final ComponentId id; public final ComponentSpecification classId; public final ComponentSpecification bundle; @@ -31,34 +33,21 @@ public final class BundleInstantiationSpecification { assert (classId!= null); } - private static final String defaultInternalBundle = "container-search-and-docproc"; - - private static BundleInstantiationSpecification getInternalSpecificationFromString(String idSpec, String classSpec) { - return new BundleInstantiationSpecification( - new ComponentSpecification(idSpec), - (classSpec == null || classSpec.isEmpty())? null : new ComponentSpecification(classSpec), - new ComponentSpecification(defaultInternalBundle)); - } - - public static BundleInstantiationSpecification getInternalSearcherSpecification(ComponentSpecification idSpec, - ComponentSpecification classSpec) { - return new BundleInstantiationSpecification(idSpec, classSpec, new ComponentSpecification(defaultInternalBundle)); - } - - // TODO: These are the same for now because they are in the same bundle. - public static BundleInstantiationSpecification getInternalHandlerSpecificationFromStrings(String idSpec, String classSpec) { - return getInternalSpecificationFromString(idSpec, classSpec); - } - - public static BundleInstantiationSpecification getInternalProcessingSpecificationFromStrings(String idSpec, String classSpec) { - return getInternalSpecificationFromString(idSpec, classSpec); + /** + * Create spec for a component from the container-search-and-docproc bundle with the given class name as id. + */ + public static BundleInstantiationSpecification fromSearchAndDocproc(String className) { + return fromSearchAndDocproc(new ComponentSpecification(className), null); } - public static BundleInstantiationSpecification getInternalSearcherSpecificationFromStrings(String idSpec, String classSpec) { - return getInternalSpecificationFromString(idSpec, classSpec); + /** + * Create spec for a component from the container-search-and-docproc bundle with the given id and classId. + */ + public static BundleInstantiationSpecification fromSearchAndDocproc(ComponentSpecification id, ComponentSpecification classId) { + return new BundleInstantiationSpecification(id, classId, new ComponentSpecification(CONTAINER_SEARCH_AND_DOCPROC)); } - public static BundleInstantiationSpecification getFromStrings(String idSpec, String classSpec, String bundleSpec) { + public static BundleInstantiationSpecification fromStrings(String idSpec, String classSpec, String bundleSpec) { return new BundleInstantiationSpecification( new ComponentSpecification(idSpec), (classSpec == null || classSpec.isEmpty())? null : new ComponentSpecification(classSpec), diff --git a/container-core/src/main/java/com/yahoo/container/di/Container.java b/container-core/src/main/java/com/yahoo/container/di/Container.java index 68dda5fbc4c..1baf217da6b 100644 --- a/container-core/src/main/java/com/yahoo/container/di/Container.java +++ b/container-core/src/main/java/com/yahoo/container/di/Container.java @@ -299,7 +299,7 @@ public class Container { } private static BundleInstantiationSpecification bundleInstantiationSpecification(ComponentsConfig.Components config) { - return BundleInstantiationSpecification.getFromStrings(config.id(), config.classId(), config.bundle()); + return BundleInstantiationSpecification.fromStrings(config.id(), config.classId(), config.bundle()); } public static class ComponentGraphResult { diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java index 11355dee24f..fb7ba1ab39e 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java @@ -50,7 +50,7 @@ public abstract class ThreadedHttpRequestHandler extends ThreadedRequestHandler this(executor, metric, false); } - // TODO: move Inject annotation here! + // TODO: deprecate this and the Context class. The context component set up in the model does not get a dedicated thread pool. public ThreadedHttpRequestHandler(Context context) { this(context.executor, context.metric); } diff --git a/container-core/src/main/java/com/yahoo/osgi/provider/model/ComponentModel.java b/container-core/src/main/java/com/yahoo/osgi/provider/model/ComponentModel.java index 5bed6fbca9f..84d52bb01c3 100644 --- a/container-core/src/main/java/com/yahoo/osgi/provider/model/ComponentModel.java +++ b/container-core/src/main/java/com/yahoo/osgi/provider/model/ComponentModel.java @@ -26,7 +26,7 @@ public class ComponentModel { } public ComponentModel(String idSpec, String classSpec, String bundleSpec, String configId) { - this(BundleInstantiationSpecification.getFromStrings(idSpec, classSpec, bundleSpec), configId); + this(BundleInstantiationSpecification.fromStrings(idSpec, classSpec, bundleSpec), configId); } // For vespamodel @@ -36,7 +36,7 @@ public class ComponentModel { // For vespamodel public ComponentModel(String idSpec, String classSpec, String bundleSpec) { - this(BundleInstantiationSpecification.getFromStrings(idSpec, classSpec, bundleSpec)); + this(BundleInstantiationSpecification.fromStrings(idSpec, classSpec, bundleSpec)); } public ComponentId getComponentId() { diff --git a/container-core/src/test/java/com/yahoo/container/di/componentgraph/core/ComponentGraphTest.java b/container-core/src/test/java/com/yahoo/container/di/componentgraph/core/ComponentGraphTest.java index f8f567e1890..5abb96527bd 100644 --- a/container-core/src/test/java/com/yahoo/container/di/componentgraph/core/ComponentGraphTest.java +++ b/container-core/src/test/java/com/yahoo/container/di/componentgraph/core/ComponentGraphTest.java @@ -102,7 +102,7 @@ public class ComponentGraphTest { } @Test - public void component_can_be_injected_into_another_component() { + public void component_can_be_explicitly_injected_into_another_component() { Node injectedComponent = mockComponentNode(SimpleComponent.class); Node targetComponent = mockComponentNode(ComponentTakingComponent.class); targetComponent.inject(injectedComponent); @@ -117,6 +117,22 @@ public class ComponentGraphTest { ComponentTakingComponent instance = componentGraph.getInstance(ComponentTakingComponent.class); assertNotNull(instance); + assertSame(injectedComponent.instance.get(), instance.injectedComponent); + } + + @Test + public void explicitly_injected_components_may_be_unused() { + Node notUsingInjected = mockComponentNode(SimpleComponent.class); + Node injectedComponent = mockComponentNode(SimpleComponent2.class); + notUsingInjected.inject(injectedComponent); + + ComponentGraph componentGraph = new ComponentGraph(); + componentGraph.add(injectedComponent); + componentGraph.add(notUsingInjected); + componentGraph.complete(); + + SimpleComponent instanceNotUsingInjected = componentGraph.getInstance(SimpleComponent.class); + assertNotNull(instanceNotUsingInjected); } @Test @@ -514,7 +530,7 @@ public class ComponentGraphTest { } public static class ComponentTakingComponent extends AbstractComponent { - private final SimpleComponent injectedComponent; + final SimpleComponent injectedComponent; public ComponentTakingComponent(SimpleComponent injectedComponent) { assertNotNull(injectedComponent); diff --git a/container-core/src/test/java/com/yahoo/osgi/provider/model/ComponentModelTest.java b/container-core/src/test/java/com/yahoo/osgi/provider/model/ComponentModelTest.java index 4433b78c807..a1e533158b5 100644 --- a/container-core/src/test/java/com/yahoo/osgi/provider/model/ComponentModelTest.java +++ b/container-core/src/test/java/com/yahoo/osgi/provider/model/ComponentModelTest.java @@ -14,7 +14,7 @@ public class ComponentModelTest { @Test public void create_from_instantiation_spec() { ComponentModel model = new ComponentModel( - BundleInstantiationSpecification.getFromStrings("id", "class", "bundle")); + BundleInstantiationSpecification.fromStrings("id", "class", "bundle")); verifyBundleSpec(model); } @@ -26,7 +26,7 @@ public class ComponentModelTest { @Test public void create_from_instantiation_spec_and_config_id() throws Exception { ComponentModel model = new ComponentModel( - BundleInstantiationSpecification.getFromStrings("id", "class", "bundle"), "configId"); + BundleInstantiationSpecification.fromStrings("id", "class", "bundle"), "configId"); verifyBundleSpec(model); assertEquals("configId", model.configId); } diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/model/VespaSearchers.java b/container-search/src/main/java/com/yahoo/search/searchchain/model/VespaSearchers.java index 85b8a563a9e..110f69c19ca 100644 --- a/container-search/src/main/java/com/yahoo/search/searchchain/model/VespaSearchers.java +++ b/container-search/src/main/java/com/yahoo/search/searchchain/model/VespaSearchers.java @@ -78,7 +78,7 @@ public class VespaSearchers { for (Class c : searchers) { searcherModels.add( new ChainedComponentModel( - BundleInstantiationSpecification.getInternalSearcherSpecificationFromStrings(c.getName(), null), + BundleInstantiationSpecification.fromSearchAndDocproc(c.getName()), Dependencies.emptyDependencies())); } return searcherModels; diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/model/federation/FederationSearcherModel.java b/container-search/src/main/java/com/yahoo/search/searchchain/model/federation/FederationSearcherModel.java index ea8275760dc..8acac776ae8 100644 --- a/container-search/src/main/java/com/yahoo/search/searchchain/model/federation/FederationSearcherModel.java +++ b/container-search/src/main/java/com/yahoo/search/searchchain/model/federation/FederationSearcherModel.java @@ -24,9 +24,11 @@ public class FederationSearcherModel extends ChainedComponentModel { public final List<TargetSpec> targets; public final boolean inheritDefaultSources; - public FederationSearcherModel(ComponentSpecification componentId, Dependencies dependencies, - List<TargetSpec> targets, boolean inheritDefaultSources) { - super(BundleInstantiationSpecification.getInternalSearcherSpecification(componentId, federationSearcherComponentSpecification), + public FederationSearcherModel(ComponentSpecification componentId, + Dependencies dependencies, + List<TargetSpec> targets, + boolean inheritDefaultSources) { + super(BundleInstantiationSpecification.fromSearchAndDocproc(componentId, federationSearcherComponentSpecification), dependencies); this.inheritDefaultSources = inheritDefaultSources; this.targets = ImmutableList.copyOf(targets); diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/model/federation/LocalProviderSpec.java b/container-search/src/main/java/com/yahoo/search/searchchain/model/federation/LocalProviderSpec.java index 3c2767430c7..c4e5f26a3eb 100644 --- a/container-search/src/main/java/com/yahoo/search/searchchain/model/federation/LocalProviderSpec.java +++ b/container-search/src/main/java/com/yahoo/search/searchchain/model/federation/LocalProviderSpec.java @@ -63,9 +63,7 @@ public class LocalProviderSpec { for (Class<? extends Searcher> c : searchers) { searcherModels.add( new ChainedComponentModel( - BundleInstantiationSpecification.getInternalSearcherSpecificationFromStrings( - c.getName(), - null), + BundleInstantiationSpecification.fromSearchAndDocproc(c.getName()), Dependencies.emptyDependencies())); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepoStats.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepoStats.java index 68ebc5e86aa..5ddf5aaff28 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepoStats.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepoStats.java @@ -8,16 +8,23 @@ import java.util.List; */ public class NodeRepoStats { + private final double totalCost; + private final double totalAllocatedCost; private final Load load; private final Load activeLoad; private final List<ApplicationStats> applicationStats; - public NodeRepoStats(Load load, Load activeLoad, List<ApplicationStats> applicationStats) { + public NodeRepoStats(double totalCost, double totalAllocatedCost, + Load load, Load activeLoad, List<ApplicationStats> applicationStats) { + this.totalCost = totalCost; + this.totalAllocatedCost = totalAllocatedCost; this.load = load; this.activeLoad = activeLoad; this.applicationStats = List.copyOf(applicationStats); } + public double totalCost() { return totalCost; } + public double totalAllocatedCost() { return totalAllocatedCost; } public Load load() { return load; } public Load activeLoad() { return activeLoad; } public List<ApplicationStats> applicationStats() { return applicationStats; } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeRepoStatsData.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeRepoStatsData.java index 36677143f7a..75ac919f776 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeRepoStatsData.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeRepoStatsData.java @@ -16,6 +16,12 @@ import java.util.stream.Collectors; @JsonInclude(JsonInclude.Include.NON_NULL) public class NodeRepoStatsData { + @JsonProperty("totalCost") + public Double totalCost; + + @JsonProperty("totalAllocatedCost") + public Double totalAllocatedCost; + @JsonProperty("load") public LoadData load; @@ -26,7 +32,8 @@ public class NodeRepoStatsData { public List<ApplicationStatsData> applications; public NodeRepoStats toNodeRepoStats() { - return new NodeRepoStats(load.toLoad(), activeLoad.toLoad(), + return new NodeRepoStats(totalCost, totalAllocatedCost, + load.toLoad(), activeLoad.toLoad(), applications.stream().map(stats -> stats.toApplicationStats()).collect(Collectors.toList())); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/StatsResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/StatsResponse.java index 19f1ac5449f..96a3c9f177d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/StatsResponse.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/StatsResponse.java @@ -30,6 +30,8 @@ public class StatsResponse extends SlimeJsonResponse { if (stats.applicationStats().isEmpty()) continue; // skip empty zones Cursor zoneObject = zonesArray.addObject(); zoneObject.setString("id", zone.toString()); + zoneObject.setDouble("totalCost", stats.totalCost()); + zoneObject.setDouble("totalAllocatedCost", stats.totalAllocatedCost()); toSlime(stats.load(), zoneObject.setObject("load")); toSlime(stats.activeLoad(), zoneObject.setObject("activeLoad")); Cursor applicationsArray = zoneObject.setArray("applications"); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java index 3f1ca3f9706..93ebdcf3171 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java @@ -107,7 +107,7 @@ public class NodeRepositoryMock implements NodeRepository { .collect(Collectors.toList()) : List.of(); - return new NodeRepoStats(Load.zero(), Load.zero(), applicationStats); + return new NodeRepoStats(0.0, 0.0, Load.zero(), Load.zero(), applicationStats); } @Override diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/stats.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/stats.json index 673767c13a0..44b52e5be2c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/stats.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/stats.json @@ -2,6 +2,8 @@ "zones": [ { "id": "prod.us-east-3", + "totalCost": 0.0, + "totalAllocatedCost": 0.0, "load": { "cpu": 0.0, "memory": 0.0, @@ -27,6 +29,8 @@ }, { "id": "prod.us-west-1", + "totalCost": 0.0, + "totalAllocatedCost": 0.0, "load": { "cpu": 0.0, "memory": 0.0, diff --git a/dist/vespa.spec b/dist/vespa.spec index de1b5382040..b94a77491b8 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -282,13 +282,14 @@ Requires: vespa-xxhash = 0.8.0 Requires: xxhash Requires: xxhash-libs >= 0.8.0 %endif -Requires: zlib -Requires: perf Requires: gdb +Requires: hostname +Requires: perf Requires: nc Requires: nghttp2 Requires: net-tools Requires: unzip +Requires: zlib Requires: zstd %if 0%{?el7} Requires: llvm7.0 diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index bbe11e18d93..0c800a5029d 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -422,13 +422,6 @@ public class Flags { "Takes effect immediately", TENANT_ID); - public static final UnboundStringFlag PROVISION_IN_EXTERNAL_ACCOUNT = defineStringFlag( - "provision-in-external-account", "", - List.of("mpolden"), "2022-05-02", "2022-09-01", - "The 12-digit AWS account ID where resources belonging to an application should be provisioned", - "Takes effect immediately", - APPLICATION_ID); - public static final UnboundBooleanFlag ENABLE_PROXY_PROTOCOL_MIXED_MODE = defineFeatureFlag( "enable-proxy-protocol-mixed-mode", true, List.of("tokle"), "2022-05-09", "2022-09-01", diff --git a/http-utils/pom.xml b/http-utils/pom.xml index 52dac0cfeea..5b26235b62e 100644 --- a/http-utils/pom.xml +++ b/http-utils/pom.xml @@ -66,10 +66,6 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-compiler-plugin</artifactId> - <configuration> - <release>${vespaClients.jdk.releaseVersion}</release> - <showDeprecation>true</showDeprecation> - </configuration> </plugin> </plugins> </build> diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepoStats.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepoStats.java index 085b89d1253..1460ce70686 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepoStats.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepoStats.java @@ -26,16 +26,23 @@ import java.util.Set; */ public class NodeRepoStats { + private final double totalCost; + private final double totalAllocatedCost; private final Load load; private final Load activeLoad; private final List<ApplicationStats> applicationStats; - private NodeRepoStats(Load load, Load activeLoad, List<ApplicationStats> applicationStats) { + private NodeRepoStats(double totalCost, double totalAllocatedCost, Load load, Load activeLoad, List<ApplicationStats> applicationStats) { + this.totalCost = totalCost; + this.totalAllocatedCost = totalAllocatedCost; this.load = load; this.activeLoad = activeLoad; this.applicationStats = List.copyOf(applicationStats); } + public double totalCost() { return totalCost; } + public double totalAllocatedCost() { return totalAllocatedCost; } + /** * Returns the current average work-extracting utilization in this node repo over all nodes. * Capacity not allocated to active nodes are taken to have 0 utilization as it provides no useful work. @@ -50,11 +57,15 @@ public class NodeRepoStats { public static NodeRepoStats computeOver(NodeRepository nodeRepository) { NodeList allNodes = nodeRepository.nodes().list(); - List<NodeTimeseries> allNodeTimeseries = nodeRepository.metricsDb().getNodeTimeseries(Duration.ofHours(1), Set.of()); + double totalCost = allNodes.hosts().stream().mapToDouble(host -> host.resources().cost()).sum(); + double totalAllocatedCost = allNodes.not().hosts().stream() + .filter(node -> node.allocation().isPresent()) + .mapToDouble(node -> node.resources().cost()).sum(); + List<NodeTimeseries> allNodeTimeseries = nodeRepository.metricsDb().getNodeTimeseries(Duration.ofHours(1), Set.of()); Pair<Load, Load> load = computeLoad(allNodes, allNodeTimeseries); List<ApplicationStats> applicationStats = computeApplicationStats(allNodes, allNodeTimeseries); - return new NodeRepoStats(load.getFirst(), load.getSecond(), applicationStats); + return new NodeRepoStats(totalCost, totalAllocatedCost, load.getFirst(), load.getSecond(), applicationStats); } private static Pair<Load, Load> computeLoad(NodeList allNodes, List<NodeTimeseries> allNodeTimeseries) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java index 368a8da0f90..a9e7ded66e6 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java @@ -186,24 +186,38 @@ public class DynamicProvisioningMaintainer extends NodeRepositoryMaintainer { .collect(Collectors.toList()); } - private List<Node> candidatesForRemoval(List<Node> nodes) { - Map<String, Node> hostsByHostname = new HashMap<>(nodes.stream() - .filter(node -> switch (node.type()) { - case host -> - // TODO: Mark empty tenant hosts as wanttoretire & wanttodeprovision elsewhere, then handle as confighost here - node.state() != Node.State.parked || node.status().wantToDeprovision(); - case confighost, controllerhost -> node.state() == Node.State.parked && node.status().wantToDeprovision(); - default -> false; - }) - .collect(Collectors.toMap(Node::hostname, Function.identity()))); + private static List<Node> candidatesForRemoval(List<Node> nodes) { + Map<String, Node> removableHostsByHostname = new HashMap<>(); + for (var node : nodes) { + if (canRemoveHost(node)) { + removableHostsByHostname.put(node.hostname(), node); + } + } + for (var node : nodes) { + if (node.parentHostname().isPresent() && !canRemoveNode(node)) { + removableHostsByHostname.remove(node.parentHostname().get()); + } + } + return List.copyOf(removableHostsByHostname.values()); + } - nodes.stream() - .filter(node -> node.allocation().isPresent()) - .flatMap(node -> node.parentHostname().stream()) - .distinct() - .forEach(hostsByHostname::remove); + private static boolean canRemoveHost(Node host) { + return switch (host.type()) { + // TODO: Mark empty tenant hosts as wanttoretire & wanttodeprovision elsewhere, then handle as confighost here + case host -> host.state() != Node.State.parked || host.status().wantToDeprovision(); + case confighost, controllerhost -> canDeprovision(host); + default -> false; + }; + } + + private static boolean canRemoveNode(Node node) { + if (node.type().isHost()) throw new IllegalArgumentException("Node " + node + " is not a child"); + return node.allocation().isEmpty() || canDeprovision(node); + } - return List.copyOf(hostsByHostname.values()); + private static boolean canDeprovision(Node node) { + return node.status().wantToDeprovision() && (node.state() == Node.State.parked || + node.state() == Node.State.failed); } private Map<String, Node> findSharedHosts(NodeList nodeList) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java index 829823913a9..aa1abb18d8c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java @@ -131,7 +131,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { infrastructureProvisionInterval = Duration.ofMinutes(3); loadBalancerExpirerInterval = Duration.ofMinutes(5); metricsInterval = Duration.ofMinutes(1); - nodeFailerInterval = Duration.ofMinutes(15); + nodeFailerInterval = Duration.ofMinutes(9); nodeFailureStatusUpdateInterval = Duration.ofMinutes(2); nodeMetricsCollectionInterval = Duration.ofMinutes(1); expeditedChangeRedeployInterval = Duration.ofMinutes(3); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java index c5d8b2518e5..dcfdb32e374 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java @@ -453,6 +453,8 @@ public class NodesV2ApiHandler extends ThreadedHttpRequestHandler { Slime slime = new Slime(); Cursor root = slime.setObject(); + root.setDouble("total-cost", stats.totalCost()); + root.setDouble("total-allocated-cost", stats.totalAllocatedCost()); toSlime(stats.load(), root.setObject("load")); toSlime(stats.activeLoad(), root.setObject("activeLoad")); Cursor applicationsArray = root.setArray("applications"); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepoStatsTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepoStatsTest.java index 62c96af7629..788c56e08c6 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepoStatsTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepoStatsTest.java @@ -31,9 +31,13 @@ public class NodeRepoStatsTest { @Test public void testEmpty() { var tester = new NodeRepositoryTester(); - assertLoad(Load.zero(), tester.nodeRepository().computeStats().load()); - assertLoad(Load.zero(), tester.nodeRepository().computeStats().activeLoad()); - assertTrue(tester.nodeRepository().computeStats().applicationStats().isEmpty()); + var stats = tester.nodeRepository().computeStats(); + + assertEquals(0, stats.totalCost(), delta); + assertEquals(0, stats.totalAllocatedCost(), delta); + assertLoad(Load.zero(), stats.load()); + assertLoad(Load.zero(), stats.activeLoad()); + assertTrue(stats.applicationStats().isEmpty()); } @Test @@ -42,9 +46,13 @@ public class NodeRepoStatsTest { tester.addHost("host1", "default"); tester.addHost("host2", "default"); tester.addHost("host3", "small"); - assertLoad(Load.zero(), tester.nodeRepository().computeStats().load()); - assertLoad(Load.zero(), tester.nodeRepository().computeStats().activeLoad()); - assertTrue(tester.nodeRepository().computeStats().applicationStats().isEmpty()); + var stats = tester.nodeRepository().computeStats(); + + assertEquals(0.76, stats.totalCost(), delta); + assertEquals(0, stats.totalAllocatedCost(), delta); + assertLoad(Load.zero(), stats.load()); + assertLoad(Load.zero(), stats.activeLoad()); + assertTrue(stats.applicationStats().isEmpty()); } @Test @@ -97,6 +105,9 @@ public class NodeRepoStatsTest { var stats = tester.nodeRepository().computeStats(); + assertEquals(26, stats.totalCost(), delta); + assertEquals(8.319999999999999, stats.totalAllocatedCost(), delta); + assertLoad(new Load(0.6180,0.5562,0.4944), stats.load()); assertLoad(new Load(0.4682,0.4214,0.3745), stats.activeLoad()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java index 905fdc57813..e5e361da379 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java @@ -570,6 +570,39 @@ public class DynamicProvisioningMaintainerTest { assertEquals(2, provisioningTester.activate(applicationId, prepared).size()); } + @Test + public void deprovision_parked_node_with_allocation() { + var tester = new DynamicProvisioningTester(); + tester.hostProvisioner.with(Behaviour.failProvisioning); + Node host4 = tester.addNode("host4", Optional.empty(), NodeType.host, Node.State.parked); + Node host41 = tester.addNode("host4-1", Optional.of("host4"), NodeType.tenant, Node.State.parked, DynamicProvisioningTester.tenantApp); + Node host42 = tester.addNode("host4-2", Optional.of("host4"), NodeType.tenant, Node.State.active, DynamicProvisioningTester.tenantApp); + Node host43 = tester.addNode("host4-3", Optional.of("host4"), NodeType.tenant, Node.State.failed, DynamicProvisioningTester.tenantApp); + + // Host and children are marked for deprovisioning + tester.nodeRepository.nodes().deprovision("host4", Agent.operator, Instant.now()); + for (var node : List.of(host4, host41, host42, host43)) { + assertTrue(tester.nodeRepository.nodes().node(node.hostname()).map(n -> n.status().wantToDeprovision()).get()); + } + + // Host and children remain parked because one child is still active + tester.maintainer.maintain(); + for (var node : List.of(host4, host41)) { + assertEquals(Node.State.parked, tester.nodeRepository.nodes().node(node.hostname()).get().state()); + } + assertEquals(Node.State.active, tester.nodeRepository.nodes().node(host42.hostname()).get().state()); + assertEquals(Node.State.failed, tester.nodeRepository.nodes().node(host43.hostname()).get().state()); + + // Last child is parked + tester.nodeRepository.nodes().park(host42.hostname(), true, Agent.system, getClass().getSimpleName()); + + // Host and children can now be removed + tester.maintainer.maintain(); + for (var node : List.of(host4, host41, host42, host43)) { + assertTrue(node.hostname() + " removed", tester.nodeRepository.nodes().node(node.hostname()).isEmpty()); + } + } + private void assertCfghost3IsActive(DynamicProvisioningTester tester) { assertEquals(5, tester.nodeRepository.nodes().list(Node.State.active).size()); assertEquals(3, tester.nodeRepository.nodes().list(Node.State.active).nodeType(NodeType.confighost).size()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/stats.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/stats.json index 8a46f8115be..017a45d2bbe 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/stats.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/stats.json @@ -1,4 +1,6 @@ { + "total-cost" : 8.591999999999999, + "total-allocated-cost": 5.356, "load": { "cpu": 0.0, "memory": 0.0, diff --git a/searchsummary/CMakeLists.txt b/searchsummary/CMakeLists.txt index 17b46499317..8c34f1b2016 100644 --- a/searchsummary/CMakeLists.txt +++ b/searchsummary/CMakeLists.txt @@ -19,6 +19,7 @@ vespa_define_module( src/tests/docsummary src/tests/docsummary/attribute_combiner src/tests/docsummary/attributedfw + src/tests/docsummary/document_id_dfw src/tests/docsummary/matched_elements_filter src/tests/docsummary/slime_summary src/tests/docsummary/summary_field_converter diff --git a/searchsummary/src/tests/docsummary/document_id_dfw/CMakeLists.txt b/searchsummary/src/tests/docsummary/document_id_dfw/CMakeLists.txt new file mode 100644 index 00000000000..3591a36c8de --- /dev/null +++ b/searchsummary/src/tests/docsummary/document_id_dfw/CMakeLists.txt @@ -0,0 +1,10 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchsummary_document_id_dfw_test_app TEST + SOURCES + document_id_dfw_test.cpp + DEPENDS + searchsummary + searchsummary_test + GTest::GTest +) +vespa_add_test(NAME searchsummary_document_id_dfw_test_app COMMAND searchsummary_document_id_dfw_test_app) diff --git a/searchsummary/src/tests/docsummary/document_id_dfw/document_id_dfw_test.cpp b/searchsummary/src/tests/docsummary/document_id_dfw/document_id_dfw_test.cpp new file mode 100644 index 00000000000..0ea78b722c0 --- /dev/null +++ b/searchsummary/src/tests/docsummary/document_id_dfw/document_id_dfw_test.cpp @@ -0,0 +1,153 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/document/base/documentid.h> +#include <vespa/document/datatype/documenttype.h> +#include <vespa/document/fieldvalue/document.h> +#include <vespa/document/repo/configbuilder.h> +#include <vespa/document/repo/documenttyperepo.h> +#include <vespa/searchsummary/docsummary/docsum_blob_entry_filter.h> +#include <vespa/searchsummary/docsummary/docsum_store_document.h> +#include <vespa/searchsummary/docsummary/document_id_dfw.h> +#include <vespa/searchsummary/docsummary/general_result.h> +#include <vespa/searchsummary/docsummary/resultconfig.h> +#include <vespa/searchsummary/docsummary/resultpacker.h> +#include <vespa/vespalib/data/slime/slime.h> +#include <vespa/vespalib/gtest/gtest.h> +#include <iostream> +#include <memory> + +using document::Document; +using document::DocumentId; +using document::DocumentType; +using document::DocumentTypeRepo; +using document::config_builder::DocumenttypesConfigBuilderHelper; +using document::config_builder::Struct; +using search::docsummary::DocsumBlobEntryFilter; +using search::docsummary::DocsumStoreDocument; +using search::docsummary::DocsumStoreValue; +using search::docsummary::DocumentIdDFW; +using search::docsummary::IDocsumStoreDocument; +using search::docsummary::ResultClass; +using search::docsummary::ResultConfig; +using search::docsummary::ResultPacker; +using search::docsummary::GeneralResult; +using vespalib::Slime; +using vespalib::slime::Cursor; +using vespalib::slime::ObjectInserter; +using vespalib::slime::SlimeInserter; + +namespace { + +const int32_t doc_type_id = 787121340; +const vespalib::string doc_type_name = "test"; +const vespalib::string header_name = doc_type_name + ".header"; +const vespalib::string body_name = doc_type_name + ".body"; + + +std::unique_ptr<const DocumentTypeRepo> +make_doc_type_repo() +{ + DocumenttypesConfigBuilderHelper builder; + builder.document(doc_type_id, doc_type_name, + Struct(header_name), Struct(body_name)); + return std::unique_ptr<const DocumentTypeRepo>(new DocumentTypeRepo(builder.config())); +} + +class DocumentIdDFWTest : public ::testing::Test +{ + vespalib::string _field_name; + vespalib::Memory _field_name_view; + DocsumBlobEntryFilter _docsum_blob_entry_filter; + std::unique_ptr<ResultConfig> _result_config; + std::unique_ptr<ResultPacker> _packer; // owns docsum blob + std::unique_ptr<const DocumentTypeRepo> _repo; + const DocumentType* _document_type; + +protected: + DocumentIdDFWTest(); + ~DocumentIdDFWTest() override; + + std::unique_ptr<DocsumStoreDocument> make_docsum_store_document(const vespalib::string &id); + std::unique_ptr<DocsumStoreValue> make_docsum_store_value(std::unique_ptr<DocsumStoreDocument> doc); + vespalib::Slime write(const DocsumStoreValue& value); + vespalib::Memory get_field_name_view() const noexcept { return _field_name_view; } +}; + +DocumentIdDFWTest::DocumentIdDFWTest() + : testing::Test(), + _field_name("documentid"), + _field_name_view(_field_name.data(), _field_name.size()), + _docsum_blob_entry_filter(DocsumBlobEntryFilter().add_skip(search::docsummary::RES_LONG_STRING)), + _result_config(std::make_unique<ResultConfig>(_docsum_blob_entry_filter)), + _packer(std::make_unique<ResultPacker>(_result_config.get())), + _repo(make_doc_type_repo()), + _document_type(_repo->getDocumentType(doc_type_name)) +{ + auto* cfg = _result_config->AddResultClass("default", 0); + cfg->AddConfigEntry(_field_name.c_str(), search::docsummary::RES_LONG_STRING); + _result_config->CreateEnumMaps(); +} + + +DocumentIdDFWTest::~DocumentIdDFWTest() = default; + + +std::unique_ptr<DocsumStoreDocument> +DocumentIdDFWTest::make_docsum_store_document(const vespalib::string& id) +{ + auto doc = std::make_unique<Document>(*_document_type, DocumentId(id)); + doc->setRepo(*_repo); + return std::make_unique<DocsumStoreDocument>(std::move(doc)); +} + +std::unique_ptr<DocsumStoreValue> +DocumentIdDFWTest::make_docsum_store_value(std::unique_ptr<DocsumStoreDocument> doc) +{ + EXPECT_TRUE(_packer->Init(0)); + const char *ptr = nullptr; + uint32_t len = 0; + EXPECT_TRUE(_packer->GetDocsumBlob(&ptr, &len)); + return std::make_unique<DocsumStoreValue>(ptr, len, std::move(doc)); +} + +vespalib::Slime +DocumentIdDFWTest::write(const DocsumStoreValue& value) +{ + auto result = std::make_unique<GeneralResult>(_result_config->LookupResultClass(0)); + EXPECT_TRUE(result->inplaceUnpack(value)); + Slime slime; + SlimeInserter top_inserter(slime); + Cursor & docsum = top_inserter.insertObject(); + ObjectInserter field_inserter(docsum, _field_name_view); + DocumentIdDFW writer; + writer.insertField(0, result.get(), nullptr, search::docsummary::RES_LONG_STRING, field_inserter); + return slime; +} + +TEST_F(DocumentIdDFWTest, insert_document_id) +{ + vespalib::string id("id::test::0"); + auto doc = make_docsum_store_document(id); + auto dsvalue = make_docsum_store_value(std::move(doc)); + auto slime = write(*dsvalue); + EXPECT_TRUE(slime.get()[get_field_name_view()].valid()); + EXPECT_EQ(id, slime.get()[get_field_name_view()].asString().make_string()); +} + +TEST_F(DocumentIdDFWTest, insert_document_id_no_document_doc) +{ + auto dsvalue = make_docsum_store_value(std::make_unique<DocsumStoreDocument>(std::unique_ptr<Document>())); + auto slime = write(*dsvalue); + EXPECT_FALSE(slime.get()[get_field_name_view()].valid()); +} + +TEST_F(DocumentIdDFWTest, insert_document_id_no_docsum_store_doc) +{ + auto dsvalue = make_docsum_store_value({}); + auto slime = write(*dsvalue); + EXPECT_FALSE(slime.get()[get_field_name_view()].valid()); +} + +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchsummary/src/vespa/searchsummary/docsummary/CMakeLists.txt b/searchsummary/src/vespa/searchsummary/docsummary/CMakeLists.txt index e3272fb36de..e262482cddb 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/CMakeLists.txt +++ b/searchsummary/src/vespa/searchsummary/docsummary/CMakeLists.txt @@ -13,6 +13,7 @@ vespa_add_library(searchsummary_docsummary OBJECT docsumstate.cpp docsumstorevalue.cpp docsumwriter.cpp + document_id_dfw.cpp dynamicteaserdfw.cpp empty_dfw.cpp general_result.cpp diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsum_store_document.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsum_store_document.cpp index e525989e972..e6c8d0b6ab8 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsum_store_document.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsum_store_document.cpp @@ -5,6 +5,7 @@ #include "summaryfieldconverter.h" #include <vespa/document/datatype/datatype.h> #include <vespa/document/fieldvalue/document.h> +#include <vespa/vespalib/data/slime/inserter.h> namespace search::docsummary { @@ -39,4 +40,14 @@ DocsumStoreDocument::insert_summary_field(const vespalib::string& field_name, ve } } +void +DocsumStoreDocument::insert_document_id(vespalib::slime::Inserter& inserter) const +{ + if (_document) { + auto id = _document->getId().toString(); + vespalib::Memory id_view(id.data(), id.size()); + inserter.insertString(id_view); + } +} + } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsum_store_document.h b/searchsummary/src/vespa/searchsummary/docsummary/docsum_store_document.h index 66a5a74fa8d..3b0bea6e721 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsum_store_document.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsum_store_document.h @@ -19,6 +19,7 @@ public: ~DocsumStoreDocument() override; std::unique_ptr<document::FieldValue> get_field_value(const vespalib::string& field_name) const override; void insert_summary_field(const vespalib::string& field_name, vespalib::slime::Inserter& inserter) const override; + void insert_document_id(vespalib::slime::Inserter& inserter) const override; }; } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumconfig.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsumconfig.cpp index 376d4f90204..fea11923858 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumconfig.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumconfig.cpp @@ -4,6 +4,7 @@ #include "attribute_combiner_dfw.h" #include "copy_dfw.h" #include "docsumwriter.h" +#include "document_id_dfw.h" #include "empty_dfw.h" #include "geoposdfw.h" #include "idocsumenvironment.h" @@ -113,6 +114,9 @@ DynamicDocsumConfig::createFieldWriter(const string & fieldName, const string & *attr_ctx, matching_elems_fields); rc = static_cast<bool>(fieldWriter); } + } else if (overrideName == "documentid") { + fieldWriter = std::make_unique<DocumentIdDFW>(); + rc = true; } else { throw IllegalArgumentException("unknown override operation '" + overrideName + "' for field '" + fieldName + "'."); } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/document_id_dfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/document_id_dfw.cpp new file mode 100644 index 00000000000..8fcf7c95cdc --- /dev/null +++ b/searchsummary/src/vespa/searchsummary/docsummary/document_id_dfw.cpp @@ -0,0 +1,31 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "document_id_dfw.h" +#include "general_result.h" +#include "i_docsum_store_document.h" + +namespace search::docsummary { + +DocumentIdDFW::DocumentIdDFW() +{ +} + +DocumentIdDFW::~DocumentIdDFW() = default; + +bool +DocumentIdDFW::IsGenerated() const +{ + return false; +} + +void +DocumentIdDFW::insertField(uint32_t, GeneralResult *gres, GetDocsumsState *, ResType, + vespalib::slime::Inserter &target) +{ + const auto* document = gres->get_document(); + if (document != nullptr) { + document->insert_document_id(target); + } +} + +} diff --git a/searchsummary/src/vespa/searchsummary/docsummary/document_id_dfw.h b/searchsummary/src/vespa/searchsummary/docsummary/document_id_dfw.h new file mode 100644 index 00000000000..f6353912384 --- /dev/null +++ b/searchsummary/src/vespa/searchsummary/docsummary/document_id_dfw.h @@ -0,0 +1,22 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "docsum_field_writer.h" + +namespace search::docsummary { + +/* + * Class for writing document id field. + */ +class DocumentIdDFW : public DocsumFieldWriter +{ +private: +public: + DocumentIdDFW(); + ~DocumentIdDFW() override; + bool IsGenerated() const override; + void insertField(uint32_t docid, GeneralResult *gres, GetDocsumsState *state, ResType type, vespalib::slime::Inserter &target) override; +}; + +} diff --git a/searchsummary/src/vespa/searchsummary/docsummary/i_docsum_store_document.h b/searchsummary/src/vespa/searchsummary/docsummary/i_docsum_store_document.h index f81412eb34c..c177568c467 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/i_docsum_store_document.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/i_docsum_store_document.h @@ -22,6 +22,7 @@ public: virtual ~IDocsumStoreDocument() = default; virtual std::unique_ptr<document::FieldValue> get_field_value(const vespalib::string& field_name) const = 0; virtual void insert_summary_field(const vespalib::string& field_name, vespalib::slime::Inserter& inserter) const = 0; + virtual void insert_document_id(vespalib::slime::Inserter& inserter) const = 0; }; } diff --git a/security-utils/pom.xml b/security-utils/pom.xml index d4b4bc8d0f6..0a2e1251d6a 100644 --- a/security-utils/pom.xml +++ b/security-utils/pom.xml @@ -71,10 +71,6 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-compiler-plugin</artifactId> - <configuration> - <release>${vespaClients.jdk.releaseVersion}</release> - <showDeprecation>true</showDeprecation> - </configuration> </plugin> <plugin> <groupId>org.codehaus.mojo</groupId> @@ -89,7 +85,10 @@ </executions> </plugin> <plugin> - <!-- Build with maven-bundle-plugin to avoid depending on jdisc_core to get the correct Import-Packages --> + <!-- + Build with maven-bundle-plugin to avoid depending on jdisc_core to get the correct Import-Packages + Jackson and BC compile scope dependencies are not embedded. Packages are instead imported through manifest. + --> <groupId>org.apache.felix</groupId> <artifactId>maven-bundle-plugin</artifactId> <extensions>true</extensions> diff --git a/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.cpp b/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.cpp index 20083b9160e..5bcead79f97 100644 --- a/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.cpp +++ b/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.cpp @@ -135,6 +135,7 @@ public: ~DocsumStoreVsmDocument() override; std::unique_ptr<document::FieldValue> get_field_value(const vespalib::string& field_name) const override; void insert_summary_field(const vespalib::string& field_name, vespalib::slime::Inserter& inserter) const override; + void insert_document_id(vespalib::slime::Inserter& inserter) const override; }; DocsumStoreVsmDocument::DocsumStoreVsmDocument(const document::Document* document) @@ -168,6 +169,16 @@ DocsumStoreVsmDocument::insert_summary_field(const vespalib::string& field_name, } } +void +DocsumStoreVsmDocument::insert_document_id(vespalib::slime::Inserter& inserter) const +{ + if (_document) { + auto id = _document->getId().toString(); + vespalib::Memory id_view(id.data(), id.size()); + inserter.insertString(id_view); + } +} + } FieldPath |