diff options
14 files changed, 69 insertions, 20 deletions
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java index cbfe2826a76..4b1c03a170c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container; +import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.api.container.ContainerServiceType; import com.yahoo.config.model.deploy.DeployState; @@ -147,6 +148,19 @@ public abstract class Container extends AbstractService implements return (parent instanceof ContainerCluster) ? ((ContainerCluster<?>) parent).getHttp() : null; } + public JettyHttpServer getDefaultHttpServer() { + return defaultHttpServer; + } + + public JettyHttpServer getHttpServer() { + Http http = getHttp(); + if (http == null) { + return defaultHttpServer; + } else { + return http.getHttpServer().orElse(null); + } + } + /** Returns the index of this node. The index of a given node is stable through changes with best effort. */ public final int index() { return index; } @@ -367,7 +381,7 @@ public abstract class Container extends AbstractService implements @Override public void getConfig(ContainerMbusConfig.Builder builder) { - builder.port(getMessagingPort()); + builder.enabled(messageBusEnabled()).port(getMessagingPort()); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index 6b6192a04ef..5adea6d50e8 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -412,6 +412,8 @@ public abstract class ContainerCluster<CONTAINER extends Container> @Override public void getConfig(DocumentmanagerConfig.Builder builder) { + if (containerDocproc != null && containerDocproc.isCompressDocuments()) + builder.enablecompression(true); if (containerDocumentApi != null) builder.ignoreundefinedfields(containerDocumentApi.ignoreUndefinedFields()); } @@ -537,7 +539,9 @@ public abstract class ContainerCluster<CONTAINER extends Container> @Override public void getConfig(IlscriptsConfig.Builder builder) { - for (SearchCluster searchCluster : Content.getSearchClusters(getRoot().configModelRepo())) { + List<SearchCluster> searchClusters = new ArrayList<>(); + searchClusters.addAll(Content.getSearchClusters(getRoot().configModelRepo())); + for (SearchCluster searchCluster : searchClusters) { searchCluster.getConfig(builder); } } @@ -609,6 +613,8 @@ public abstract class ContainerCluster<CONTAINER extends Container> public void setEnvironmentVars(String environmentVars) { this.environmentVars = environmentVars; } + public String getEnvironmentVars() { return environmentVars; } + public Optional<String> getJvmGCOptions() { return Optional.ofNullable(jvmGCOptions); } public final void setRpcServerEnabled(boolean rpcServerEnabled) { this.rpcServerEnabled = rpcServerEnabled; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/ContainerDocproc.java b/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/ContainerDocproc.java index f2233680e9b..8be02f77ab3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/ContainerDocproc.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/ContainerDocproc.java @@ -35,7 +35,7 @@ public class ContainerDocproc extends ContainerSubsystem<DocprocChains> private Map<Pair<String, String>, String> fieldNameSchemaMap = new HashMap<>(); public ContainerDocproc(ContainerCluster cluster, DocprocChains chains) { - this(cluster, chains, new Options( null, null, null, null, null, null)); + this(cluster, chains, new Options(false, null, null, null, null, null, null)); } public ContainerDocproc(ContainerCluster cluster, DocprocChains chains, Options options) { @@ -60,6 +60,10 @@ public class ContainerDocproc extends ContainerSubsystem<DocprocChains> } } + public boolean isCompressDocuments() { + return options.compressDocuments; + } + public boolean isPreferLocalNode() { return preferLocalNode; } @@ -71,6 +75,8 @@ public class ContainerDocproc extends ContainerSubsystem<DocprocChains> @Override public void getConfig(ContainerMbusConfig.Builder builder) { builder.maxpendingcount(getMaxMessagesInQueue()); + if (getMaxQueueMbSize() != null) + builder.maxpendingsize(getMaxQueueMbSize()); //yes, this shall be set in megabytes. } private int getMaxMessagesInQueue() { @@ -131,6 +137,8 @@ public class ContainerDocproc extends ContainerSubsystem<DocprocChains> } public static class Options { + // Whether or not to compress documents after processing them. + public final boolean compressDocuments; public final Integer maxMessagesInQueue; public final Integer maxQueueMbSize; @@ -140,7 +148,8 @@ public class ContainerDocproc extends ContainerSubsystem<DocprocChains> public final Double documentExpansionFactor; public final Integer containerCoreMemory; - public Options(Integer maxMessagesInQueue, Integer maxQueueMbSize, Integer maxQueueTimeMs, Double maxConcurrentFactor, Double documentExpansionFactor, Integer containerCoreMemory) { + public Options(boolean compressDocuments, Integer maxMessagesInQueue, Integer maxQueueMbSize, Integer maxQueueTimeMs, Double maxConcurrentFactor, Double documentExpansionFactor, Integer containerCoreMemory) { + this.compressDocuments = compressDocuments; this.maxMessagesInQueue = maxMessagesInQueue; this.maxQueueMbSize = maxQueueMbSize; this.maxQueueTimeMs = maxQueueTimeMs; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocprocOptionsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocprocOptionsBuilder.java index e0dffca2bc5..1239cbf9bdc 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocprocOptionsBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocprocOptionsBuilder.java @@ -10,6 +10,7 @@ import org.w3c.dom.Element; public class DocprocOptionsBuilder { public static ContainerDocproc.Options build(Element spec) { return new ContainerDocproc.Options( + getCompression(spec), getMaxMessagesInQueue(spec), getSizeInMegabytes(spec.getAttribute("maxqueuebytesize")), getTime(spec.getAttribute("maxqueuewait")), @@ -24,6 +25,10 @@ public class DocprocOptionsBuilder { Integer.parseInt(integer); } + private static boolean getCompression(Element spec) { + return (spec.hasAttribute("compressdocuments") && spec.getAttribute("compressdocuments").equals("true")); + } + private static Double getFactor(String factor) { return factor == null || factor.trim().isEmpty() ? null : diff --git a/config-model/src/main/resources/schema/docproc.rnc b/config-model/src/main/resources/schema/docproc.rnc index 42902f7180f..1e7e28b2002 100644 --- a/config-model/src/main/resources/schema/docproc.rnc +++ b/config-model/src/main/resources/schema/docproc.rnc @@ -22,7 +22,6 @@ DocProcV3 = attribute version { "3.0" }, GenericConfig* ) -# TODO Here we need a thorough cleaning DocprocClusterAttributes = attribute compressdocuments { xsd:boolean }? & attribute numnodesperclient { xsd:positiveInteger }? & attribute preferlocalnode { xsd:boolean }? & @@ -33,7 +32,6 @@ DocprocClusterAttributes = attribute compressdocuments { xsd:boolean }? & attribute documentexpansionfactor { xsd:double { minExclusive = "0.0" } }? & attribute containercorememory { xsd:nonNegativeInteger }? -# TODO Here we need a thorough cleaning ClusterV3 = element cluster { attribute name { xsd:NCName } & DocprocClusterAttributes? & diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/DocprocBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/DocprocBuilderTest.java index 2044fd2ab39..77681489dac 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/DocprocBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/DocprocBuilderTest.java @@ -1,12 +1,14 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container.xml; +import com.yahoo.config.docproc.DocprocConfig; import com.yahoo.config.docproc.SchemamappingConfig; import com.yahoo.config.model.builder.xml.test.DomBuilderTest; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.container.ComponentsConfig; import com.yahoo.container.core.ChainsConfig; import com.yahoo.container.jdisc.ContainerMbusConfig; +import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.search.config.QrStartConfig; import com.yahoo.vespa.model.HostPorts; import com.yahoo.vespa.model.container.ApplicationContainer; @@ -36,10 +38,12 @@ import static org.junit.Assert.assertTrue; public class DocprocBuilderTest extends DomBuilderTest { private ApplicationContainerCluster cluster; + private DocumentmanagerConfig documentmanagerConfig; private ContainerMbusConfig containerMbusConfig; private ComponentsConfig componentsConfig; private ChainsConfig chainsConfig; private SchemamappingConfig schemamappingConfig; + private DocprocConfig docprocConfig; private QrStartConfig qrStartConfig; @Before @@ -54,8 +58,10 @@ public class DocprocBuilderTest extends DomBuilderTest { chainsConfig = root.getConfig(ChainsConfig.class, cluster.getConfigId() + "/component/com.yahoo.docproc.jdisc.DocumentProcessingHandler"); + documentmanagerConfig = root.getConfig(DocumentmanagerConfig.class, cluster.getConfigId()); schemamappingConfig = root.getConfig(SchemamappingConfig.class, cluster.getContainers().get(0).getConfigId()); qrStartConfig = root.getConfig(QrStartConfig.class, cluster.getConfigId()); + docprocConfig = root.getConfig(DocprocConfig.class, cluster.getConfigId()); } private Element servicesXml() { @@ -64,7 +70,7 @@ public class DocprocBuilderTest extends DomBuilderTest { " <nodes>", " <node hostalias='mockhost' baseport='1500' />", " </nodes>", - " <document-processing preferlocalnode='true' numnodesperclient='2' maxqueuebytesize='100m' maxmessagesinqueue='300' maxqueuewait='200'>", + " <document-processing compressdocuments='true' preferlocalnode='true' numnodesperclient='2' maxqueuebytesize='100m' maxmessagesinqueue='300' maxqueuewait='200'>", " <documentprocessor id='docproc1' class='com.yahoo.Docproc1' bundle='docproc1bundle'/>", " <chain id='chein'>", " <documentprocessor id='docproc2'/>", @@ -77,6 +83,7 @@ public class DocprocBuilderTest extends DomBuilderTest { @Test public void testDocprocCluster() { assertEquals("banan", cluster.getName()); + assertTrue(cluster.getDocproc().isCompressDocuments()); //assertTrue(cluster.getContainerDocproc().isPreferLocalNode()); //assertEquals(2, cluster.getContainerDocproc().getNumNodesPerClient()); List<ApplicationContainer> services = cluster.getContainers(); @@ -98,9 +105,16 @@ public class DocprocBuilderTest extends DomBuilderTest { } @Test + public void testDocumentManagerConfig() { + assertTrue(documentmanagerConfig.enablecompression()); + } + + @Test public void testContainerMbusConfig() { + assertTrue(containerMbusConfig.enabled()); assertTrue(containerMbusConfig.port() >= HostPorts.BASE_PORT); assertEquals(300, containerMbusConfig.maxpendingcount()); + assertEquals(100, containerMbusConfig.maxpendingsize()); } @Test diff --git a/container-messagebus/src/main/resources/configdefinitions/container.jdisc.container-mbus.def b/container-messagebus/src/main/resources/configdefinitions/container.jdisc.container-mbus.def index bd43c13aba3..143e02a30ef 100644 --- a/container-messagebus/src/main/resources/configdefinitions/container.jdisc.container-mbus.def +++ b/container-messagebus/src/main/resources/configdefinitions/container.jdisc.container-mbus.def @@ -22,6 +22,10 @@ transport_events_before_wakeup int default=1 # Dynamic throttling is used, and works better than anything else. maxpendingcount int default=2048 +enabled bool default=false +#maxpendingsize is set in megabytes! +maxpendingsize int default=100 + #The amount of input data that the service can process concurrently maxConcurrentFactor double default=0.2 range=[0.0-1.0] diff --git a/document/src/vespa/document/config/documentmanager.def b/document/src/vespa/document/config/documentmanager.def index 0d0f3876f15..b1929e42d34 100644 --- a/document/src/vespa/document/config/documentmanager.def +++ b/document/src/vespa/document/config/documentmanager.def @@ -5,6 +5,9 @@ namespace=document.config ## Whether attempts to set an undefined field should be ignored rather than causing an error ignoreundefinedfields bool default=false +## Whether to enable compression in this process. +enablecompression bool default=false + ## Prefer "Vespa 8" format for the "position" type usev8geopositions bool default=false diff --git a/document/src/vespa/document/config/documenttypes.def b/document/src/vespa/document/config/documenttypes.def index 0c135db7b0d..3138e71e025 100644 --- a/document/src/vespa/document/config/documenttypes.def +++ b/document/src/vespa/document/config/documenttypes.def @@ -5,6 +5,9 @@ namespace=document.config ## Whether attempts to set an undefined field should be ignored rather than causing an error ignoreundefinedfields bool default=false +## Whether to enable compression in this process. +enablecompression bool default=false + ## Prefer "Vespa 8" format for the "position" type usev8geopositions bool default=false diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusVisitorSession.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusVisitorSession.java index 1c729008e2c..3607f652a51 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusVisitorSession.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusVisitorSession.java @@ -230,6 +230,7 @@ public class MessageBusVisitorSession implements VisitorSession { @Override public Sender createSender(ReplyHandler replyHandler, VisitorParameters visitorParameters) { + messageBus.setMaxPendingCount(0); SourceSessionParams sessionParams = createSourceSessionParams(visitorParameters); return new MessageBusSender(messageBus.createSourceSession(replyHandler, sessionParams)); } @@ -307,7 +308,7 @@ public class MessageBusVisitorSession implements VisitorSession { private static final Logger log = Logger.getLogger(MessageBusVisitorSession.class.getName()); - private static final AtomicLong sessionCounter = new AtomicLong(0); + private static AtomicLong sessionCounter = new AtomicLong(0); private static long getNextSessionId() { return sessionCounter.incrementAndGet(); } @@ -335,7 +336,7 @@ public class MessageBusVisitorSession implements VisitorSession { private boolean done = false; private boolean destroying = false; // For testing and sanity checking private final Object completionMonitor = new Object(); - private final Trace trace; + private Trace trace; /** * We keep our own track of pending messages since the sender's pending * count cannot be relied on in an async task execution context. This diff --git a/messagebus/src/main/java/com/yahoo/messagebus/MessageBus.java b/messagebus/src/main/java/com/yahoo/messagebus/MessageBus.java index 19d3b5b3e43..3bd5d5d4890 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/MessageBus.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/MessageBus.java @@ -470,7 +470,6 @@ public class MessageBus implements ConfigHandler, NetworkOwner, MessageHandler, * * @return The resender. */ - @Deprecated // Remove on 9 public Resender getResender() { return resender; } @@ -481,7 +480,6 @@ public class MessageBus implements ConfigHandler, NetworkOwner, MessageHandler, * * @return The pending count. */ - @Deprecated // Package private on 9 public synchronized int getPendingCount() { return pendingCount; } @@ -492,7 +490,6 @@ public class MessageBus implements ConfigHandler, NetworkOwner, MessageHandler, * * @return The pending size. */ - @Deprecated // Package private on 9 public synchronized int getPendingSize() { return pendingSize; } @@ -503,7 +500,6 @@ public class MessageBus implements ConfigHandler, NetworkOwner, MessageHandler, * * @param maxCount The max count. */ - @Deprecated // Remove on 9 public void setMaxPendingCount(int maxCount) { maxPendingCount = maxCount; } @@ -512,7 +508,6 @@ public class MessageBus implements ConfigHandler, NetworkOwner, MessageHandler, * Gets maximum number of messages that can be received without being * replied to yet. */ - @Deprecated // Remove on 9 public int getMaxPendingCount() { return maxPendingCount; } @@ -523,7 +518,6 @@ public class MessageBus implements ConfigHandler, NetworkOwner, MessageHandler, * * @param maxSize The max size. */ - @Deprecated // Remove on 9 public void setMaxPendingSize(int maxSize) { maxPendingSize = maxSize; } @@ -532,7 +526,6 @@ public class MessageBus implements ConfigHandler, NetworkOwner, MessageHandler, * Gets maximum combined size of messages that can be received without * being replied to yet. */ - @Deprecated // Remove on 9 public int getMaxPendingSize() { return maxPendingSize; } diff --git a/messagebus/src/test/java/com/yahoo/messagebus/ChokeTestCase.java b/messagebus/src/test/java/com/yahoo/messagebus/ChokeTestCase.java index 6d50fc769be..60ec6c400a9 100755 --- a/messagebus/src/test/java/com/yahoo/messagebus/ChokeTestCase.java +++ b/messagebus/src/test/java/com/yahoo/messagebus/ChokeTestCase.java @@ -14,6 +14,7 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; +import java.net.UnknownHostException; import java.util.ArrayList; import java.util.List; import static org.junit.Assert.assertEquals; @@ -54,7 +55,6 @@ public class ChokeTestCase { } @Test - @SuppressWarnings("deprecation") public void testMaxCount() { int max = 10; dstServer.mb.setMaxPendingCount(max); @@ -103,7 +103,6 @@ public class ChokeTestCase { } @Test - @SuppressWarnings("deprecation") public void testMaxSize() { int size = createMessage("msg").getApproxSize(); int max = size * 10; diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerTest.java index 2fc34112517..f3ea8fb5a80 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerTest.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerTest.java @@ -25,7 +25,7 @@ public class FeedHandlerTest { FeedHandler handler = new FeedHandler( new RejectingContainerThreadpool(), new CollectingMetric(), - new DocumentTypeManager(new DocumentmanagerConfig.Builder().build()), + new DocumentTypeManager(new DocumentmanagerConfig.Builder().enablecompression(true).build()), null /* session cache */, MetricReceiver.nullImplementation); var responseHandler = new RequestHandlerTestDriver.MockResponseHandler(); diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerV3Test.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerV3Test.java index 5b8b5b1827f..dcabc1f338e 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerV3Test.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerV3Test.java @@ -108,7 +108,7 @@ public class FeedHandlerV3Test { } private FeedHandlerV3 setupFeederHandler(Executor threadPool) { - DocumentTypeManager docMan = new DocumentTypeManager(new DocumentmanagerConfig.Builder().build()); + DocumentTypeManager docMan = new DocumentTypeManager(new DocumentmanagerConfig.Builder().enablecompression(true).build()); FeedHandlerV3 feedHandlerV3 = new FeedHandlerV3( threadPool, metric, |