diff options
67 files changed, 122 insertions, 20 deletions
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java index cbfe2826a76..4b1c03a170c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container; +import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.api.container.ContainerServiceType; import com.yahoo.config.model.deploy.DeployState; @@ -147,6 +148,19 @@ public abstract class Container extends AbstractService implements return (parent instanceof ContainerCluster) ? ((ContainerCluster<?>) parent).getHttp() : null; } + public JettyHttpServer getDefaultHttpServer() { + return defaultHttpServer; + } + + public JettyHttpServer getHttpServer() { + Http http = getHttp(); + if (http == null) { + return defaultHttpServer; + } else { + return http.getHttpServer().orElse(null); + } + } + /** Returns the index of this node. The index of a given node is stable through changes with best effort. */ public final int index() { return index; } @@ -367,7 +381,7 @@ public abstract class Container extends AbstractService implements @Override public void getConfig(ContainerMbusConfig.Builder builder) { - builder.port(getMessagingPort()); + builder.enabled(messageBusEnabled()).port(getMessagingPort()); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index 6b6192a04ef..5adea6d50e8 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -412,6 +412,8 @@ public abstract class ContainerCluster<CONTAINER extends Container> @Override public void getConfig(DocumentmanagerConfig.Builder builder) { + if (containerDocproc != null && containerDocproc.isCompressDocuments()) + builder.enablecompression(true); if (containerDocumentApi != null) builder.ignoreundefinedfields(containerDocumentApi.ignoreUndefinedFields()); } @@ -537,7 +539,9 @@ public abstract class ContainerCluster<CONTAINER extends Container> @Override public void getConfig(IlscriptsConfig.Builder builder) { - for (SearchCluster searchCluster : Content.getSearchClusters(getRoot().configModelRepo())) { + List<SearchCluster> searchClusters = new ArrayList<>(); + searchClusters.addAll(Content.getSearchClusters(getRoot().configModelRepo())); + for (SearchCluster searchCluster : searchClusters) { searchCluster.getConfig(builder); } } @@ -609,6 +613,8 @@ public abstract class ContainerCluster<CONTAINER extends Container> public void setEnvironmentVars(String environmentVars) { this.environmentVars = environmentVars; } + public String getEnvironmentVars() { return environmentVars; } + public Optional<String> getJvmGCOptions() { return Optional.ofNullable(jvmGCOptions); } public final void setRpcServerEnabled(boolean rpcServerEnabled) { this.rpcServerEnabled = rpcServerEnabled; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/ContainerDocproc.java b/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/ContainerDocproc.java index f2233680e9b..8be02f77ab3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/ContainerDocproc.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/ContainerDocproc.java @@ -35,7 +35,7 @@ public class ContainerDocproc extends ContainerSubsystem<DocprocChains> private Map<Pair<String, String>, String> fieldNameSchemaMap = new HashMap<>(); public ContainerDocproc(ContainerCluster cluster, DocprocChains chains) { - this(cluster, chains, new Options( null, null, null, null, null, null)); + this(cluster, chains, new Options(false, null, null, null, null, null, null)); } public ContainerDocproc(ContainerCluster cluster, DocprocChains chains, Options options) { @@ -60,6 +60,10 @@ public class ContainerDocproc extends ContainerSubsystem<DocprocChains> } } + public boolean isCompressDocuments() { + return options.compressDocuments; + } + public boolean isPreferLocalNode() { return preferLocalNode; } @@ -71,6 +75,8 @@ public class ContainerDocproc extends ContainerSubsystem<DocprocChains> @Override public void getConfig(ContainerMbusConfig.Builder builder) { builder.maxpendingcount(getMaxMessagesInQueue()); + if (getMaxQueueMbSize() != null) + builder.maxpendingsize(getMaxQueueMbSize()); //yes, this shall be set in megabytes. } private int getMaxMessagesInQueue() { @@ -131,6 +137,8 @@ public class ContainerDocproc extends ContainerSubsystem<DocprocChains> } public static class Options { + // Whether or not to compress documents after processing them. + public final boolean compressDocuments; public final Integer maxMessagesInQueue; public final Integer maxQueueMbSize; @@ -140,7 +148,8 @@ public class ContainerDocproc extends ContainerSubsystem<DocprocChains> public final Double documentExpansionFactor; public final Integer containerCoreMemory; - public Options(Integer maxMessagesInQueue, Integer maxQueueMbSize, Integer maxQueueTimeMs, Double maxConcurrentFactor, Double documentExpansionFactor, Integer containerCoreMemory) { + public Options(boolean compressDocuments, Integer maxMessagesInQueue, Integer maxQueueMbSize, Integer maxQueueTimeMs, Double maxConcurrentFactor, Double documentExpansionFactor, Integer containerCoreMemory) { + this.compressDocuments = compressDocuments; this.maxMessagesInQueue = maxMessagesInQueue; this.maxQueueMbSize = maxQueueMbSize; this.maxQueueTimeMs = maxQueueTimeMs; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocprocOptionsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocprocOptionsBuilder.java index e0dffca2bc5..1239cbf9bdc 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocprocOptionsBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocprocOptionsBuilder.java @@ -10,6 +10,7 @@ import org.w3c.dom.Element; public class DocprocOptionsBuilder { public static ContainerDocproc.Options build(Element spec) { return new ContainerDocproc.Options( + getCompression(spec), getMaxMessagesInQueue(spec), getSizeInMegabytes(spec.getAttribute("maxqueuebytesize")), getTime(spec.getAttribute("maxqueuewait")), @@ -24,6 +25,10 @@ public class DocprocOptionsBuilder { Integer.parseInt(integer); } + private static boolean getCompression(Element spec) { + return (spec.hasAttribute("compressdocuments") && spec.getAttribute("compressdocuments").equals("true")); + } + private static Double getFactor(String factor) { return factor == null || factor.trim().isEmpty() ? null : diff --git a/config-model/src/main/resources/schema/docproc.rnc b/config-model/src/main/resources/schema/docproc.rnc index 42902f7180f..1e7e28b2002 100644 --- a/config-model/src/main/resources/schema/docproc.rnc +++ b/config-model/src/main/resources/schema/docproc.rnc @@ -22,7 +22,6 @@ DocProcV3 = attribute version { "3.0" }, GenericConfig* ) -# TODO Here we need a thorough cleaning DocprocClusterAttributes = attribute compressdocuments { xsd:boolean }? & attribute numnodesperclient { xsd:positiveInteger }? & attribute preferlocalnode { xsd:boolean }? & @@ -33,7 +32,6 @@ DocprocClusterAttributes = attribute compressdocuments { xsd:boolean }? & attribute documentexpansionfactor { xsd:double { minExclusive = "0.0" } }? & attribute containercorememory { xsd:nonNegativeInteger }? -# TODO Here we need a thorough cleaning ClusterV3 = element cluster { attribute name { xsd:NCName } & DocprocClusterAttributes? & diff --git a/config-model/src/test/configmodel/types/documentmanager.cfg b/config-model/src/test/configmodel/types/documentmanager.cfg index 0269efcc4ca..c471935d9da 100644 --- a/config-model/src/test/configmodel/types/documentmanager.cfg +++ b/config-model/src/test/configmodel/types/documentmanager.cfg @@ -1,4 +1,5 @@ ignoreundefinedfields false +enablecompression false usev8geopositions false doctype[0].name "document" doctype[0].idx 10000 diff --git a/config-model/src/test/configmodel/types/documenttypes.cfg b/config-model/src/test/configmodel/types/documenttypes.cfg index b1c3075df2c..d35928a7a4b 100644 --- a/config-model/src/test/configmodel/types/documenttypes.cfg +++ b/config-model/src/test/configmodel/types/documenttypes.cfg @@ -1,4 +1,5 @@ ignoreundefinedfields false +enablecompression false usev8geopositions false doctype[0].name "document" doctype[0].idx 10000 diff --git a/config-model/src/test/configmodel/types/documenttypes_with_doc_field.cfg b/config-model/src/test/configmodel/types/documenttypes_with_doc_field.cfg index be4a18f2ab2..f8c7dacd3af 100644 --- a/config-model/src/test/configmodel/types/documenttypes_with_doc_field.cfg +++ b/config-model/src/test/configmodel/types/documenttypes_with_doc_field.cfg @@ -1,4 +1,5 @@ ignoreundefinedfields false +enablecompression false usev8geopositions false doctype[0].name "document" doctype[0].idx 10000 diff --git a/config-model/src/test/configmodel/types/references/documentmanager_multiple_imported_fields.cfg b/config-model/src/test/configmodel/types/references/documentmanager_multiple_imported_fields.cfg index 778fe2a44c4..bf2ae28c417 100644 --- a/config-model/src/test/configmodel/types/references/documentmanager_multiple_imported_fields.cfg +++ b/config-model/src/test/configmodel/types/references/documentmanager_multiple_imported_fields.cfg @@ -1,4 +1,5 @@ ignoreundefinedfields false +enablecompression false usev8geopositions false doctype[0].name "document" doctype[0].idx 10000 diff --git a/config-model/src/test/configmodel/types/references/documentmanager_refs_to_other_types.cfg b/config-model/src/test/configmodel/types/references/documentmanager_refs_to_other_types.cfg index 4d0a8396d91..7f93eea8e90 100644 --- a/config-model/src/test/configmodel/types/references/documentmanager_refs_to_other_types.cfg +++ b/config-model/src/test/configmodel/types/references/documentmanager_refs_to_other_types.cfg @@ -1,4 +1,5 @@ ignoreundefinedfields false +enablecompression false usev8geopositions false doctype[0].name "document" doctype[0].idx 10000 diff --git a/config-model/src/test/configmodel/types/references/documentmanager_refs_to_same_type.cfg b/config-model/src/test/configmodel/types/references/documentmanager_refs_to_same_type.cfg index 17d6264e138..ecb499b5ed3 100644 --- a/config-model/src/test/configmodel/types/references/documentmanager_refs_to_same_type.cfg +++ b/config-model/src/test/configmodel/types/references/documentmanager_refs_to_same_type.cfg @@ -1,4 +1,5 @@ ignoreundefinedfields false +enablecompression false usev8geopositions false doctype[0].name "document" doctype[0].idx 10000 diff --git a/config-model/src/test/configmodel/types/references/documenttypes_multiple_imported_fields.cfg b/config-model/src/test/configmodel/types/references/documenttypes_multiple_imported_fields.cfg index 9e76cb14b20..6073faca616 100644 --- a/config-model/src/test/configmodel/types/references/documenttypes_multiple_imported_fields.cfg +++ b/config-model/src/test/configmodel/types/references/documenttypes_multiple_imported_fields.cfg @@ -1,4 +1,5 @@ ignoreundefinedfields false +enablecompression false usev8geopositions false doctype[0].name "document" doctype[0].idx 10000 diff --git a/config-model/src/test/configmodel/types/references/documenttypes_ref_to_self_type.cfg b/config-model/src/test/configmodel/types/references/documenttypes_ref_to_self_type.cfg index e8398b159dd..92bde94f54a 100644 --- a/config-model/src/test/configmodel/types/references/documenttypes_ref_to_self_type.cfg +++ b/config-model/src/test/configmodel/types/references/documenttypes_ref_to_self_type.cfg @@ -1,4 +1,5 @@ ignoreundefinedfields false +enablecompression false usev8geopositions false documenttype[].id 2987301 documenttype[].name "ad" diff --git a/config-model/src/test/configmodel/types/references/documenttypes_refs_to_other_types.cfg b/config-model/src/test/configmodel/types/references/documenttypes_refs_to_other_types.cfg index 0d817fd4e07..5ea07de4124 100644 --- a/config-model/src/test/configmodel/types/references/documenttypes_refs_to_other_types.cfg +++ b/config-model/src/test/configmodel/types/references/documenttypes_refs_to_other_types.cfg @@ -1,4 +1,5 @@ ignoreundefinedfields false +enablecompression false usev8geopositions false doctype[0].name "document" doctype[0].idx 10000 diff --git a/config-model/src/test/configmodel/types/references/documenttypes_refs_to_same_type.cfg b/config-model/src/test/configmodel/types/references/documenttypes_refs_to_same_type.cfg index f681a8de9ba..f31e5f6e7c0 100644 --- a/config-model/src/test/configmodel/types/references/documenttypes_refs_to_same_type.cfg +++ b/config-model/src/test/configmodel/types/references/documenttypes_refs_to_same_type.cfg @@ -1,4 +1,5 @@ ignoreundefinedfields false +enablecompression false usev8geopositions false doctype[0].name "document" doctype[0].idx 10000 diff --git a/config-model/src/test/derived/advanced/documentmanager.cfg b/config-model/src/test/derived/advanced/documentmanager.cfg index 375d42bc63e..1ad50f57e93 100644 --- a/config-model/src/test/derived/advanced/documentmanager.cfg +++ b/config-model/src/test/derived/advanced/documentmanager.cfg @@ -1,4 +1,5 @@ ignoreundefinedfields false +enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/annotationsimplicitstruct/documentmanager.cfg b/config-model/src/test/derived/annotationsimplicitstruct/documentmanager.cfg index c5acbbf87e3..1ca66a7aea2 100644 --- a/config-model/src/test/derived/annotationsimplicitstruct/documentmanager.cfg +++ b/config-model/src/test/derived/annotationsimplicitstruct/documentmanager.cfg @@ -1,4 +1,5 @@ ignoreundefinedfields false +enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/annotationsinheritance/documentmanager.cfg b/config-model/src/test/derived/annotationsinheritance/documentmanager.cfg index edf5fd13d1b..8dc07ae8eab 100644 --- a/config-model/src/test/derived/annotationsinheritance/documentmanager.cfg +++ b/config-model/src/test/derived/annotationsinheritance/documentmanager.cfg @@ -1,4 +1,5 @@ ignoreundefinedfields false +enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/annotationsinheritance2/documentmanager.cfg b/config-model/src/test/derived/annotationsinheritance2/documentmanager.cfg index d931cae2048..67c47032995 100644 --- a/config-model/src/test/derived/annotationsinheritance2/documentmanager.cfg +++ b/config-model/src/test/derived/annotationsinheritance2/documentmanager.cfg @@ -1,4 +1,5 @@ ignoreundefinedfields false +enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/annotationspolymorphy/documentmanager.cfg b/config-model/src/test/derived/annotationspolymorphy/documentmanager.cfg index 76882afd3c5..dbe6054ce6c 100644 --- a/config-model/src/test/derived/annotationspolymorphy/documentmanager.cfg +++ b/config-model/src/test/derived/annotationspolymorphy/documentmanager.cfg @@ -1,4 +1,5 @@ ignoreundefinedfields false +enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/annotationsreference/documentmanager.cfg b/config-model/src/test/derived/annotationsreference/documentmanager.cfg index ef0166ba4a7..d27ff5c9d07 100644 --- a/config-model/src/test/derived/annotationsreference/documentmanager.cfg +++ b/config-model/src/test/derived/annotationsreference/documentmanager.cfg @@ -1,4 +1,5 @@ ignoreundefinedfields false +enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/annotationssimple/documentmanager.cfg b/config-model/src/test/derived/annotationssimple/documentmanager.cfg index a076d5a7479..a52837c5c06 100644 --- a/config-model/src/test/derived/annotationssimple/documentmanager.cfg +++ b/config-model/src/test/derived/annotationssimple/documentmanager.cfg @@ -1,4 +1,5 @@ ignoreundefinedfields false +enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/annotationsstruct/documentmanager.cfg b/config-model/src/test/derived/annotationsstruct/documentmanager.cfg index d0b527ddaf3..4ab3f376d9d 100644 --- a/config-model/src/test/derived/annotationsstruct/documentmanager.cfg +++ b/config-model/src/test/derived/annotationsstruct/documentmanager.cfg @@ -1,4 +1,5 @@ ignoreundefinedfields false +enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/annotationsstructarray/documentmanager.cfg b/config-model/src/test/derived/annotationsstructarray/documentmanager.cfg index 80792768c34..c2078dfa671 100644 --- a/config-model/src/test/derived/annotationsstructarray/documentmanager.cfg +++ b/config-model/src/test/derived/annotationsstructarray/documentmanager.cfg @@ -1,4 +1,5 @@ ignoreundefinedfields false +enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/arrays/documentmanager.cfg b/config-model/src/test/derived/arrays/documentmanager.cfg index ef3841f180b..836f3903079 100644 --- a/config-model/src/test/derived/arrays/documentmanager.cfg +++ b/config-model/src/test/derived/arrays/documentmanager.cfg @@ -1,4 +1,5 @@ ignoreundefinedfields false +enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/attributeprefetch/documentmanager.cfg b/config-model/src/test/derived/attributeprefetch/documentmanager.cfg index 5733c7d1bbd..a5c063108e4 100644 --- a/config-model/src/test/derived/attributeprefetch/documentmanager.cfg +++ b/config-model/src/test/derived/attributeprefetch/documentmanager.cfg @@ -1,4 +1,5 @@ ignoreundefinedfields false +enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/complex/documentmanager.cfg b/config-model/src/test/derived/complex/documentmanager.cfg index f44ab48c255..dd0b7095d93 100644 --- a/config-model/src/test/derived/complex/documentmanager.cfg +++ b/config-model/src/test/derived/complex/documentmanager.cfg @@ -1,4 +1,5 @@ ignoreundefinedfields false +enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/declstruct/documentmanager.cfg b/config-model/src/test/derived/declstruct/documentmanager.cfg index 7b95c78d584..992d210dbe2 100644 --- a/config-model/src/test/derived/declstruct/documentmanager.cfg +++ b/config-model/src/test/derived/declstruct/documentmanager.cfg @@ -1,4 +1,5 @@ ignoreundefinedfields false +enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/duplicate_struct/documentmanager.cfg b/config-model/src/test/derived/duplicate_struct/documentmanager.cfg index af20f058eed..4742a75205a 100644 --- a/config-model/src/test/derived/duplicate_struct/documentmanager.cfg +++ b/config-model/src/test/derived/duplicate_struct/documentmanager.cfg @@ -1,4 +1,5 @@ ignoreundefinedfields false +enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/duplicate_struct/documenttypes.cfg b/config-model/src/test/derived/duplicate_struct/documenttypes.cfg index b751b1ebbbf..45c1bd8700f 100644 --- a/config-model/src/test/derived/duplicate_struct/documenttypes.cfg +++ b/config-model/src/test/derived/duplicate_struct/documenttypes.cfg @@ -1,4 +1,5 @@ ignoreundefinedfields false +enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/emptydefault/documentmanager.cfg b/config-model/src/test/derived/emptydefault/documentmanager.cfg index 63732728b05..bf2a39df89f 100644 --- a/config-model/src/test/derived/emptydefault/documentmanager.cfg +++ b/config-model/src/test/derived/emptydefault/documentmanager.cfg @@ -1,4 +1,5 @@ ignoreundefinedfields false +enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/id/documentmanager.cfg b/config-model/src/test/derived/id/documentmanager.cfg index 583ca8497b8..33cdbe5b996 100644 --- a/config-model/src/test/derived/id/documentmanager.cfg +++ b/config-model/src/test/derived/id/documentmanager.cfg @@ -1,4 +1,5 @@ ignoreundefinedfields false +enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/imported_fields_inherited_reference/documenttypes.cfg b/config-model/src/test/derived/imported_fields_inherited_reference/documenttypes.cfg index aa960e70326..c6cd1a2949d 100644 --- a/config-model/src/test/derived/imported_fields_inherited_reference/documenttypes.cfg +++ b/config-model/src/test/derived/imported_fields_inherited_reference/documenttypes.cfg @@ -1,4 +1,5 @@ ignoreundefinedfields false +enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/indexswitches/documentmanager.cfg b/config-model/src/test/derived/indexswitches/documentmanager.cfg index a902a5954a5..fa91cb0e554 100644 --- a/config-model/src/test/derived/indexswitches/documentmanager.cfg +++ b/config-model/src/test/derived/indexswitches/documentmanager.cfg @@ -1,4 +1,5 @@ ignoreundefinedfields false +enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/inheritance/documentmanager.cfg b/config-model/src/test/derived/inheritance/documentmanager.cfg index fd4994dbefb..52cac7bfa79 100644 --- a/config-model/src/test/derived/inheritance/documentmanager.cfg +++ b/config-model/src/test/derived/inheritance/documentmanager.cfg @@ -1,4 +1,5 @@ ignoreundefinedfields false +enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/inheritdiamond/documentmanager.cfg b/config-model/src/test/derived/inheritdiamond/documentmanager.cfg index 9f1207b77a3..7d5dacbe00f 100644 --- a/config-model/src/test/derived/inheritdiamond/documentmanager.cfg +++ b/config-model/src/test/derived/inheritdiamond/documentmanager.cfg @@ -1,4 +1,5 @@ ignoreundefinedfields false +enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/inheritfromgrandparent/documentmanager.cfg b/config-model/src/test/derived/inheritfromgrandparent/documentmanager.cfg index 68503ebd5aa..537c452c38c 100644 --- a/config-model/src/test/derived/inheritfromgrandparent/documentmanager.cfg +++ b/config-model/src/test/derived/inheritfromgrandparent/documentmanager.cfg @@ -1,4 +1,5 @@ ignoreundefinedfields false +enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/inheritfromparent/documentmanager.cfg b/config-model/src/test/derived/inheritfromparent/documentmanager.cfg index c8fdeaedf11..d246e886a3d 100644 --- a/config-model/src/test/derived/inheritfromparent/documentmanager.cfg +++ b/config-model/src/test/derived/inheritfromparent/documentmanager.cfg @@ -1,4 +1,5 @@ ignoreundefinedfields false +enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/inheritfromparent/documenttypes.cfg b/config-model/src/test/derived/inheritfromparent/documenttypes.cfg index fd9623f716a..efd8170d95e 100644 --- a/config-model/src/test/derived/inheritfromparent/documenttypes.cfg +++ b/config-model/src/test/derived/inheritfromparent/documenttypes.cfg @@ -1,4 +1,5 @@ ignoreundefinedfields false +enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/mail/onlydoc/documentmanager.cfg b/config-model/src/test/derived/mail/onlydoc/documentmanager.cfg index 6bf972c1c80..27ac015e630 100644 --- a/config-model/src/test/derived/mail/onlydoc/documentmanager.cfg +++ b/config-model/src/test/derived/mail/onlydoc/documentmanager.cfg @@ -1,3 +1,4 @@ +enablecompression false usev8geopositions false datatype[].id 1381038251 datatype[].structtype[].name "position" diff --git a/config-model/src/test/derived/multi_struct/documentmanager.cfg b/config-model/src/test/derived/multi_struct/documentmanager.cfg index 753196b4d02..e37a3dc51c6 100644 --- a/config-model/src/test/derived/multi_struct/documentmanager.cfg +++ b/config-model/src/test/derived/multi_struct/documentmanager.cfg @@ -1,4 +1,5 @@ ignoreundefinedfields false +enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/multi_struct/documenttypes.cfg b/config-model/src/test/derived/multi_struct/documenttypes.cfg index 4926675582b..93452602f86 100644 --- a/config-model/src/test/derived/multi_struct/documenttypes.cfg +++ b/config-model/src/test/derived/multi_struct/documenttypes.cfg @@ -1,4 +1,5 @@ ignoreundefinedfields false +enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/namecollision/documentmanager.cfg b/config-model/src/test/derived/namecollision/documentmanager.cfg index 79807197b48..d8cf44a9a3d 100644 --- a/config-model/src/test/derived/namecollision/documentmanager.cfg +++ b/config-model/src/test/derived/namecollision/documentmanager.cfg @@ -1,4 +1,5 @@ ignoreundefinedfields false +enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/prefixexactattribute/documentmanager.cfg b/config-model/src/test/derived/prefixexactattribute/documentmanager.cfg index 3158348d5be..d516eaf7886 100644 --- a/config-model/src/test/derived/prefixexactattribute/documentmanager.cfg +++ b/config-model/src/test/derived/prefixexactattribute/documentmanager.cfg @@ -1,4 +1,5 @@ ignoreundefinedfields false +enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/ranktypes/documentmanager.cfg b/config-model/src/test/derived/ranktypes/documentmanager.cfg index ade7795c3ca..46457fb479d 100644 --- a/config-model/src/test/derived/ranktypes/documentmanager.cfg +++ b/config-model/src/test/derived/ranktypes/documentmanager.cfg @@ -1,4 +1,5 @@ ignoreundefinedfields false +enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/reference_from_several/documentmanager.cfg b/config-model/src/test/derived/reference_from_several/documentmanager.cfg index bdcb6a04236..28f40aeee5b 100644 --- a/config-model/src/test/derived/reference_from_several/documentmanager.cfg +++ b/config-model/src/test/derived/reference_from_several/documentmanager.cfg @@ -1,4 +1,5 @@ ignoreundefinedfields false +enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/schemainheritance/documentmanager.cfg b/config-model/src/test/derived/schemainheritance/documentmanager.cfg index a706c57909e..1fe61cf2bd2 100644 --- a/config-model/src/test/derived/schemainheritance/documentmanager.cfg +++ b/config-model/src/test/derived/schemainheritance/documentmanager.cfg @@ -1,4 +1,5 @@ ignoreundefinedfields false +enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/streamingstruct/documentmanager.cfg b/config-model/src/test/derived/streamingstruct/documentmanager.cfg index c29ff0b3489..b94f23a9c7b 100644 --- a/config-model/src/test/derived/streamingstruct/documentmanager.cfg +++ b/config-model/src/test/derived/streamingstruct/documentmanager.cfg @@ -1,4 +1,5 @@ ignoreundefinedfields false +enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/structandfieldset/documentmanager.cfg b/config-model/src/test/derived/structandfieldset/documentmanager.cfg index e9bbc06d77a..e1169e3ca5d 100644 --- a/config-model/src/test/derived/structandfieldset/documentmanager.cfg +++ b/config-model/src/test/derived/structandfieldset/documentmanager.cfg @@ -1,4 +1,5 @@ ignoreundefinedfields false +enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/structanyorder/documentmanager.cfg b/config-model/src/test/derived/structanyorder/documentmanager.cfg index a1e8bb41a97..eac63515944 100644 --- a/config-model/src/test/derived/structanyorder/documentmanager.cfg +++ b/config-model/src/test/derived/structanyorder/documentmanager.cfg @@ -1,4 +1,5 @@ ignoreundefinedfields false +enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/structinheritance/documentmanager.cfg b/config-model/src/test/derived/structinheritance/documentmanager.cfg index 5897b00c07b..37240887e3b 100644 --- a/config-model/src/test/derived/structinheritance/documentmanager.cfg +++ b/config-model/src/test/derived/structinheritance/documentmanager.cfg @@ -1,4 +1,5 @@ ignoreundefinedfields false +enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/structinheritance/documenttypes.cfg b/config-model/src/test/derived/structinheritance/documenttypes.cfg index 3f9fdf85734..16521b920a7 100644 --- a/config-model/src/test/derived/structinheritance/documenttypes.cfg +++ b/config-model/src/test/derived/structinheritance/documenttypes.cfg @@ -1,4 +1,5 @@ ignoreundefinedfields false +enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/tensor/documentmanager.cfg b/config-model/src/test/derived/tensor/documentmanager.cfg index bae2db34040..f52fe073208 100644 --- a/config-model/src/test/derived/tensor/documentmanager.cfg +++ b/config-model/src/test/derived/tensor/documentmanager.cfg @@ -1,4 +1,5 @@ ignoreundefinedfields false +enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/tensor/documenttypes.cfg b/config-model/src/test/derived/tensor/documenttypes.cfg index d10ecd37c8f..3081a5bd0c4 100644 --- a/config-model/src/test/derived/tensor/documenttypes.cfg +++ b/config-model/src/test/derived/tensor/documenttypes.cfg @@ -1,4 +1,5 @@ ignoreundefinedfields false +enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/derived/types/documentmanager.cfg b/config-model/src/test/derived/types/documentmanager.cfg index ace183e1ab0..118f7d279e6 100644 --- a/config-model/src/test/derived/types/documentmanager.cfg +++ b/config-model/src/test/derived/types/documentmanager.cfg @@ -1,4 +1,5 @@ ignoreundefinedfields false +enablecompression false usev8geopositions false doctype[].name "document" doctype[].idx 10000 diff --git a/config-model/src/test/examples/fieldoftypedocument-doctypes.cfg b/config-model/src/test/examples/fieldoftypedocument-doctypes.cfg index ebc72c2d73b..2efc2f40d21 100644 --- a/config-model/src/test/examples/fieldoftypedocument-doctypes.cfg +++ b/config-model/src/test/examples/fieldoftypedocument-doctypes.cfg @@ -1,4 +1,5 @@ ignoreundefinedfields false +enablecompression false usev8geopositions false doctype[0].name "document" doctype[0].idx 10000 diff --git a/config-model/src/test/examples/fieldoftypedocument.cfg b/config-model/src/test/examples/fieldoftypedocument.cfg index 5aca758a5f4..10c66ce3e93 100644 --- a/config-model/src/test/examples/fieldoftypedocument.cfg +++ b/config-model/src/test/examples/fieldoftypedocument.cfg @@ -1,4 +1,5 @@ ignoreundefinedfields false +enablecompression false usev8geopositions false doctype[0].name "document" doctype[0].idx 10000 diff --git a/config-model/src/test/examples/structresult.cfg b/config-model/src/test/examples/structresult.cfg index c2c84eb46b0..2e3904b7110 100644 --- a/config-model/src/test/examples/structresult.cfg +++ b/config-model/src/test/examples/structresult.cfg @@ -1,4 +1,5 @@ ignoreundefinedfields false +enablecompression false usev8geopositions false doctype[0].name "document" doctype[0].idx 10000 diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/DocprocBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/DocprocBuilderTest.java index 2044fd2ab39..77681489dac 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/DocprocBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/DocprocBuilderTest.java @@ -1,12 +1,14 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container.xml; +import com.yahoo.config.docproc.DocprocConfig; import com.yahoo.config.docproc.SchemamappingConfig; import com.yahoo.config.model.builder.xml.test.DomBuilderTest; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.container.ComponentsConfig; import com.yahoo.container.core.ChainsConfig; import com.yahoo.container.jdisc.ContainerMbusConfig; +import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.search.config.QrStartConfig; import com.yahoo.vespa.model.HostPorts; import com.yahoo.vespa.model.container.ApplicationContainer; @@ -36,10 +38,12 @@ import static org.junit.Assert.assertTrue; public class DocprocBuilderTest extends DomBuilderTest { private ApplicationContainerCluster cluster; + private DocumentmanagerConfig documentmanagerConfig; private ContainerMbusConfig containerMbusConfig; private ComponentsConfig componentsConfig; private ChainsConfig chainsConfig; private SchemamappingConfig schemamappingConfig; + private DocprocConfig docprocConfig; private QrStartConfig qrStartConfig; @Before @@ -54,8 +58,10 @@ public class DocprocBuilderTest extends DomBuilderTest { chainsConfig = root.getConfig(ChainsConfig.class, cluster.getConfigId() + "/component/com.yahoo.docproc.jdisc.DocumentProcessingHandler"); + documentmanagerConfig = root.getConfig(DocumentmanagerConfig.class, cluster.getConfigId()); schemamappingConfig = root.getConfig(SchemamappingConfig.class, cluster.getContainers().get(0).getConfigId()); qrStartConfig = root.getConfig(QrStartConfig.class, cluster.getConfigId()); + docprocConfig = root.getConfig(DocprocConfig.class, cluster.getConfigId()); } private Element servicesXml() { @@ -64,7 +70,7 @@ public class DocprocBuilderTest extends DomBuilderTest { " <nodes>", " <node hostalias='mockhost' baseport='1500' />", " </nodes>", - " <document-processing preferlocalnode='true' numnodesperclient='2' maxqueuebytesize='100m' maxmessagesinqueue='300' maxqueuewait='200'>", + " <document-processing compressdocuments='true' preferlocalnode='true' numnodesperclient='2' maxqueuebytesize='100m' maxmessagesinqueue='300' maxqueuewait='200'>", " <documentprocessor id='docproc1' class='com.yahoo.Docproc1' bundle='docproc1bundle'/>", " <chain id='chein'>", " <documentprocessor id='docproc2'/>", @@ -77,6 +83,7 @@ public class DocprocBuilderTest extends DomBuilderTest { @Test public void testDocprocCluster() { assertEquals("banan", cluster.getName()); + assertTrue(cluster.getDocproc().isCompressDocuments()); //assertTrue(cluster.getContainerDocproc().isPreferLocalNode()); //assertEquals(2, cluster.getContainerDocproc().getNumNodesPerClient()); List<ApplicationContainer> services = cluster.getContainers(); @@ -98,9 +105,16 @@ public class DocprocBuilderTest extends DomBuilderTest { } @Test + public void testDocumentManagerConfig() { + assertTrue(documentmanagerConfig.enablecompression()); + } + + @Test public void testContainerMbusConfig() { + assertTrue(containerMbusConfig.enabled()); assertTrue(containerMbusConfig.port() >= HostPorts.BASE_PORT); assertEquals(300, containerMbusConfig.maxpendingcount()); + assertEquals(100, containerMbusConfig.maxpendingsize()); } @Test diff --git a/container-messagebus/src/main/resources/configdefinitions/container.jdisc.container-mbus.def b/container-messagebus/src/main/resources/configdefinitions/container.jdisc.container-mbus.def index bd43c13aba3..143e02a30ef 100644 --- a/container-messagebus/src/main/resources/configdefinitions/container.jdisc.container-mbus.def +++ b/container-messagebus/src/main/resources/configdefinitions/container.jdisc.container-mbus.def @@ -22,6 +22,10 @@ transport_events_before_wakeup int default=1 # Dynamic throttling is used, and works better than anything else. maxpendingcount int default=2048 +enabled bool default=false +#maxpendingsize is set in megabytes! +maxpendingsize int default=100 + #The amount of input data that the service can process concurrently maxConcurrentFactor double default=0.2 range=[0.0-1.0] diff --git a/document/src/vespa/document/config/documentmanager.def b/document/src/vespa/document/config/documentmanager.def index 0d0f3876f15..b1929e42d34 100644 --- a/document/src/vespa/document/config/documentmanager.def +++ b/document/src/vespa/document/config/documentmanager.def @@ -5,6 +5,9 @@ namespace=document.config ## Whether attempts to set an undefined field should be ignored rather than causing an error ignoreundefinedfields bool default=false +## Whether to enable compression in this process. +enablecompression bool default=false + ## Prefer "Vespa 8" format for the "position" type usev8geopositions bool default=false diff --git a/document/src/vespa/document/config/documenttypes.def b/document/src/vespa/document/config/documenttypes.def index 0c135db7b0d..3138e71e025 100644 --- a/document/src/vespa/document/config/documenttypes.def +++ b/document/src/vespa/document/config/documenttypes.def @@ -5,6 +5,9 @@ namespace=document.config ## Whether attempts to set an undefined field should be ignored rather than causing an error ignoreundefinedfields bool default=false +## Whether to enable compression in this process. +enablecompression bool default=false + ## Prefer "Vespa 8" format for the "position" type usev8geopositions bool default=false diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusVisitorSession.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusVisitorSession.java index 1c729008e2c..3607f652a51 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusVisitorSession.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusVisitorSession.java @@ -230,6 +230,7 @@ public class MessageBusVisitorSession implements VisitorSession { @Override public Sender createSender(ReplyHandler replyHandler, VisitorParameters visitorParameters) { + messageBus.setMaxPendingCount(0); SourceSessionParams sessionParams = createSourceSessionParams(visitorParameters); return new MessageBusSender(messageBus.createSourceSession(replyHandler, sessionParams)); } @@ -307,7 +308,7 @@ public class MessageBusVisitorSession implements VisitorSession { private static final Logger log = Logger.getLogger(MessageBusVisitorSession.class.getName()); - private static final AtomicLong sessionCounter = new AtomicLong(0); + private static AtomicLong sessionCounter = new AtomicLong(0); private static long getNextSessionId() { return sessionCounter.incrementAndGet(); } @@ -335,7 +336,7 @@ public class MessageBusVisitorSession implements VisitorSession { private boolean done = false; private boolean destroying = false; // For testing and sanity checking private final Object completionMonitor = new Object(); - private final Trace trace; + private Trace trace; /** * We keep our own track of pending messages since the sender's pending * count cannot be relied on in an async task execution context. This diff --git a/messagebus/src/main/java/com/yahoo/messagebus/MessageBus.java b/messagebus/src/main/java/com/yahoo/messagebus/MessageBus.java index 19d3b5b3e43..3bd5d5d4890 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/MessageBus.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/MessageBus.java @@ -470,7 +470,6 @@ public class MessageBus implements ConfigHandler, NetworkOwner, MessageHandler, * * @return The resender. */ - @Deprecated // Remove on 9 public Resender getResender() { return resender; } @@ -481,7 +480,6 @@ public class MessageBus implements ConfigHandler, NetworkOwner, MessageHandler, * * @return The pending count. */ - @Deprecated // Package private on 9 public synchronized int getPendingCount() { return pendingCount; } @@ -492,7 +490,6 @@ public class MessageBus implements ConfigHandler, NetworkOwner, MessageHandler, * * @return The pending size. */ - @Deprecated // Package private on 9 public synchronized int getPendingSize() { return pendingSize; } @@ -503,7 +500,6 @@ public class MessageBus implements ConfigHandler, NetworkOwner, MessageHandler, * * @param maxCount The max count. */ - @Deprecated // Remove on 9 public void setMaxPendingCount(int maxCount) { maxPendingCount = maxCount; } @@ -512,7 +508,6 @@ public class MessageBus implements ConfigHandler, NetworkOwner, MessageHandler, * Gets maximum number of messages that can be received without being * replied to yet. */ - @Deprecated // Remove on 9 public int getMaxPendingCount() { return maxPendingCount; } @@ -523,7 +518,6 @@ public class MessageBus implements ConfigHandler, NetworkOwner, MessageHandler, * * @param maxSize The max size. */ - @Deprecated // Remove on 9 public void setMaxPendingSize(int maxSize) { maxPendingSize = maxSize; } @@ -532,7 +526,6 @@ public class MessageBus implements ConfigHandler, NetworkOwner, MessageHandler, * Gets maximum combined size of messages that can be received without * being replied to yet. */ - @Deprecated // Remove on 9 public int getMaxPendingSize() { return maxPendingSize; } diff --git a/messagebus/src/test/java/com/yahoo/messagebus/ChokeTestCase.java b/messagebus/src/test/java/com/yahoo/messagebus/ChokeTestCase.java index 6d50fc769be..60ec6c400a9 100755 --- a/messagebus/src/test/java/com/yahoo/messagebus/ChokeTestCase.java +++ b/messagebus/src/test/java/com/yahoo/messagebus/ChokeTestCase.java @@ -14,6 +14,7 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; +import java.net.UnknownHostException; import java.util.ArrayList; import java.util.List; import static org.junit.Assert.assertEquals; @@ -54,7 +55,6 @@ public class ChokeTestCase { } @Test - @SuppressWarnings("deprecation") public void testMaxCount() { int max = 10; dstServer.mb.setMaxPendingCount(max); @@ -103,7 +103,6 @@ public class ChokeTestCase { } @Test - @SuppressWarnings("deprecation") public void testMaxSize() { int size = createMessage("msg").getApproxSize(); int max = size * 10; diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerTest.java index 2fc34112517..f3ea8fb5a80 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerTest.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerTest.java @@ -25,7 +25,7 @@ public class FeedHandlerTest { FeedHandler handler = new FeedHandler( new RejectingContainerThreadpool(), new CollectingMetric(), - new DocumentTypeManager(new DocumentmanagerConfig.Builder().build()), + new DocumentTypeManager(new DocumentmanagerConfig.Builder().enablecompression(true).build()), null /* session cache */, MetricReceiver.nullImplementation); var responseHandler = new RequestHandlerTestDriver.MockResponseHandler(); diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerV3Test.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerV3Test.java index 5b8b5b1827f..dcabc1f338e 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerV3Test.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerV3Test.java @@ -108,7 +108,7 @@ public class FeedHandlerV3Test { } private FeedHandlerV3 setupFeederHandler(Executor threadPool) { - DocumentTypeManager docMan = new DocumentTypeManager(new DocumentmanagerConfig.Builder().build()); + DocumentTypeManager docMan = new DocumentTypeManager(new DocumentmanagerConfig.Builder().enablecompression(true).build()); FeedHandlerV3 feedHandlerV3 = new FeedHandlerV3( threadPool, metric, |