diff options
25 files changed, 3 insertions, 1118 deletions
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomV20ClientsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomV20ClientsBuilder.java index 6a02619bdb4..74e3ac581c4 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomV20ClientsBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomV20ClientsBuilder.java @@ -2,26 +2,11 @@ package com.yahoo.vespa.model.builder.xml.dom; import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.vespa.config.content.spooler.SpoolerConfig; -import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.text.XML; -import com.yahoo.vespa.defaults.Defaults; -import com.yahoo.vespa.model.SimpleConfigProducer; -import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder.DomConfigProducerBuilder; import com.yahoo.vespa.model.clients.Clients; -import com.yahoo.vespa.model.clients.VespaSpoolMaster; -import com.yahoo.vespa.model.clients.VespaSpooler; -import com.yahoo.vespa.model.clients.VespaSpoolerProducer; -import com.yahoo.vespa.model.clients.VespaSpoolerService; -import com.yahoo.vespaclient.config.FeederConfig; import org.w3c.dom.Element; -import org.w3c.dom.Node; import org.w3c.dom.NodeList; -import java.io.Serializable; -import java.util.ArrayList; -import java.util.List; - /** * Builds the Clients plugin * @@ -38,13 +23,8 @@ public class DomV20ClientsBuilder { this.clients = clients; } - public void build(DeployState deployState, Element spec) { - NodeList children = spec.getElementsByTagName("spoolers"); - for (int i = 0; i < children.getLength(); i++) { - createSpoolers(deployState, clients.getConfigProducer(), (Element) children.item(i), clients); - } - - children = spec.getElementsByTagName("load-types"); + public void build(Element spec) { + NodeList children = spec.getElementsByTagName("load-types"); for (int i = 0; i < children.getLength(); i++) { createLoadTypes((Element) children.item(i), clients); } @@ -57,283 +37,4 @@ public class DomV20ClientsBuilder { } } - /** - * Creates VespaSpooler objects using the given xml Element. - */ - private void createSpoolers(DeployState deployState, AbstractConfigProducer pcp, Element element, Clients clients) { - String jvmArgs = null; - if (element.hasAttribute(VespaDomBuilder.JVMARGS_ATTRIB_NAME)) jvmArgs=element.getAttribute(VespaDomBuilder.JVMARGS_ATTRIB_NAME); - SimpleConfigProducer spoolerCfg = new VespaDomBuilder.DomSimpleConfigProducerBuilder(element.getNodeName()). - build(deployState, pcp, element); - Element spoolersFeederOptions = findFeederOptions(element); - createSpoolMasters(deployState, spoolerCfg, element); - for (Element e : XML.getChildren(element, "spooler")) { - String configId = e.getAttribute("id").trim(); - FeederConfig.Builder feederConfig = getFeederConfig(spoolersFeederOptions, e); - SpoolerConfig.Builder spoolConfig = getSpoolConfig(e); - if (configId.length() == 0) { - int index = clients.getVespaSpoolers().size(); - VespaSpoolerService spoolerService = new VespaSpoolerServiceBuilder(index, new VespaSpooler(feederConfig, spoolConfig)). - build(deployState, spoolerCfg, e); - if ("".equals(spoolerService.getJvmOptions()) && jvmArgs!=null) spoolerService.setJvmOptions(jvmArgs); - spoolerService.setProp("index", String.valueOf(index)); - clients.getVespaSpoolers().add(spoolerService); - } else { - new VespaSpoolerProducerBuilder(configId, new VespaSpooler(feederConfig, spoolConfig)). - build(deployState, spoolerCfg, e); - } - } - } - - private void createSpoolMasters(DeployState deployState, SimpleConfigProducer producer, Element element) { - int i=0; - for (Element e : XML.getChildren(element, "spoolmaster")) - new VespaSpoolMasterBuilder(i++).build(deployState, producer, e); - } - - private SpoolerConfig.Builder getSpoolConfig(Element conf) { - SpoolerConfig.Builder builder = new SpoolerConfig.Builder(); - if (conf.getAttributes().getNamedItem("directory") != null) { - builder.directory(Defaults.getDefaults().underVespaHome(conf.getAttributes().getNamedItem("directory").getNodeValue())); - } - if (conf.getAttributes().getNamedItem("keepsuccess") != null) { - builder.keepsuccess(getBooleanFromAttribute(conf, "keepsuccess")); - } - if (conf.getAttributes().getNamedItem("maxfailuresize") != null) { - builder.maxfailuresize(getIntegerFromAttribute(conf, "maxfailuresize")); - } - if (conf.getAttributes().getNamedItem("maxfatalfailuresize") != null) { - builder.maxfatalfailuresize(getIntegerFromAttribute(conf, "maxfatalfailuresize")); - } - if (conf.getAttributes().getNamedItem("threads") != null) { - builder.threads(getIntegerFromAttribute(conf, "threads")); - } - if (conf.getAttributes().getNamedItem("maxretries") != null) { - builder.maxretries(getIntegerFromAttribute(conf, "maxretries")); - } - - NodeList children = conf.getElementsByTagName("parsers"); - if (children.getLength() == 1) { - children = ((Element)children.item(0)).getElementsByTagName("parser"); - } - - for (int i=0; i < children.getLength(); i++) { - Element e = (Element)children.item(i); - - String type = e.getAttributes().getNamedItem("type").getNodeValue(); - NodeList params = e.getElementsByTagName("parameter"); - - SpoolerConfig.Parsers.Builder parserBuilder = new SpoolerConfig.Parsers.Builder(); - parserBuilder.classname(type); - if (params.getLength() > 0) { - List<SpoolerConfig.Parsers.Parameters.Builder> parametersBuilders = new ArrayList<>(); - for (int j = 0; j < params.getLength(); j++) { - SpoolerConfig.Parsers.Parameters.Builder parametersBuilder = new SpoolerConfig.Parsers.Parameters.Builder(); - Element p = (Element) params.item(j); - parametersBuilder.key(getStringFromAttribute(p, "key")); - parametersBuilder.value(getStringFromAttribute(p, "value")); - parametersBuilders.add(parametersBuilder); - } - parserBuilder.parameters(parametersBuilders); - } - - builder.parsers.add(parserBuilder); - } - return builder; - } - - Boolean getBooleanFromAttribute(Element e, String attributeName) { - return Boolean.parseBoolean(e.getAttributes().getNamedItem(attributeName).getNodeValue()); - } - - Integer getIntegerFromAttribute(Element e, String attributeName) { - return Integer.parseInt(e.getAttributes().getNamedItem(attributeName).getNodeValue()); - } - - String getStringFromAttribute(Element e, String attributeName) { - return e.getAttributes().getNamedItem(attributeName).getNodeValue(); - } - - private FeederConfig.Builder getFeederConfig(Element gatewaysFeederOptions, Element e) { - FeederOptionsParser foParser = new FeederOptionsParser(); - if (gatewaysFeederOptions!=null) { - foParser.parseFeederOptions(gatewaysFeederOptions).getFeederConfig(); - } - foParser.parseFeederOptions(e); - return foParser.getFeederConfig(); - } - - /** - * Finds the feederoptions subelement in the given xml Element. - * - * @param element The xml Element - * @return The feederoptions xml Element - */ - private Element findFeederOptions(Element element) { - for (Element child : XML.getChildren(element)) { - if (child.getNodeName().equals("feederoptions")) { - return child; - } - } - return null; - } - - private static class VespaSpoolerServiceBuilder extends DomConfigProducerBuilder<VespaSpoolerService> { - private int index; - private VespaSpooler spoolerConfig; - - public VespaSpoolerServiceBuilder(int index, VespaSpooler spoolerConfig) { - this.index = index; - this.spoolerConfig = spoolerConfig; - } - - @Override - protected VespaSpoolerService doBuild(DeployState deployState, AbstractConfigProducer parent, Element spec) { - return new VespaSpoolerService(parent, index, spoolerConfig); - } - } - - private static class VespaSpoolerProducerBuilder extends DomConfigProducerBuilder<VespaSpoolerProducer> { - private String name=null; - private VespaSpooler spooler; - - public VespaSpoolerProducerBuilder(String name, VespaSpooler spooler) { - this.name = name; - this.spooler = spooler; - } - - @Override - protected VespaSpoolerProducer doBuild(DeployState deployState, AbstractConfigProducer parent, Element producerSpec) { - return new VespaSpoolerProducer(parent, name, spooler); - } - } - - private static class VespaSpoolMasterBuilder extends DomConfigProducerBuilder<VespaSpoolMaster> { - int index; - - public VespaSpoolMasterBuilder(int index) { - super(); - this.index = index; - } - - @Override - protected VespaSpoolMaster doBuild(DeployState deployState, AbstractConfigProducer parent, Element spec) { - return new VespaSpoolMaster(parent, index); - } - } - - /** - * This class parses the feederoptions xml tag and produces Vespa config output. - * - * @author Gunnar Gauslaa Bergem - */ - private class FeederOptionsParser implements Serializable { - private static final long serialVersionUID = 1L; - // All member variables are objects so that we can switch on null values. - private Boolean abortondocumenterror = null; - private String route = null; - private Integer maxpendingdocs = null; - private Integer maxpendingbytes = null; - private Boolean retryenabled = null; - private Double retrydelay = null; - private Double timeout = null; - private Integer tracelevel = null; - private Integer mbusport = null; - private String docprocChain = null; - - /** - * Constructs an empty feeder options object with all members set to null. - */ - public FeederOptionsParser() { - // empty - } - - /** - * Parses the content of the given XML element as feeder options. - * - * @param conf The XML element to parse. - */ - public FeederOptionsParser parseFeederOptions(Element conf) { - for (Node node : XML.getChildren(conf)) { - String nodename = node.getNodeName(); - Node firstchild = node.getFirstChild(); - String childval = (firstchild != null) ? firstchild.getNodeValue() : null; - - switch (nodename) { - case "abortondocumenterror": - abortondocumenterror = Boolean.valueOf(childval); - break; - case "maxpendingdocs": - maxpendingdocs = Integer.valueOf(childval); - break; - case "maxpendingbytes": - maxpendingbytes = Integer.valueOf(childval); - break; - case "retryenabled": - retryenabled = Boolean.valueOf(childval); - break; - case "retrydelay": - retrydelay = Double.valueOf(childval); - break; - case "timeout": - timeout = Double.valueOf(childval); - break; - case "route": - route = childval; - break; - case "tracelevel": - tracelevel = Integer.valueOf(childval); - break; - case "mbusport": - mbusport = Integer.valueOf(childval); - break; - case "docprocchain": - docprocChain = childval; - break; - } - } - return this; - } - - /** - * Returns a feeder options config string of the content of this. - * - * @return A config string. - */ - public FeederConfig.Builder getFeederConfig() { - FeederConfig.Builder builder = new FeederConfig.Builder(); - if (abortondocumenterror != null) { - builder.abortondocumenterror(abortondocumenterror); - } - if (route != null && route.length() > 0) { - builder.route(route); - } - if (maxpendingdocs != null) { - builder.maxpendingdocs(maxpendingdocs); - } - if (maxpendingbytes != null) { - builder.maxpendingbytes(maxpendingbytes); - } - if (retryenabled != null) { - builder.retryenabled(retryenabled); - } - if (retrydelay != null) { - builder.retrydelay(retrydelay); - } - if (timeout != null) { - builder.timeout(timeout); - } - if (tracelevel != null) { - builder.tracelevel(tracelevel); - } - if (mbusport != null) { - builder.mbusport(mbusport); - } - if (docprocChain != null && docprocChain.length() > 0) { - builder.docprocchain(docprocChain); - } - return builder; - } - } - } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/clients/Clients.java b/config-model/src/main/java/com/yahoo/vespa/model/clients/Clients.java index 75c10ea2667..9b33568b61e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/clients/Clients.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/clients/Clients.java @@ -22,23 +22,10 @@ import java.util.List; public class Clients extends ConfigModel { private static final long serialVersionUID = 1L; - private List<VespaSpoolerService> vespaSpoolers = new LinkedList<>(); private LoadTypeSet loadTypes = new LoadTypeSet(); - private final AbstractConfigProducer parent; - public Clients(ConfigModelContext modelContext) { super(modelContext); - this.parent = modelContext.getParentProducer(); - } - - public AbstractConfigProducer getConfigProducer() { - return parent; - } - - /** Returns the mutable list of spoolers which is then used to add spoolers */ - public List<VespaSpoolerService> getVespaSpoolers() { - return vespaSpoolers; } public LoadTypeSet getLoadTypes() { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/clients/VespaSpoolMaster.java b/config-model/src/main/java/com/yahoo/vespa/model/clients/VespaSpoolMaster.java deleted file mode 100644 index 7013a09009a..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/clients/VespaSpoolMaster.java +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.clients; - -import com.yahoo.config.model.producer.AbstractConfigProducer; -import com.yahoo.vespa.model.AbstractService; - -/** - * The spoolmaster program, which is used when multiple spooler instances are used to provide - * multi colo HTTP feeding. - * @author vegardh - * - */ -public class VespaSpoolMaster extends AbstractService { - - public VespaSpoolMaster(AbstractConfigProducer parent, int index) { - super(parent, "spoolmaster."+index); - } - - @Override - public int getPortCount() { - // TODO Auto-generated method stub - return 0; - } - - @Override - public String getStartupCommand() { - return "exec vespa-spoolmaster"; - } -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/clients/VespaSpooler.java b/config-model/src/main/java/com/yahoo/vespa/model/clients/VespaSpooler.java deleted file mode 100644 index c750a5d2a89..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/clients/VespaSpooler.java +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.clients; - -import com.yahoo.vespa.config.content.spooler.SpoolerConfig; -import com.yahoo.config.subscription.ConfigInstanceUtil; -import com.yahoo.vespaclient.config.FeederConfig; - -/** - * Holds configuration for VespaSpoolers. Actual services use VespaSpoolerService, - * while virtual services can be generated for external spoolers (VespaSpoolerProducer). - * - * @author <a href="mailto:thomasg@yahoo-inc.com">Gunnar Gauslaa Bergem</a> - * @author Vidar Larsen - */ -public class VespaSpooler { - private final SpoolerConfig.Builder spoolConfig; - private final FeederConfig.Builder feederConfig; - - public VespaSpooler(FeederConfig.Builder feederConfig, SpoolerConfig.Builder spoolConfig) { - this.feederConfig = feederConfig; - this.spoolConfig = spoolConfig; - } - - public void getConfig(SpoolerConfig.Builder builder) { - ConfigInstanceUtil.setValues(builder, spoolConfig); - } - - public void getConfig(FeederConfig.Builder builder) { - ConfigInstanceUtil.setValues(builder, feederConfig); - } -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/clients/VespaSpoolerProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/clients/VespaSpoolerProducer.java deleted file mode 100644 index 3fae5389a94..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/clients/VespaSpoolerProducer.java +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.clients; - -import com.yahoo.vespa.config.content.spooler.SpoolerConfig; -import com.yahoo.config.model.producer.AbstractConfigProducer; -import com.yahoo.vespaclient.config.FeederConfig; - -/** - * This model represents a config producer for spooler used for feeding documents to Vespa. - * - * @author <a href="mailto:gunnarga@yahoo-inc.com">Gunnar Gauslaa Bergem</a> - * @author Vidar Larsen - */ -public class VespaSpoolerProducer extends AbstractConfigProducer implements SpoolerConfig.Producer, FeederConfig.Producer { - private static final long serialVersionUID = 1L; - private VespaSpooler spoolerConfig; - - public VespaSpoolerProducer(AbstractConfigProducer parent, String configId, VespaSpooler spooler) { - super(parent, configId); - spoolerConfig = spooler; - } - - @Override - public void getConfig(SpoolerConfig.Builder builder) { - spoolerConfig.getConfig(builder); - } - - @Override - public void getConfig(FeederConfig.Builder builder) { - spoolerConfig.getConfig(builder); - } -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/clients/VespaSpoolerService.java b/config-model/src/main/java/com/yahoo/vespa/model/clients/VespaSpoolerService.java deleted file mode 100644 index 378c85dc325..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/clients/VespaSpoolerService.java +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.clients; - -import com.yahoo.vespa.config.content.spooler.SpoolerConfig; -import com.yahoo.config.model.producer.AbstractConfigProducer; -import com.yahoo.vespa.model.AbstractService; -import com.yahoo.vespaclient.config.FeederConfig; - -/** - * This model represents a spooler used for feeding documents to Vespa. - * - * @author <a href="mailto:gunnarga@yahoo-inc.com">Gunnar Gauslaa Bergem</a> - * @author Vidar Larsen - */ -public class VespaSpoolerService extends AbstractService implements SpoolerConfig.Producer, FeederConfig.Producer { - private static final long serialVersionUID = 1L; - private VespaSpooler spooler; - - public VespaSpoolerService(AbstractConfigProducer parent, int index, VespaSpooler spooler) { - super(parent, "spooler." + index); - this.spooler = spooler; - } - - public int getPortCount() { - return 0; - } - - public String getStartupCommand() { - return "exec vespaspooler "+ getJvmOptions(); - } - - @Override - public void getConfig(SpoolerConfig.Builder builder) { - spooler.getConfig(builder); - } - - @Override - public void getConfig(FeederConfig.Builder builder) { - spooler.getConfig(builder); - } -} diff --git a/config-model/src/main/resources/schema/clients-v2.rnc b/config-model/src/main/resources/schema/clients-v2.rnc index 1076d332dd5..5e50135edee 100644 --- a/config-model/src/main/resources/schema/clients-v2.rnc +++ b/config-model/src/main/resources/schema/clients-v2.rnc @@ -1,8 +1,7 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. Clients20 = attribute version { "2.0" } & - LoadTypes? & - Spoolers20? + LoadTypes? LoadTypes = element load-types { element type { @@ -11,38 +10,6 @@ LoadTypes = element load-types { }* } -Spoolers20 = element spoolers { - attribute jvmargs { text }? & - SpoolMaster20* & - FeederOptions20? & - Spooler20+ -} - -Spooler20 = element spooler { - service.attlist? & - attribute id { xsd:string }? & - attribute directory { xsd:string }? & - attribute keepsuccess { xsd:boolean }? & - attribute maxfailuresize { xsd:integer }? & - attribute maxfatalfailuresize { xsd:integer }? & - attribute threads { xsd:integer }? & - attribute maxretries { xsd:integer }? & - FeederOptionsOpts20? & - element parsers { - element parser { - attribute type { text } & - element parameter { - attribute key { xsd:string { minLength = "1" } } & - attribute value { xsd:string { minLength = "1" } } - }* - }+ - }? -} - -SpoolMaster20 = element spoolmaster { - service.attlist? -} - FeederOptions20 = element feederoptions { FeederOptionsOpts20 & DocProcChain? diff --git a/config-model/src/test/cfg/clients/advancedconfig.v2/hosts.xml b/config-model/src/test/cfg/clients/advancedconfig.v2/hosts.xml deleted file mode 100644 index 9eafb2106cb..00000000000 --- a/config-model/src/test/cfg/clients/advancedconfig.v2/hosts.xml +++ /dev/null @@ -1,7 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<hosts> - <host name="localhost"> - <alias>node1</alias> - </host> -</hosts> diff --git a/config-model/src/test/cfg/clients/advancedconfig.v2/searchdefinitions/music.sd b/config-model/src/test/cfg/clients/advancedconfig.v2/searchdefinitions/music.sd deleted file mode 100644 index da9e2d14c55..00000000000 --- a/config-model/src/test/cfg/clients/advancedconfig.v2/searchdefinitions/music.sd +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -search music { - document music { - field f1 type string { - indexing: summary | index - # index-to: f1, all - } - field f2 type string { - indexing: summary | index - # index-to: f2, all - } - } -} diff --git a/config-model/src/test/cfg/clients/advancedconfig.v2/services.xml b/config-model/src/test/cfg/clients/advancedconfig.v2/services.xml deleted file mode 100644 index e6102b6f8fd..00000000000 --- a/config-model/src/test/cfg/clients/advancedconfig.v2/services.xml +++ /dev/null @@ -1,70 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<services xmlns=""> - <admin version="2.0"> - <adminserver hostalias="node1" /> - <logserver hostalias="node1" /> - <slobroks> - <slobrok hostalias="node1" /> - </slobroks> - </admin> - - <clients version="2.0"> - <spoolers> - <feederoptions> - <timeout>90</timeout> - </feederoptions> - - <spooler hostalias="node1" keepsuccess="true"> - <abortondocumenterror>false</abortondocumenterror> - <maxpendingbytes>8000</maxpendingbytes> - <parsers> - <parser type="com.yahoo.vespaspooler.XMLFileParser"/> - <parser type="com.yahoo.vespaspooler.MusicFileParser"/> - <parser type="com.yahoo.vespaspooler.MusicParser"> - <parameter key="route" value="default"/> - </parser> - </parsers> - </spooler> - - <spooler hostalias="node1"> - <abortondocumenterror>false</abortondocumenterror> - <maxpendingbytes>4000</maxpendingbytes> - <timeout>50</timeout> - <parsers> - <parser type="com.yahoo.vespaspooler.MusicParser"> - <parameter key="route" value="othercluster"/> - </parser> - </parsers> - </spooler> - - <spooler id="plan9"> - <route>myroute</route> - <mbusport>14064</mbusport> - - <parsers> - <parser type="com.yahoo.vespaspooler.MusicFileParser" /> - </parsers> - </spooler> - - </spoolers> - </clients> - - <container version="1.0"> - <search/> - <nodes> - <node hostalias="node1"/> - </nodes> - </container> - - <content version="1.0" id="music"> - <redundancy>2</redundancy> - <documents> - <document type="music" mode="index"/> - </documents> - <nodes> - <node hostalias="node1" distribution-key="0" /> - </nodes> - </content> - -</services> diff --git a/config-model/src/test/cfg/clients/simpleconfig.v2.docprocv3/hosts.xml b/config-model/src/test/cfg/clients/simpleconfig.v2.docprocv3/hosts.xml deleted file mode 100644 index 9eafb2106cb..00000000000 --- a/config-model/src/test/cfg/clients/simpleconfig.v2.docprocv3/hosts.xml +++ /dev/null @@ -1,7 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<hosts> - <host name="localhost"> - <alias>node1</alias> - </host> -</hosts> diff --git a/config-model/src/test/cfg/clients/simpleconfig.v2.docprocv3/searchdefinitions/music.sd b/config-model/src/test/cfg/clients/simpleconfig.v2.docprocv3/searchdefinitions/music.sd deleted file mode 100644 index da9e2d14c55..00000000000 --- a/config-model/src/test/cfg/clients/simpleconfig.v2.docprocv3/searchdefinitions/music.sd +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -search music { - document music { - field f1 type string { - indexing: summary | index - # index-to: f1, all - } - field f2 type string { - indexing: summary | index - # index-to: f2, all - } - } -} diff --git a/config-model/src/test/cfg/clients/simpleconfig.v2.docprocv3/services.xml b/config-model/src/test/cfg/clients/simpleconfig.v2.docprocv3/services.xml deleted file mode 100644 index ded66571026..00000000000 --- a/config-model/src/test/cfg/clients/simpleconfig.v2.docprocv3/services.xml +++ /dev/null @@ -1,63 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<services> - - <admin version="2.0"> - <adminserver hostalias="node1" /> - <logserver hostalias="node1" /> - <slobroks> - <slobrok hostalias="node1" /> - </slobroks> - </admin> - - <clients version="2.0"> - <spoolers> - <spooler hostalias="node1" maxfailuresize="100000" maxfatalfailuresize="1000000" threads="5"> - <abortondocumenterror>false</abortondocumenterror> - <maxpendingbytes>8000</maxpendingbytes> - <tracelevel>7</tracelevel> - <parsers> - <parser type="com.yahoo.vespaspooler.XMLFileParser"/> - <parser type="com.yahoo.vespaspooler.MusicFileParser"/> - <parser type="com.yahoo.vespaspooler.MusicParser"> - <parameter key="route" value="default"/> - <parameter key="foo" value="bar"/> - </parser> - </parsers> - </spooler> - </spoolers> - </clients> - - <container version="1.0"> - - <nodes> - <node hostalias="node1"/> - </nodes> - - <document-api/> - - <document-processing> - <chain id="main"> - <documentprocessor id="com.yahoo.docprocs.FoobarDocumentProcessor" /> - </chain> - </document-processing> - - <search/> - - </container> - - <content version="1.0" id="music"> - - <redundancy>2</redundancy> - - <documents> - <document type="music" mode="index"/> - </documents> - - <nodes> - <node hostalias="node1" distribution-key="0" /> - </nodes> - - </content> - -</services> diff --git a/config-model/src/test/cfg/clients/simpleconfig.v2/searchdefinitions/.gitignore b/config-model/src/test/cfg/clients/simpleconfig.v2/searchdefinitions/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 --- a/config-model/src/test/cfg/clients/simpleconfig.v2/searchdefinitions/.gitignore +++ /dev/null diff --git a/config-model/src/test/java/com/yahoo/vespa/model/clients/test/SpoolerTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/clients/test/SpoolerTestCase.java deleted file mode 100644 index 7e0c31dd998..00000000000 --- a/config-model/src/test/java/com/yahoo/vespa/model/clients/test/SpoolerTestCase.java +++ /dev/null @@ -1,145 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.clients.test; - -import com.yahoo.vespa.config.content.spooler.SpoolerConfig; -import com.yahoo.vespa.model.VespaModel; -import com.yahoo.vespa.model.test.utils.CommonVespaModelSetup; -import com.yahoo.vespaclient.config.FeederConfig; -import org.junit.Test; - -import java.util.*; - -import static org.junit.Assert.assertEquals; - -/** - * @author Thomas Gundersen - */ -public class SpoolerTestCase { - - @Test - public void testSimple() throws Exception { - VespaModel model = createModel("src/test/cfg/clients/simpleconfig.v2.docprocv3"); - - SpoolerConfig.Builder builder = new SpoolerConfig.Builder(); - SpoolerConfig.Parsers.Builder parserBuilder1 = createParserBuilder("com.yahoo.vespaspooler.XMLFileParser"); - SpoolerConfig.Parsers.Builder parserBuilder2 = createParserBuilder("com.yahoo.vespaspooler.MusicFileParser"); - LinkedHashMap<String, String> parameters = new LinkedHashMap<>(); - parameters.put("route", "default"); - parameters.put("foo", "bar"); - SpoolerConfig.Parsers.Builder parserBuilder3 = createParserBuilder("com.yahoo.vespaspooler.MusicParser", - parameters); - builder.maxfailuresize(100000). - maxfatalfailuresize(1000000). - threads(5). - parsers(Arrays.asList(parserBuilder1, parserBuilder2, parserBuilder3)); - final int spoolerIndex = 0; - testSpoolerConfigBuilder(model, spoolerIndex, builder); - - FeederConfig.Builder feederBuilder = new FeederConfig.Builder(). - abortondocumenterror(false). - maxpendingbytes(8000). - tracelevel(7); - testFeederConfigBuilder(model, spoolerIndex, feederBuilder); - } - - @Test - public void testAdvanced() throws Exception { - VespaModel model = createModel("src/test/cfg/clients/advancedconfig.v2"); - - SpoolerConfig.Builder builder = new SpoolerConfig.Builder(); - SpoolerConfig.Parsers.Builder parserBuilder1 = createParserBuilder("com.yahoo.vespaspooler.XMLFileParser"); - SpoolerConfig.Parsers.Builder parserBuilder2 = createParserBuilder("com.yahoo.vespaspooler.MusicFileParser"); - LinkedHashMap<String, String> parameters = new LinkedHashMap<>(); - parameters.put("route", "default"); - SpoolerConfig.Parsers.Builder parserBuilder3 = createParserBuilder("com.yahoo.vespaspooler.MusicParser", - parameters); - builder.keepsuccess(true). - parsers(Arrays.asList(parserBuilder1, parserBuilder2, parserBuilder3)); - int spoolerIndex = 0; - testSpoolerConfigBuilder(model, spoolerIndex, builder); - - FeederConfig.Builder feederBuilder = new FeederConfig.Builder(). - abortondocumenterror(false). - maxpendingbytes(8000). - timeout(90.0); - testFeederConfigBuilder(model, spoolerIndex, feederBuilder); - - builder = new SpoolerConfig.Builder(); - parameters = new LinkedHashMap<>(); - parameters.put("route", "othercluster"); - - parserBuilder1 = createParserBuilder("com.yahoo.vespaspooler.MusicParser", - parameters); - builder.keepsuccess(false). - parsers(parserBuilder1); - spoolerIndex = 1; - testSpoolerConfigBuilder(model, spoolerIndex, builder); - - feederBuilder = new FeederConfig.Builder(). - abortondocumenterror(false). - maxpendingbytes(4000). - timeout(50.0); - testFeederConfigBuilder(model, spoolerIndex, feederBuilder); - - builder = new SpoolerConfig.Builder(); - parserBuilder1 = new SpoolerConfig.Parsers.Builder(); - parserBuilder1.classname("com.yahoo.vespaspooler.MusicFileParser"); - builder.parsers(parserBuilder1); - String id = "plan9"; - testSpoolerConfigBuilder(model, "clients/spoolers/" + id, builder); - - feederBuilder = new FeederConfig.Builder(). - route("myroute"). - mbusport(14064). - timeout(90.0); - testFeederConfigBuilder(model, "clients/spoolers/" + id, feederBuilder); - } - - SpoolerConfig.Parsers.Builder createParserBuilder(String className) { - return createParserBuilder(className, new HashMap<String, String>()); - } - - SpoolerConfig.Parsers.Builder createParserBuilder(String className, Map<String, String> parameters) { - SpoolerConfig.Parsers.Builder builder = new SpoolerConfig.Parsers.Builder(); - builder.classname(className); - if (!parameters.isEmpty()) { - List<SpoolerConfig.Parsers.Parameters.Builder> parametersBuilders = new ArrayList<>(); - for (Map.Entry<String, String> entry : parameters.entrySet()) { - final SpoolerConfig.Parsers.Parameters.Builder parametersBuilder = new SpoolerConfig.Parsers.Parameters.Builder(); - parametersBuilder.key(entry.getKey()).value(entry.getValue()); - parametersBuilders.add(parametersBuilder); - } - builder.parameters(parametersBuilders); - } - return builder; - } - - private void testSpoolerConfigBuilder(VespaModel model, int index, SpoolerConfig.Builder expected) throws Exception { - testSpoolerConfigBuilder(model, "clients/spoolers/spooler." + index, expected); - } - - private void testSpoolerConfigBuilder(VespaModel model, String id, SpoolerConfig.Builder expected) throws Exception { - SpoolerConfig.Builder b = new SpoolerConfig.Builder(); - model.getConfig(b, id); - SpoolerConfig config = new SpoolerConfig(b); - SpoolerConfig expectedConfig = new SpoolerConfig(expected); - assertEquals(expectedConfig, config); - } - - private void testFeederConfigBuilder(VespaModel model, int index, FeederConfig.Builder expected) throws Exception { - testFeederConfigBuilder(model, "clients/spoolers/spooler." + index, expected); - } - - private void testFeederConfigBuilder(VespaModel model, String id, FeederConfig.Builder expected) throws Exception { - FeederConfig.Builder b = new FeederConfig.Builder(); - model.getConfig(b, id); - FeederConfig config = new FeederConfig(b); - FeederConfig expectedConfig = new FeederConfig(expected); - assertEquals(expectedConfig, config); - } - - private VespaModel createModel(String configFile) throws Exception { - return CommonVespaModelSetup.createVespaModelWithMusic(configFile); - } - -} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/configdefs/spooler.def b/configserver/src/test/java/com/yahoo/vespa/config/server/configdefs/spooler.def deleted file mode 100644 index 2721af7e278..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/configdefs/spooler.def +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -# Which directory to find spool files in. -directory string default="/home/vespa/var/spool/vespa" - -# If true, move successfully processed files to <directory>/success -keepsuccess bool default=false - -# Trace level on error messages from messagebus -tracelevel int default=5 - -# Which parsers to use and config for each of them. -parsers[].classname string -parsers[].parameters[].key string -parsers[].parameters[].value string diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/spooler.clients-spooler-spooler.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/spooler.clients-spooler-spooler.cfg deleted file mode 100644 index 038d655e83c..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/spooler.clients-spooler-spooler.cfg +++ /dev/null @@ -1,13 +0,0 @@ -directory /home/vespa/var/spool/vespa/ -keepsuccess false -parsers[4] -parsers[0].classname com.yahoo.vespaspooler.XMLFileParser -parsers[0].parameters[0] -parsers[1].classname com.yahoo.mail.vespa.spooler.MailFileParser -parsers[1].parameters[0] -parsers[2].classname com.yahoo.mail.vespa.spooler.UserDeleteParser -parsers[2].parameters[0] -parsers[3].classname com.yahoo.mail.vespa.spooler.VespaGrimParser -parsers[3].parameters[1] -parsers[3].parameters[0].key chunksize -parsers[3].parameters[0].value 5 diff --git a/vespaclient-core/CMakeLists.txt b/vespaclient-core/CMakeLists.txt index facea0b447d..218276cf590 100644 --- a/vespaclient-core/CMakeLists.txt +++ b/vespaclient-core/CMakeLists.txt @@ -1,3 +1,2 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. install_config_definition(src/main/resources/configdefinitions/feeder.def vespaclient.config.feeder.def) -install_config_definition(src/main/resources/configdefinitions/spooler.def vespa.config.content.spooler.spooler.def) diff --git a/vespaclient-core/src/main/resources/configdefinitions/spooler.def b/vespaclient-core/src/main/resources/configdefinitions/spooler.def deleted file mode 100644 index a03010b0f32..00000000000 --- a/vespaclient-core/src/main/resources/configdefinitions/spooler.def +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=vespa.config.content.spooler - -# The root directory under which the spooler will scan for content files -directory string default="var/spool/vespa" - -# If true, move successfully processed files to <directory>/success -keepsuccess bool default=false - -# How many bytes to store maximum in the failures directory (delete files instead of moving to failures if this is exceeded) -# If 0, don't restrict -maxfailuresize int default=0 -maxfatalfailuresize int default=0 - -# How many bytes to store maximum in the successes directory (delete files instead of moving to successes if this is exceeded) -# If 0, don't restrict -maxsuccesssize int default=0 - -# How many worker threads to use -threads int default=10 - -# Which parsers to use and config for each of them. -parsers[].classname string -parsers[].parameters[].key string -parsers[].parameters[].value string - -# The time spent in failures (seconds) until moving to fatalfailures (default 3 days) -maxfailuretime int default=259200 - -# The minimum number of retries before moving to fatalfailures -minfailureretries int default=5 - -# The maximum number of retries when getting transient errors sending messages to backend -maxretries int default=5 diff --git a/vespaclient/CMakeLists.txt b/vespaclient/CMakeLists.txt index ed61d730629..8be8751f2c9 100644 --- a/vespaclient/CMakeLists.txt +++ b/vespaclient/CMakeLists.txt @@ -15,7 +15,6 @@ vespa_define_module( src/vespa/vespaclient/clusterlist APPS - src/vespa/vespaclient/spoolmaster src/vespa/vespaclient/vdsstates src/vespa/vespaclient/vesparoute ) diff --git a/vespaclient/src/vespa/vespaclient/spoolmaster/.gitignore b/vespaclient/src/vespa/vespaclient/spoolmaster/.gitignore deleted file mode 100644 index 81b1279cdb1..00000000000 --- a/vespaclient/src/vespa/vespaclient/spoolmaster/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -.depend -Makefile -vespa-spoolmaster diff --git a/vespaclient/src/vespa/vespaclient/spoolmaster/CMakeLists.txt b/vespaclient/src/vespa/vespaclient/spoolmaster/CMakeLists.txt deleted file mode 100644 index 5e5d8b17ff3..00000000000 --- a/vespaclient/src/vespa/vespaclient/spoolmaster/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(vespaclient_spoolmaster_app - SOURCES - main.cpp - application.cpp - OUTPUT_NAME vespa-spoolmaster - INSTALL bin - DEPENDS -) diff --git a/vespaclient/src/vespa/vespaclient/spoolmaster/application.cpp b/vespaclient/src/vespa/vespaclient/spoolmaster/application.cpp deleted file mode 100644 index 7406e17430d..00000000000 --- a/vespaclient/src/vespa/vespaclient/spoolmaster/application.cpp +++ /dev/null @@ -1,199 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/defaults.h> -#include <thread> -#include <iostream> -#include <algorithm> -#include <dirent.h> -#include <unistd.h> -#include <cstring> -#include <sys/stat.h> - -#include "application.h" - -namespace { - -std::string masterInbox() { - return vespa::Defaults::underVespaHome("var/spool/master/inbox"); -} - -std::string outboxParent() { - return vespa::Defaults::underVespaHome("var/spool/vespa"); -} - -} - -namespace spoolmaster { - -Application::Application() - : _masterInbox(masterInbox()), - _inboxFiles(), - _outboxParentDir(outboxParent()), - _outboxes() -{ - // empty -} - -Application::~Application() -{ - // empty -} - -bool -Application::scanInbox() -{ - std::vector<std::string> rv; - DIR *d = opendir(_masterInbox.c_str()); - if (d == NULL) { - perror(_masterInbox.c_str()); - mkdir(_masterInbox.c_str(), 0775); - return false; - } - - struct dirent *entry; - while ((entry = readdir(d)) != NULL) { - if (strcmp(entry->d_name, ".") == 0) continue; - if (strcmp(entry->d_name, "..") == 0) continue; - - std::string fn = _masterInbox; - fn.append("/"); - fn.append(entry->d_name); - - struct stat sb; - if (stat(fn.c_str(), &sb) == 0) { - if (S_ISREG(sb.st_mode)) { - rv.push_back(fn); - } - } else { - perror(fn.c_str()); - } - } - closedir(d); - - if (access(_masterInbox.c_str(), W_OK) < 0) { - perror(_masterInbox.c_str()); - return false; - } - - _inboxFiles = rv; - return (rv.size() > 0); -} - -bool -Application::findOutboxes() -{ - std::vector<std::string> rv; - DIR *d = opendir(_outboxParentDir.c_str()); - if (d == NULL) { - perror(_outboxParentDir.c_str()); - return false; - } - struct dirent *entry; - while ((entry = readdir(d)) != NULL) { - if (strcmp(entry->d_name, ".") == 0) continue; - if (strcmp(entry->d_name, "..") == 0) continue; - - /* XXX: should check if d_name starts with "colo." ? */ - - std::string fn = _outboxParentDir; - fn.append("/"); - fn.append(entry->d_name); - fn.append("/inbox"); - - if (fn == _masterInbox) continue; - - struct stat sb; - if (stat(fn.c_str(), &sb) == 0) { - if (S_ISDIR(sb.st_mode)) { - if (access(fn.c_str(), W_OK) < 0) { - std::cerr << "Cannot write to directory "; - perror(fn.c_str()); - continue; - } - rv.push_back(fn); - } - } else { - perror(fn.c_str()); - } - } - closedir(d); - if (rv.size() > 0) { - std::sort(rv.begin(), rv.end()); - sviter_t ni = rv.begin(); - sviter_t oi = _outboxes.begin(); - - while (ni != rv.end()) { - const std::string &newval = *ni; - if (oi == _outboxes.end()) { - std::cerr << "Found new slave inbox: " << newval << std::endl; - ++ni; - continue; - } - const std::string &oldval = *oi; - if (newval == oldval) { - ++ni; - ++oi; - } else if (newval < oldval) { - std::cerr << "Found new slave inbox: " << newval << std::endl; - ++ni; - } else /* oldval < newval */ { - std::cerr << "Slave inbox removed: " << oldval << std::endl; - ++oi; - } - } - _outboxes = rv; - return true; - } - std::cerr << "Did not find any slave inboxes in: " << _outboxParentDir << std::endl; - return false; -} - -void -Application::moveLinks() -{ - for (sviter_t fni = _inboxFiles.begin(); fni != _inboxFiles.end(); ++fni) { - const std::string& filename = *fni; - size_t ldp = filename.rfind("/"); - std::string basename = filename.substr(ldp+1); - for (sviter_t obi = _outboxes.begin(); obi != _outboxes.end(); ++obi) { - std::string newFn = *obi; - newFn.append("/"); - newFn.append(basename); - - std::cout << "linking " << filename << " -> " << newFn << std::endl; - if (link(filename.c_str(), newFn.c_str()) < 0) { - std::cerr << "linking " << filename << " -> " << newFn; - perror("failed"); - return; - } - } - if (unlink(filename.c_str()) < 0) { - std::cerr << "cannot remove " << filename; - perror(", error"); - } - } -} - - -int -Application::Main() -{ - bool aborted = false; - findOutboxes(); - - try { - while (!aborted) { - if (scanInbox() && findOutboxes()) { - moveLinks(); - } else { - std::this_thread::sleep_for(std::chrono::milliseconds(200)); - } - } - } - catch(std::exception &e) { - fprintf(stderr, "ERROR: %s\n", e.what()); - return EXIT_FAILURE; - } - return EXIT_SUCCESS; -} - -} // namespace spoolmaster diff --git a/vespaclient/src/vespa/vespaclient/spoolmaster/application.h b/vespaclient/src/vespa/vespaclient/spoolmaster/application.h deleted file mode 100644 index 0cd8e1a917a..00000000000 --- a/vespaclient/src/vespa/vespaclient/spoolmaster/application.h +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include <vespa/fastos/app.h> -#include <string> -#include <vector> - -namespace spoolmaster { -/** - * main spoolmaster application class - */ -class Application : public FastOS_Application { -private: - std::string _masterInbox; - std::vector<std::string> _inboxFiles; - - std::string _outboxParentDir; - std::vector<std::string> _outboxes; - - typedef std::vector<std::string>::iterator sviter_t; - - bool scanInbox(); - bool findOutboxes(); - void moveLinks(); -public: - Application(); - ~Application(); - int Main() override; -}; - -} diff --git a/vespaclient/src/vespa/vespaclient/spoolmaster/main.cpp b/vespaclient/src/vespa/vespaclient/spoolmaster/main.cpp deleted file mode 100644 index 40a262734bf..00000000000 --- a/vespaclient/src/vespa/vespaclient/spoolmaster/main.cpp +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "application.h" - -int -main(int argc, char** argv) -{ - spoolmaster::Application *app = new spoolmaster::Application(); - int retVal = app->Entry(argc, argv); - delete app; - return retVal; -} - |