diff options
Diffstat (limited to 'config-model/src')
17 files changed, 331 insertions, 58 deletions
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java index a69c524e863..5855dedc415 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java @@ -932,8 +932,8 @@ public class RankProfile implements Serializable, Cloneable { public static class Macro implements Serializable, Cloneable { private final String name; - private String textualExpression=null; - private RankingExpression expression=null; + private String textualExpression = null; + private RankingExpression expression = null; private List<String> formalParams = new ArrayList<>(); /** True if this should be inlined into calling expressions. Useful for very cheap macros. */ @@ -998,6 +998,7 @@ public class RankProfile implements Serializable, Cloneable { } public static final class DiversitySettings { + private String attribute = null; private int minGroups = 0; private double cutoffFactor = 10; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java index 07b79c4f604..b11d5962713 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java @@ -212,8 +212,9 @@ public class RawRankProfile implements RankProfilesConfig.Producer { private Map<String, String> deriveMacroProperties(Map<String, ExpressionFunction> eMacros) { SerializationContext context = new SerializationContext(eMacros); for (Map.Entry<String, ExpressionFunction> e : eMacros.entrySet()) { - String script = e.getValue().getBody().getRoot().toString(new StringBuilder(), context, null, null).toString(); - context.addFunctionSerialization(RankingExpression.propertyName(e.getKey()), script); + String expression = e.getValue().getBody().getRoot().toString(new StringBuilder(), context, null, null).toString(); + context.addFunctionSerialization(RankingExpression.propertyName(e.getKey()), expression); + } return context.serializedFunctions(); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Configserver.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Configserver.java index e8f1f59310b..2a32549b6bf 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/Configserver.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Configserver.java @@ -1,16 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin; -import java.io.BufferedReader; -import java.io.InputStream; -import java.io.InputStreamReader; - -import java.util.logging.Logger; - import com.yahoo.config.model.api.ConfigServerSpec; -import com.yahoo.log.LogLevel; import com.yahoo.config.model.producer.AbstractConfigProducer; -import static com.yahoo.vespa.defaults.Defaults.getDefaults; import com.yahoo.vespa.model.AbstractService; /** @@ -26,11 +18,13 @@ import com.yahoo.vespa.model.AbstractService; */ public class Configserver extends AbstractService { private static final long serialVersionUID = 1L; - private static final int defaultPort = 19070; - private static final Logger log = Logger.getLogger(Configserver.class.getName()); + public static final int defaultRpcPort = 19070; - public Configserver(AbstractConfigProducer parent, String name) { + private final int rpcPort; + + public Configserver(AbstractConfigProducer parent, String name, int rpcPort) { super(parent, name); + this.rpcPort = rpcPort; portsMeta.on(0).tag("rpc").tag("config"); portsMeta.on(1).tag("http").tag("config").tag("state"); setProp("clustertype", "admin"); @@ -41,16 +35,7 @@ public class Configserver extends AbstractService { * Returns the desired base port for this service. */ public int getWantedPort() { - try { - // TODO: Provide configserver port as argument when creating this service instead - Process process = new ProcessBuilder(getDefaults().underVespaHome("bin/vespa-print-default"), "configserver_rpc_port").start(); - InputStream in = process.getInputStream(); - BufferedReader reader = new BufferedReader(new InputStreamReader(in)); - return Integer.parseInt(reader.readLine().trim()); - } catch (Exception exception) { - log.log(LogLevel.DEBUG, "Error reading port from script, using " + defaultPort); - return defaultPort; - } + return rpcPort; } /** @@ -84,7 +69,7 @@ public class Configserver extends AbstractService { return getRelativePort(1); } - public ConfigServerSpec getConfigServerSpec() { + ConfigServerSpec getConfigServerSpec() { return new Spec(getHostName(), getConfigServerRpcPort(), getConfigServerHttpPort(), ZooKeepersConfigProvider.zkPort); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index b7c9116dc60..a7a93e75293 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -130,6 +130,10 @@ public class VespaMetricSet { metrics.add(new Metric("http.status.4xx.rate")); metrics.add(new Metric("http.status.5xx.rate")); + metrics.add(new Metric("jdisc.http.request.uri_length.average")); + metrics.add(new Metric("jdisc.http.request.uri_length.max")); + metrics.add(new Metric("jdisc.http.request.content_size.average")); + metrics.add(new Metric("jdisc.http.request.content_size.max")); return metrics; } @@ -169,6 +173,7 @@ public class VespaMetricSet { metrics.add(new Metric("active_queries.average", "active_queries")); metrics.add(new Metric("feed.latency.average")); metrics.add(new Metric("queries.rate", "queries")); + metrics.add(new Metric("query_container_latency.average")); metrics.add(new Metric("query_latency.average", "mean_query_latency")); metrics.add(new Metric("query_latency.max", "max_query_latency")); metrics.add(new Metric("query_latency.95percentile", "95p_query_latency")); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java index 96e120cae92..297c4a6e9b9 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java @@ -34,7 +34,7 @@ public abstract class DomAdminBuilderBase extends VespaDomBuilder.DomConfigProdu private static final String DEFAULT_CLUSTER_NAME = "vespa"; private final ApplicationType applicationType; - private final List<ConfigServerSpec> configServerSpecs; + protected final List<ConfigServerSpec> configServerSpecs; private final FileRegistry fileRegistry; protected final boolean multitenant; @@ -52,7 +52,7 @@ public abstract class DomAdminBuilderBase extends VespaDomBuilder.DomConfigProdu HostSystem hostSystem = parent.getHostSystem(); HostResource host = new HostResource(Host.createConfigServerHost(hostSystem, spec.getHostName())); hostSystem.addBoundHost(host); - Configserver configserver = new Configserver(parent, spec.getHostName()); + Configserver configserver = new Configserver(parent, spec.getHostName(), spec.getConfigServerPort()); configserver.setHostResource(host); configserver.setBasePort(configserver.getWantedPort()); configserver.initService(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java index da8c73c0958..75cd755a91d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java @@ -23,6 +23,7 @@ import org.w3c.dom.Element; import java.util.List; import java.util.ArrayList; +import java.util.Objects; import java.util.logging.Level; /** @@ -102,7 +103,7 @@ public class DomAdminV2Builder extends DomAdminBuilderBase { return cluster; } - // Extra stupid because configservers tag is voluntary + // Extra stupid because configservers tag is optional private List<Configserver> getConfigServers(AbstractConfigProducer parent, Element adminE) { SimpleConfigProducer configServers = new SimpleConfigProducer(parent, "configservers"); List<Configserver> cfgs = new ArrayList<>(); @@ -114,7 +115,7 @@ public class DomAdminV2Builder extends DomAdminBuilderBase { } else { parent.deployLogger().log(LogLevel.INFO, "Specifying configserver without parent element configservers in services.xml is deprecated"); } - Configserver cfgs0 = new ConfigserverBuilder(0).build(configServers, configserverE); + Configserver cfgs0 = new ConfigserverBuilder(0, configServerSpecs).build(configServers, configserverE); cfgs0.setProp("index", 0); cfgs.add(cfgs0); return cfgs; @@ -122,7 +123,7 @@ public class DomAdminV2Builder extends DomAdminBuilderBase { // configservers tag in use int i = 0; for (Element configserverE : XML.getChildren(configserversE, "configserver")) { - Configserver cfgsrv = new ConfigserverBuilder(i).build(configServers, configserverE); + Configserver cfgsrv = new ConfigserverBuilder(i, configServerSpecs).build(configServers, configserverE); cfgsrv.setProp("index", i); cfgs.add(cfgsrv); i++; @@ -161,16 +162,21 @@ public class DomAdminV2Builder extends DomAdminBuilderBase { } private static class ConfigserverBuilder extends DomConfigProducerBuilder<Configserver> { - int i; + private final int i; + private final int rpcPort; - public ConfigserverBuilder(int i) { + public ConfigserverBuilder(int i, List<ConfigServerSpec> configServerSpec) { this.i = i; + Objects.requireNonNull(configServerSpec); + if (configServerSpec.size() > 0) + this.rpcPort = configServerSpec.get(0).getConfigServerPort(); + else + this.rpcPort = Configserver.defaultRpcPort; } @Override - protected Configserver doBuild(AbstractConfigProducer parent, - Element spec) { - return new Configserver(parent, "configserver." + i); + protected Configserver doBuild(AbstractConfigProducer parent, Element spec) { + return new Configserver(parent, "configserver." + i, rpcPort); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomComponentBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomComponentBuilder.java index da643c687d9..81f19f00fbb 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomComponentBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomComponentBuilder.java @@ -36,7 +36,7 @@ public class DomComponentBuilder extends VespaDomBuilder.DomConfigProducerBuilde private Component buildComponent(Element spec) { BundleInstantiationSpecification bundleSpec = - BundleInstantiationSpecificationBuilder.build(spec, false).nestInNamespace(namespace); + BundleInstantiationSpecificationBuilder.build(spec).nestInNamespace(namespace); return new Component<Component<?, ?>, ComponentModel>(new ComponentModel(bundleSpec)); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomFilterBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomFilterBuilder.java index 2c6364c4c34..74cc2b8421c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomFilterBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomFilterBuilder.java @@ -13,6 +13,6 @@ import org.w3c.dom.Element; public class DomFilterBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Component> { @Override protected Component doBuild(AbstractConfigProducer ancestor, Element element) { - return new HttpFilter(BundleInstantiationSpecificationBuilder.build(element, false)); + return new HttpFilter(BundleInstantiationSpecificationBuilder.build(element)); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java index 23f65fd9fb1..28a4b9dfea5 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java @@ -12,18 +12,8 @@ import org.w3c.dom.Element; /** * @author gjoranv - * @since 5.1.6 */ public class DomHandlerBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Handler> { - private final boolean legacyMode; - - public DomHandlerBuilder(boolean legacyMode) { - this.legacyMode = legacyMode; - } - - public DomHandlerBuilder() { - this(false); - } @Override protected Handler doBuild(AbstractConfigProducer ancestor, Element handlerElement) { @@ -41,7 +31,7 @@ public class DomHandlerBuilder extends VespaDomBuilder.DomConfigProducerBuilder< } protected Handler<? super Component<?, ?>> getHandler(Element handlerElement) { - BundleInstantiationSpecification bundleSpec = BundleInstantiationSpecificationBuilder.build(handlerElement, legacyMode); + BundleInstantiationSpecification bundleSpec = BundleInstantiationSpecificationBuilder.build(handlerElement); return new Handler<>( new ComponentModel(bundleSpec)); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/ServletBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/ServletBuilder.java index 3d779289340..d2d9e9d2ca6 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/ServletBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/ServletBuilder.java @@ -27,7 +27,7 @@ public class ServletBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Ser } private SimpleComponent createServletComponent(Element servletElement) { - ComponentModel componentModel = new ComponentModel(BundleInstantiationSpecificationBuilder.build(servletElement, false)); + ComponentModel componentModel = new ComponentModel(BundleInstantiationSpecificationBuilder.build(servletElement)); return new SimpleComponent(componentModel); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/ChainedComponentModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/ChainedComponentModelBuilder.java index 700b042a745..68b37d938a0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/ChainedComponentModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/ChainedComponentModelBuilder.java @@ -15,7 +15,7 @@ public class ChainedComponentModelBuilder extends GenericChainedComponentModelBu public ChainedComponentModelBuilder(Element spec) { super(spec); - bundleInstantiationSpec = BundleInstantiationSpecificationBuilder.build(spec, false); + bundleInstantiationSpec = BundleInstantiationSpecificationBuilder.build(spec); } public ChainedComponentModel build() { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/GUIHandler.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/GUIHandler.java new file mode 100644 index 00000000000..025075be8fd --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/GUIHandler.java @@ -0,0 +1,27 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.search; + +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.container.bundle.BundleInstantiationSpecification; +import com.yahoo.osgi.provider.model.ComponentModel; +import com.yahoo.vespa.model.container.component.Handler; + + +/** + * @author Henrik Høiness + */ + +public class GUIHandler extends Handler<AbstractConfigProducer<?>> { + public static final String BUNDLE = "container-search-gui"; + public static final String CLASS = "com.yahoo.search.query.gui.GUIHandler"; + public static final String BINDING = "*/querybuilder/*"; + + public GUIHandler() { + super(new ComponentModel(bundleSpec(CLASS, BUNDLE))); + } + + public static BundleInstantiationSpecification bundleSpec(String className, String bundle) { + return BundleInstantiationSpecification.getFromStrings(className, className, bundle); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilder.java index 69ee9048b28..2828fcf09d0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilder.java @@ -4,11 +4,10 @@ package com.yahoo.vespa.model.container.xml; import com.yahoo.config.model.builder.xml.XmlHelper; import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.component.ComponentSpecification; -import com.yahoo.processing.handler.ProcessingHandler; -import com.yahoo.search.handler.SearchHandler; import org.w3c.dom.Element; -import java.util.*; +import java.util.Arrays; +import java.util.List; /** * This object builds a bundle instantiation spec from an XML element. @@ -17,14 +16,13 @@ import java.util.*; */ public class BundleInstantiationSpecificationBuilder { - public static BundleInstantiationSpecification build(Element spec, boolean legacyMode) { + public static BundleInstantiationSpecification build(Element spec) { ComponentSpecification id = XmlHelper.getIdRef(spec); ComponentSpecification classId = getComponentSpecification(spec, "class"); ComponentSpecification bundle = getComponentSpecification(spec, "bundle"); BundleInstantiationSpecification instSpec = new BundleInstantiationSpecification(id, classId, bundle); - if ( ! legacyMode) // TODO: Remove? - validate(instSpec); + validate(instSpec); return bundle == null ? setBundleForKnownClass(instSpec) : instSpec; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index a007c4765c0..d81026c54d1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -23,6 +23,7 @@ import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.Rotation; import com.yahoo.config.provision.Zone; import com.yahoo.container.jdisc.config.MetricDefaultsConfig; +import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.search.rendering.RendererRegistry; import com.yahoo.text.XML; import com.yahoo.vespa.defaults.Defaults; @@ -47,6 +48,7 @@ import com.yahoo.vespa.model.container.IdentityProvider; import com.yahoo.vespa.model.container.SecretStore; import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.container.component.FileStatusHandlerComponent; +import com.yahoo.vespa.model.container.component.Handler; import com.yahoo.vespa.model.container.component.chain.ProcessingHandler; import com.yahoo.vespa.model.container.docproc.ContainerDocproc; import com.yahoo.vespa.model.container.docproc.DocprocChains; @@ -55,6 +57,7 @@ import com.yahoo.vespa.model.container.http.xml.HttpBuilder; import com.yahoo.vespa.model.container.jersey.xml.RestApiBuilder; import com.yahoo.vespa.model.container.processing.ProcessingChains; import com.yahoo.vespa.model.container.search.ContainerSearch; +import com.yahoo.vespa.model.container.search.GUIHandler; import com.yahoo.vespa.model.container.search.PageTemplates; import com.yahoo.vespa.model.container.search.QueryProfiles; import com.yahoo.vespa.model.container.search.SemanticRules; @@ -378,6 +381,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { cluster.setSearch(buildSearch(cluster, searchElement, queryProfiles, semanticRules)); addSearchHandler(cluster, searchElement); + addGUIHandler(cluster); validateAndAddConfiguredComponents(cluster, searchElement, "renderer", ContainerModelBuilder::validateRendererElement); } } @@ -670,6 +674,13 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { cluster.addComponent(searchHandler); } + private void addGUIHandler(ContainerCluster cluster) { + Handler<?> guiHandler = new GUIHandler(); + guiHandler.addServerBindings("http://"+GUIHandler.BINDING, "https://"+GUIHandler.BINDING); + cluster.addComponent(guiHandler); + } + + private String[] serverBindings(Element searchElement, String... defaultBindings) { List<Element> bindings = XML.getChildren(searchElement, "binding"); if (bindings.isEmpty()) diff --git a/config-model/src/main/python/ES_Vespa_parser.py b/config-model/src/main/python/ES_Vespa_parser.py new file mode 100644 index 00000000000..477b0db4744 --- /dev/null +++ b/config-model/src/main/python/ES_Vespa_parser.py @@ -0,0 +1,222 @@ +# Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +import json +import argparse +import os, sys + +# Parsing Elastic Search documents to Vespa documents +# Example of usage: python ES_Vespa_parser.py my_index.json my_index_mapping.json +__author__ = 'henrhoi' + + +class ElasticSearchParser: + document_file = None + mapping_file = None + application_name = None + search_definitions = {} + path = "" + _all = True + all_mappings = {} + no_index = [] + types = [] + + def __init__(self): + parser = argparse.ArgumentParser() + parser.add_argument("documents_path", help="location of file with documents to be parsed", type=str) + parser.add_argument("mappings_path", help="location of file with mappings", type=str) + parser.add_argument("--application_name", help="name of application", default="application_name", type=str) + args = parser.parse_args() + + self.document_file = args.documents_path + self.mapping_file = args.mappings_path + self.application_name = args.application_name + + def main(self): + self.path = os.getcwd() + "/application/" + try: + os.mkdir(self.path, 0o777) + print(" > Created folder '" + self.path + "'") + except OSError: + print(" > Folder '" + self.path + "' already existed") + + try: + os.makedirs(self.path + "searchdefinitions/", 0o777) + print(" > Created folder '" + self.path + "searchdefinitions/" + "'") + except OSError: + print(" > Folder '" + self.path + "searchdefinitions/" + "' already existed") + + self.parse() + self.createServices_xml() + self.createHosts_xml() + + def getMapping(self, type): + unparsed_mapping_file = open(self.mapping_file, "r") + type_mapping = {} + for line in unparsed_mapping_file: + data = json.loads(line) + index = list(data.keys())[0] + mappings = data[index]["mappings"][type]["properties"] + + # Checking if some fields could be no-index + try: + _all_enabled = data[index]["mappings"][type]["_all"]["enabled"] + if not _all_enabled: + self._all = False + print(" > All fields in the document type '" + type + "' is not searchable. Go inside " + self.path + type + ".sd to add which fields that should be searchable") + except KeyError: + print(" > All fields in the document type '" + type + "' is searchable") + + self.walk(mappings, type_mapping, "properties") + + unparsed_mapping_file.close() + if type not in self.search_definitions: + self.search_definitions[type] = True + self.types.append(type) + self.createSearchDefinition(type, type_mapping) + + # Adding mapping to global map with mappings + self.all_mappings[type] = type_mapping + return type_mapping + + def parse(self): + file_path = self.path + "documents" + ".json" + unparsed_document_file = open(self.document_file, "r") + vespa_docs = open(file_path, "w") + + for line in unparsed_document_file: + data = json.loads(line) + type = data["_type"] + + parsed_data = { + "put": "id:"+self.application_name+":" + type + "::" + data["_id"], + "fields": {} + } + + # Checking for already existing mapping for a type, if not create a new + if type in self.all_mappings: + mapping = self.all_mappings[type] + else: + mapping = self.getMapping(type) + + for key, item in mapping.items(): + try: + parsed_data["fields"][key] = data["_source"][key] + except KeyError: + continue + + json.dump(parsed_data, vespa_docs) + vespa_docs.write("\n") + + vespa_docs.close() + unparsed_document_file.close() + print(" > Parsed all documents '" + ", ".join(self.types) + "'" + "' at '" + file_path + "'") + + def createSearchDefinition(self, type, type_mapping): + file_path = self.path + "searchdefinitions/" + type + ".sd" + new_sd = open(file_path, "w") + new_sd.write("search " + type + " {\n") + new_sd.write(" document " + type + " {\n") + + for key, item in type_mapping.items(): + new_sd.write(" field " + key + " type " + self.get_type(item) + " {\n") + new_sd.write(" indexing: " + self.get_indexing(key, self.get_type(item)) + "\n") + new_sd.write(" }\n") + + new_sd.write(" }\n") + new_sd.write("}\n") + new_sd.close() + print(" > Created search definition for '" + type + "' at '" + file_path + "'") + + def createServices_xml(self): + file_path = self.path + "services.xml" + new_services = open(file_path, "w") + template = ("<?xml version='1.0' encoding='UTF-8'?>" + "<services version='1.0'>\n\n" + " <container id='default' version='1.0'>\n" + " <search/>\n" + " <document-api/>\n" + " <nodes>\n" + " <node hostalias='node1'/>\n" + " </nodes>\n" + " </container>\n\n" + " <content id='content' version='1.0'>\n" + " <redundancy>1</redundancy>\n" + " <search>\n" + " <visibility-delay>1.0</visibility-delay>\n" + " </search>\n" + " <documents>\n") + + for i in range(0, len(self.types)): + template += " <document mode='index' type='" + self.types[i] + "'/>\n" + + template += (" </documents>\n" + " <nodes>\n" + " <node hostalias='node1' distribution-key=\"0\"/>\n" + " </nodes>\n" + " <engine>\n" + " <proton>\n" + " <searchable-copies>1</searchable-copies>\n" + " </proton>\n" + " </engine>\n" + " </content>\n\n" + "</services>") + + new_services.write(template) + new_services.close() + print(" > Created services.xml at '" + file_path + "'") + + def createHosts_xml(self): + file_path = self.path + "hosts.xml" + new_hosts = open(file_path, "w") + template = ("<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n" + "<hosts>\n" + " <host name=\"localhost\">\n" + " <alias>node1</alias>\n" + " </host>\n" + "</hosts>") + + new_hosts.write(template) + new_hosts.close() + print(" > Created hosts.xml at '" + file_path + "'") + + def get_type(self, type): + return { + "text": "string", + "keyword": "string", + "date": "string", + "long": "long", + "double": "double", + "boolean": "string", + "ip": "text", + "byte": "byte", + "float": "float", + + }[type] + + def get_indexing(self, key, key_type): + if not self._all: + return "summary" + + if key not in self.no_index: + if key_type == "string": + return "summary | index" + else: + return "summary | attribute" + + return "summary" + + def walk(self, node, mapping, parent): + for key, item in node.items(): + if isinstance(item, dict): + self.walk(item, mapping, key) + elif key == "type": + mapping[parent] = item + elif key == "include_in_all": + if not item: # Field should not be searchable + self.no_index.append(parent) + elif key == "index" and parent != "properties": + if item == "no": # Field should not be searchable + self.no_index.append(parent) + + +if __name__ == '__main__': + ElasticSearchParser().main() diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilderTest.java index c58f3308ced..a651bbb7772 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilderTest.java @@ -53,7 +53,7 @@ public class BundleInstantiationSpecificationBuilderTest { InputStream xmlStream = IOUtils.toInputStream(xml); Element component = XmlHelper.getDocumentBuilder().parse(xmlStream).getDocumentElement(); - BundleInstantiationSpecification spec = BundleInstantiationSpecificationBuilder.build(component, false); + BundleInstantiationSpecification spec = BundleInstantiationSpecificationBuilder.build(component); assertThat(spec.bundle, is(ComponentSpecification.fromString(expectedBundle))); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/SearchBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/SearchBuilderTest.java index cc8733fc9e2..30f1df6a394 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/SearchBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/SearchBuilderTest.java @@ -8,6 +8,8 @@ import com.yahoo.searchdefinition.parser.ParseException; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.container.Container; import com.yahoo.vespa.model.container.ContainerCluster; +import com.yahoo.vespa.model.container.component.Handler; +import com.yahoo.vespa.model.container.search.GUIHandler; import com.yahoo.vespa.model.test.utils.ApplicationPackageUtils; import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; import org.junit.Test; @@ -32,6 +34,31 @@ public class SearchBuilderTest extends ContainerModelBuilderTestBase { return root.getConfig(ChainsConfig.class, "default/component/com.yahoo.search.handler.SearchHandler"); } + @Test + public void gui_search_handler_is_always_included_when_search_is_specified() throws Exception{ + Element clusterElem = DomBuilderTest.parse( + "<jdisc id='default' version='1.0'>", + " <search />", + nodesXml, + "</jdisc>"); + + createModel(root, clusterElem); + + String discBindingsConfig = root.getConfig(JdiscBindingsConfig.class, "default").toString(); + assertThat(discBindingsConfig, containsString(GUIHandler.BINDING)); + + ContainerCluster cluster = (ContainerCluster)root.getChildren().get("default"); + + GUIHandler guiHandler = null; + for (Handler<?> handler : cluster.getHandlers()) { + if (handler instanceof GUIHandler) { + guiHandler = (GUIHandler) handler; + } + } + if (guiHandler == null) fail(); + } + + @Test public void search_handler_bindings_can_be_overridden() throws Exception { |